Compare commits
384 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5a182f3f16 | |||
| 889b843d6d | |||
| 9b91119239 | |||
| 1e74d67da9 | |||
| ed013bbdda | |||
| 4b40619d1d | |||
| 3ce1f2b036 | |||
| 673f045212 | |||
| df640e3684 | |||
| 59bea9d888 | |||
| e2c91afda2 | |||
| eb47c4d69a | |||
| 91d56f2bbb | |||
| 12b847360c | |||
| 2e736f40ad | |||
| 9c3e5ecb30 | |||
| 27ce316206 | |||
| 7280b61d28 | |||
| 79f1ecd378 | |||
| b6791d37e0 | |||
| 4f1b4f8c03 | |||
| 869580c9df | |||
| 1a96cd20da | |||
| 8e637cf86a | |||
| 93c6e1facb | |||
| 4a4bbc056a | |||
| c59b366560 | |||
| d0aded5401 | |||
| 8f9c683b7c | |||
| e82878af57 | |||
| d60e858e95 | |||
| f6ce9ae0a0 | |||
| b4f26401f5 | |||
| 2b5004fb33 | |||
| 00f4cb1f26 | |||
| e091e50862 | |||
| 27ce07b111 | |||
| f06fe5afc0 | |||
| 37e920237c | |||
| 7279650772 | |||
| d8ab70b166 | |||
| fb8703a4c6 | |||
| ac3fa8b2b6 | |||
| efcb4f9ca1 | |||
| 7a7a31e8e3 | |||
| 3fdcbabd48 | |||
| b2e8c7627f | |||
| bd08d13011 | |||
| 10c69f592e | |||
| fe0b2dc7fe | |||
| a0f007fd43 | |||
| 1b27babd17 | |||
| 100d2242fa | |||
| 089b1b38fd | |||
| ac465839a5 | |||
| ef280b7bb5 | |||
| 233737f333 | |||
| 909904e928 | |||
| 59ab47de8c | |||
| f252a0709f | |||
| d6d9638981 | |||
| 020248fbda | |||
| a06a0dbfff | |||
| 96c24e0688 | |||
| c156350183 | |||
| 15b25ff509 | |||
| 5163d7d6a0 | |||
| 9fd343d328 | |||
| aef88f3337 | |||
| 12af1d6c76 | |||
| 5433bff3db | |||
| 355509aa7c | |||
| 7fb78a7e29 | |||
| bfd8d58142 | |||
| 8ad472d1bc | |||
| fc5e2cd751 | |||
| 9540791441 | |||
| 058da1627f | |||
| cb9b3fe404 | |||
| 12315b2f2d | |||
| 0a4a051866 | |||
| f47d961947 | |||
| c701d0d694 | |||
| d4a3edf849 | |||
| e164eac1eb | |||
| c81591b498 | |||
| d8c03d9d95 | |||
| ffb133f0b9 | |||
| b3fe321f0d | |||
| 69d210c51d | |||
| 06a88e9897 | |||
| aee30f68e8 | |||
| 3db9a884de | |||
| 0037b8cb12 | |||
| 205b13dbb9 | |||
| b938a03367 | |||
| d474c2be9f | |||
| 96390beeb9 | |||
| fe63a0a352 | |||
| 2e36e9c234 | |||
| b83aa6d97d | |||
| f42fcbcb3c | |||
| f70ba354f8 | |||
| fc17047290 | |||
| 7bbc2cbddf | |||
| 9a78daa880 | |||
| 5d532d3dbc | |||
| a22c26eb9d | |||
| b84c467255 | |||
| 53124e94a6 | |||
| 838068fd31 | |||
| 71282f3556 | |||
| 7080dcc45a | |||
| 5a62714dc2 | |||
| c930d18f1e | |||
| 583c81c9a8 | |||
| 483ec8b8e0 | |||
| 328446864a | |||
| 106480ef52 | |||
| d3a45f958d | |||
| 6657c6b9c0 | |||
| 6c5e2fd921 | |||
| 65842b87bf | |||
| b323c52364 | |||
| 321a6efc81 | |||
| 5ecb2a998b | |||
| bb6df93b51 | |||
| 096a049deb | |||
| b9caf524c3 | |||
| 197de1d718 | |||
| 18d4f08f2b | |||
| ddf450d87d | |||
| e3b2113365 | |||
| 6a85f45604 | |||
| a2d1aedcad | |||
| e955faa97e | |||
| 2b755bf99a | |||
| ce1162a055 | |||
| 447786b447 | |||
| 2b4dbe087a | |||
| 6804bcfd4e | |||
| d2221eb0b9 | |||
| 9ac6de7c7d | |||
| f5613be495 | |||
| 819b6c8e4f | |||
| 9c1fb07f33 | |||
| fa31872903 | |||
| b08b79eafe | |||
| e43d916c24 | |||
| 14b185db35 | |||
| baa51102fd | |||
| 5577f5e7e5 | |||
| 61892592e4 | |||
| 1e2ca46946 | |||
| 8224c8f1a8 | |||
| d7f39895f5 | |||
| 790bd35385 | |||
| 34c9fd42fe | |||
| 19a336870c | |||
| 5ba5942d0b | |||
| aa20d9102c | |||
| 097be1700b | |||
| f1457d823d | |||
| 12e66299e6 | |||
| e9a01de9da | |||
| 5f793ea01a | |||
| e86f7efed3 | |||
| 78bfad0ec6 | |||
| f5efadfd07 | |||
| c26eca2f01 | |||
| 9f1879a9f3 | |||
| c24c7bf7ef | |||
| 5fac3d6004 | |||
| 7605d2e9f5 | |||
| 7bf5412d63 | |||
| b53ceb1a7c | |||
| 647581b4ff | |||
| db9b1995bc | |||
| 7563435837 | |||
| e39f2abb61 | |||
| 47dad7143f | |||
| d81f41dc72 | |||
| cc6be10532 | |||
| 0ac548bfce | |||
| 9c1d0586ef | |||
| 21ec5e4773 | |||
| e4a5a6654d | |||
| 01bc35aa97 | |||
| e33d85c01d | |||
| 4e73ba8900 | |||
| 4df52b3da9 | |||
| 844d86410c | |||
| 9514e1033d | |||
| b2236bf65f | |||
| 31535b07e0 | |||
| 066b6e6f04 | |||
| a1e2b76a9f | |||
| 29e5b8251d | |||
| 02168a5c55 | |||
| a6b6661320 | |||
| 0c72666a7a | |||
| 9023321db9 | |||
| 7ef097529c | |||
| e7df69efe4 | |||
| 2c7efa2733 | |||
| 2742549827 | |||
| 3c45f5cb09 | |||
| d5d1d7bfbd | |||
| f4e2088b69 | |||
| 55425058a8 | |||
| eec0d17607 | |||
| 43f5304ba4 | |||
| b21472dd8c | |||
| e3991d55be | |||
| 864d1ed31e | |||
| 76ef1e53ca | |||
| 7dd298f04e | |||
| 66452621c6 | |||
| 0b23d93957 | |||
| 4281a6d81b | |||
| fe919a51aa | |||
| 0dcf0eb503 | |||
| e2fb99d5d8 | |||
| a3ee613b60 | |||
| d2197c6a57 | |||
| e5997bc50e | |||
| 75138016ce | |||
| fc4772efb0 | |||
| 0bde428fa0 | |||
| 16ca3a059c | |||
| 1bfc80e6db | |||
| fa705a2ef6 | |||
| 81f807e124 | |||
| a4f7c88785 | |||
| 54259f070f | |||
| 26cef3ab06 | |||
| 00ad230ffe | |||
| 3d920a8113 | |||
| f6eac1964a | |||
| e1b5b9d163 | |||
| 094833f831 | |||
| 3579ce0add | |||
| 0188860d1c | |||
| 871a1ac059 | |||
| 298a182fbb | |||
| e71223507c | |||
| 02187f0f3c | |||
| 3e2f62daeb | |||
| c8b8a8dbb4 | |||
| 9e4c5f11c5 | |||
| b16e72c10e | |||
| 5cb9b0b30b | |||
| c925001a8a | |||
| e494334720 | |||
| 1554540431 | |||
| fef09690d2 | |||
| 3a975fa74f | |||
| 69c6353fb1 | |||
| 2b9bf3cd3f | |||
| d99e16314b | |||
| e4203e0f15 | |||
| 2a5366d9c3 | |||
| 9c9cf24ddc | |||
| ec0d8ad735 | |||
| fc9e9d13e6 | |||
| d0cb7e66f4 | |||
| bd66480ccc | |||
| b05f1f6bcf | |||
| 12ad9c9436 | |||
| c8ce7cdb37 | |||
| 56d4dd8b84 | |||
| 86fdc5aca9 | |||
| 87c5bc59dc | |||
| 21b30b21c2 | |||
| f94f1c0c4e | |||
| a6a9883451 | |||
| 03116a97d1 | |||
| 2abc78ab6d | |||
| 6e54b52376 | |||
| 7cef2ea361 | |||
| 5b5ee0e8fd | |||
| 7bcc4f00b1 | |||
| 34ec44e3d4 | |||
| f607e21c13 | |||
| 5940e9fc6a | |||
| 98cfd76232 | |||
| 09e68a614f | |||
| 88290caadc | |||
| f442dce9a4 | |||
| 4f03f39e9a | |||
| 38b667799c | |||
| a5f74d6abc | |||
| d0e09301a1 | |||
| 177915ea6b | |||
| b89008e745 | |||
| cfa1cbbed2 | |||
| aa1881ec44 | |||
| 744e35ee9b | |||
| 884b74cb8c | |||
| 67316a0fbd | |||
| 501f994c7a | |||
| c38f0cf2b0 | |||
| 729ee56bde | |||
| f0a3f04398 | |||
| 49013c3cc4 | |||
| 32cbc3620e | |||
| ae9c1b0744 | |||
| 9eb8cac507 | |||
| 69ed763640 | |||
| 369650aeeb | |||
| 0280366cc3 | |||
| ac2b52cc4d | |||
| 8faa304c94 | |||
| 95817f1296 | |||
| e6981bad99 | |||
| 99717f03e2 | |||
| 04f53b882c | |||
| 558153f29c | |||
| 91ba9d24a1 | |||
| e8ecb5537c | |||
| 6e4563b2a0 | |||
| 435aef867c | |||
| e76f312adb | |||
| eac470e781 | |||
| 685a86215b | |||
| 8a259241b5 | |||
| fd41dded3d | |||
| db020789e1 | |||
| 186c151046 | |||
| c4c253a03a | |||
| f833882e07 | |||
| 0505e5d5bd | |||
| a4140c4e4c | |||
| 0e928b5506 | |||
| ce9efb6f6c | |||
| 73961f67f2 | |||
| 48fe68b87c | |||
| f1820b2b43 | |||
| 78f810a326 | |||
| 9207b066db | |||
| 0d41f50ace | |||
| b294d2025d | |||
| 12a75ef13d | |||
| 5494fd1862 | |||
| 6c6d746532 | |||
| 7563ec9158 | |||
| 4e1a3c6312 | |||
| 3eb1f39c4b | |||
| 7e82decc01 | |||
| af7850d786 | |||
| 62c7a29b46 | |||
| 78a043db08 | |||
| ec3f0ce237 | |||
| eb47dfb6ce | |||
| 7a2465fecf | |||
| 4b8b2a5a6d | |||
| 4521e248b3 | |||
| 4c32d66a63 | |||
| a635ac3bc6 | |||
| d59286b8b3 | |||
| 05641d4076 | |||
| 7288220341 | |||
| ea7b850ef0 | |||
| e85dd082d6 | |||
| 034aaa03c8 | |||
| 9b6b03dac5 | |||
| fd6f359e1c | |||
| e99a7d67c0 | |||
| 45a1620f6b | |||
| 1955efdae5 | |||
| a5c42de28c | |||
| 19c4354224 | |||
| 8e439176fd | |||
| 73c0fbdd7e | |||
| bffb76625c | |||
| d8ef8b11b0 | |||
| fb49dc9ad2 | |||
| 27bf96f23f | |||
| 0f1f2e593f | |||
| 4b5ae562c7 | |||
| d6783fe327 | |||
| 2fe1095758 | |||
| 8a3b4136a6 | |||
| 7323ca72b2 |
@@ -1,37 +1,43 @@
|
||||
# Agent Directives
|
||||
|
||||
- **Plan first.** Enter plan mode for any non-trivial task (3+ steps or an architectural decision) and write the spec before building. If something goes sideways, stop and re-plan instead of pushing on.
|
||||
- **Plan first.** Enter plan mode for any non-trivial task (3+ steps or an architectural decision) and write the spec before building. The plan file is `internal/work/plans/<slug>.md` from the first draft; the harness plan folder outside the repo is never the storage place, at most a copy for the approval dialog. If something goes sideways, stop and re-plan instead of pushing on.
|
||||
- **Use subagents liberally** for research, exploration, and parallel analysis — one focus each — to keep the main context clean.
|
||||
- **Verify before claiming done.** Run the tests, check the logs, show the evidence. Diff against `main` where behaviour could regress.
|
||||
- **Fix bugs autonomously.** Given a report, a failing test, or an error, resolve it — no hand-holding.
|
||||
- **Capture corrections.** After any correction from the user, record the pattern and a preventing rule in `tasks/lessons.md`; review it at session start.
|
||||
- Read files over 500 LOC in chunks. Never treat one grep as proof of absence.
|
||||
|
||||
Codex reviews your output once you are done.
|
||||
## Where knowledge lives
|
||||
|
||||
| Content | Location | Rule |
|
||||
|---|---|---|
|
||||
| Customer documentation | separate repo `autopilotmonitor-docs` (GitBook; baked into the MCP `search_docs` corpus) | Any change ⇒ MCP redeploy. Commit in that repo (`cd` first). |
|
||||
| Technical concept docs (OKF bundle) | `internal/` — private submodule, entry point `internal/docs/index.md` | Read the index first. Concept docs only for invariants, mechanisms and architecture a later change could silently break. |
|
||||
| Decisions, including "we will not build X" | `internal/decisions.md` | One entry per decision: date, decision, why, applies-to, status. |
|
||||
| Working files, lessons, plans | `internal/work/` | Rules in `internal/CLAUDE.md`. |
|
||||
| Agent review criteria | `internal/docs/agent/architecture-principles.md` | Check every agent change against it. |
|
||||
| Private operator instructions | `internal/CLAUDE.md` | Imported at the end of this file. Absent for contributors without submodule access; nothing in it may be copied into this public repo. |
|
||||
| Infrastructure and operator scripts | `infra/` — private submodule | Bicep plus deploy/backup/migration scripts. Scripts never carry credentials; secrets come from env or a gitignored file next to the script. |
|
||||
| Skills | `.claude/commands/` — private submodule | Each skill's frontmatter says when to use it. |
|
||||
| Claude auto-memory | outside the repo, per machine | Private notes only. Anything the project must keep goes into `internal/`. |
|
||||
|
||||
## Hard rules
|
||||
|
||||
- Agent projects stay on .NET Framework 4.8.
|
||||
- .NET build warnings are errors; only the NU1901–NU1904 advisories remain warnings.
|
||||
- Never put real customer domains, tenant names or tenant IDs into code, tests, commits, comments or public docs.
|
||||
- Deploy order is Backend → Web → MCP. Backend deploys are `workflow_dispatch` only. Verify the deployed version before claiming anything is live.
|
||||
- Submodules (`internal/`, `infra/`, `.claude/commands/`): commit inside the submodule first, then bump the gitlink here.
|
||||
- Stage explicit file lists; never `git add -A` or a directory. Pushing is the user's call.
|
||||
- New API responses need a typed DTO plus the parity fact. MCP vocabularies are generated from the wire types, never retyped.
|
||||
- Every route belongs in the route policy catalog; every new event type goes into the MCP resource catalog.
|
||||
- Workflow files under `.github/` go through zizmor before they are committed (`zizmor .` from the repository root, findings from severity low on): fix every finding or ignore it in `zizmor.yml` with a reason. The `zizmor` job in `action-pins.yml` runs the same check on every push.
|
||||
- The about, terms and privacy pages under `src/Web/autopilot-monitor-web/app/` state facts customers rely on: verify every claim against the code before writing it and update them in the same change as the fact they describe.
|
||||
|
||||
## Code Quality
|
||||
|
||||
- Simplicity first: minimal, targeted changes. No temporary fixes — find the root cause.
|
||||
- Where architecture is flawed, state is duplicated, or patterns are inconsistent, propose and implement the structural fix. Ask what a perfectionist reviewer would reject, and fix that too.
|
||||
- On non-trivial changes, ask whether there is a more elegant way before presenting. Skip it for simple fixes — don't over-engineer.
|
||||
- English for code comments and everything that carries knowledge.
|
||||
|
||||
## Task Management
|
||||
|
||||
Keep the plan in `tasks/todo.md` as checkable items, confirm it before implementing, tick items off as you go, and add a review section when done.
|
||||
|
||||
## Customer-Facing Claims
|
||||
|
||||
`src/Web/autopilot-monitor-web/app/{about,terms,privacy}/page.tsx` and `autopilotmonitor-docs/trust/*` state facts customers and security reviewers rely on. They drift silently.
|
||||
|
||||
- Verify every claim against the code before writing it — never carry one forward because it was already on the page.
|
||||
- Update them in the same change as: runtime/framework versions, roles, notification providers, agent deployment or lifecycle, diagnostics payload, isolation/delegation model, retention caps, sub-processors, or any default governing what is collected.
|
||||
- Describe only what customers can actually use; operator-only infrastructure is not a feature.
|
||||
- Trust pages: durable phrasing over exact figures, and bump their "Last reviewed" date.
|
||||
|
||||
## Technical Docs — OKF Bundle (`docs/`)
|
||||
|
||||
Contributor/AI-facing docs (customer docs live in the `autopilotmonitor-docs` repo). Start at `docs/index.md` before opening individual files.
|
||||
|
||||
- New durable knowledge (architecture decisions, flows, non-obvious mechanisms) → an OKF concept doc with YAML frontmatter: `type` (mandatory), plus `title`, `description`, `resource`, `tags`, `timestamp`.
|
||||
- Register it in `docs/index.md`, note the change in `docs/log.md` (ISO date). `index.md` and `log.md` are reserved names.
|
||||
- English, structural markdown (`# Schema`, `# Examples`, `# Citations`). Links between docs are RELATIVE — never `/`-prefixed, that breaks GitHub navigation.
|
||||
Private half of these instructions (process, working files, technical docs, operations): @../internal/CLAUDE.md
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
# Autopilot-Monitor — Agent Architecture Principles
|
||||
|
||||
The principles are **not nice-to-haves**, they are review criteria: any code
|
||||
change that violates a principle is either (a) a bug, (b) a reason to change the principle
|
||||
(with user sign-off), or (c) an explicitly documented exception with rationale.
|
||||
|
||||
---
|
||||
|
||||
## L.1 Truth Hierarchy (immutable)
|
||||
|
||||
`SignalLog` (input) → `Journal` (decisions) → `Snapshot` (cache) → `Projections` (derivable).
|
||||
No projection patches backwards into the journal or the signal log. On inconsistency, the
|
||||
lower layer always wins (e.g. discard the snapshot, do not "repair" the signal log).
|
||||
|
||||
## L.2 Event Sourcing as Foundation
|
||||
|
||||
`SignalLog` is the only input truth. The reducer is pure:
|
||||
`(oldState, signal) → (newState, transition, effects)`. The backend can re-derive at any
|
||||
time from `SignalLog`. Replay yields the same `Journal` content.
|
||||
|
||||
## L.3 Immutable State
|
||||
|
||||
All DTOs in the decision core are `sealed` with get-only properties. No in-place mutation.
|
||||
"Change" = new instance via `With…` method. This is the difference between "purely testable"
|
||||
and "subtle heisenbugs".
|
||||
|
||||
## L.4 Hypothesis Model instead of Flow Plugins
|
||||
|
||||
Fuzziness is constitutive (White-Glove, enrollment type, device-only). A single reducer with
|
||||
hypothesis fields (`Unknown/Weak/Strong/Confirmed/Rejected`), organized by scenario in partial
|
||||
classes. **No** plugin runtime, because clean flow boundaries do not match reality.
|
||||
|
||||
## L.5 Evidence Determinism
|
||||
|
||||
Signal-log determinism is guaranteed: same log → same outcome. Every signal carries an
|
||||
`Evidence` record with `Kind` (Raw / Derived / Synthetic) and mandatory fields. A signal
|
||||
without evidence = adapter rejection = bug.
|
||||
|
||||
## L.6 Signal Schema Versioning
|
||||
|
||||
Every `DecisionSignalKind` has a `SchemaVersion`. The reducer dispatches on
|
||||
`(Kind, SchemaVersion)` — old versions stay replay-capable. A new version or a new kind
|
||||
without a replay fixture in `tests/fixtures/signal-kinds/{kind}-v{n}.json` = merge block.
|
||||
|
||||
## L.7 Timer Monopoly
|
||||
|
||||
Only the Deadline Scheduler (the EffectRunner role) holds decision-relevant timers. Collectors,
|
||||
signal adapters and the orchestrator must not. `Task.Delay` or `new Timer(...)` in those
|
||||
layers with session-state impact = merge block. `IClock` interface + virtual clock for
|
||||
testability.
|
||||
|
||||
## L.8 Classifier as Post-Hook Kernel Service
|
||||
|
||||
`IClassifier` interface, swappable. The `ClassifierVerdict` flows back into the log as a
|
||||
synthetic `ClassifierVerdictIssued` signal. `InputHash` (SHA256 over classifier-relevant
|
||||
snapshot fields) is the anti-loop guard. Classifiers are not baked into the state machine.
|
||||
|
||||
## L.9 Clean-Agent Principle on Refactors
|
||||
|
||||
For structural refactors: build the new project in parallel to the legacy one, do not flag-
|
||||
route inside the same process. Strict separation rule: no direct reference to legacy
|
||||
projects. Code is copied, not shared. Legacy runs out, then gets removed.
|
||||
|
||||
## L.10 Generic Telemetry Transport
|
||||
|
||||
One spool for all telemetry types (events, signals, decision transitions). `TelemetryItemId`
|
||||
is the type-agnostic upload cursor; the type-specific counters (`Event.Sequence`,
|
||||
`SignalOrdinal`, `StepIndex`) remain storage / UI / reducer semantics. Drain guarantee:
|
||||
"coordinated + idempotent + resumable" — **not** cross-table atomic, which is not achievable
|
||||
with three separate tables anyway.
|
||||
|
||||
## L.11 Separation: Agent Lifecycle vs Session State
|
||||
|
||||
`DecisionState` is session-pure. Agent-process flags (crash, admin actions, boot time,
|
||||
heartbeat status) live separately in `agent-lifecycle.json`, managed by the orchestrator,
|
||||
never by the reducer. Prevents replay dilution by lifecycle events.
|
||||
|
||||
## L.12 Flush-on-Append Persistence
|
||||
|
||||
No batching of append operations. `FileOptions.WriteThrough` plus an explicit
|
||||
`FlushAsync(flushToDisk: true)` per append. Every successful `Append` return means: on disk.
|
||||
Durability > performance in the crash-prone OOBE context.
|
||||
|
||||
## L.13 UI Primacy — End-Users vs Dev Tooling
|
||||
|
||||
Product views (EventTimeline, PhaseTimeline) stay slim and stable. Debug / analysis tools
|
||||
(Inspector with DecisionGraph, SignalStream, VerifierReport) live on their own subpage behind
|
||||
an `adminMode` gate. The session-detail page does not get overloaded with dev components.
|
||||
|
||||
## L.14 Forward Compatibility by Default
|
||||
|
||||
Enums via `StringEnumConverter` + `UnknownFallbackEnumConverter` (unknown values → default,
|
||||
never crash). New schema fields are nullable. The `ReducerVersion` stamp per journal row
|
||||
allows older backends to read newer rows. Deletions only after a major version bump with
|
||||
user sign-off.
|
||||
|
||||
## L.15 Test Discipline
|
||||
|
||||
Flow / scenario tests are prioritized over structural tests. Coverage targets per component
|
||||
(DecisionCore ≥95 %, classifier 100 %, backend functions ≥85 %). A missing fixture for a
|
||||
scenario = merge block. CI threshold check is a build failure, not a warning.
|
||||
|
||||
## L.16 Fail-Safe Reducer Exceptions
|
||||
|
||||
Exception in the reducer → `DecisionTransition` with `DeadEndReason=reducer_exception`, state
|
||||
unchanged (fail-safe, never destroy state). Signal counts as processed (no retry loop). After
|
||||
five consecutive exceptions → the orchestrator emits
|
||||
`enrollment_failed(reason=reducer_instability)`.
|
||||
|
||||
## L.17 Effect Error Handling by Class
|
||||
|
||||
Three effect classes with their own strategy:
|
||||
|
||||
- **Transient** (retry with exponential backoff 100 ms / 400 ms / 1600 ms):
|
||||
`EmitEventTimelineEntry`, `PersistSnapshot`
|
||||
- **Critical** (session abort, e.g. on timer-register failure): `ScheduleDeadline`,
|
||||
`CancelDeadline`
|
||||
- **Optional** (best-effort, classifier exception → `Inconclusive` verdict, no abort):
|
||||
`RunClassifier`
|
||||
|
||||
Every effect failure is logged as a `DecisionTransition`.
|
||||
|
||||
## L.18 Implementation Discipline
|
||||
|
||||
- **R1** Pre-flight plan check before every work step
|
||||
- **R2** Delegate self-contained sub-tasks **and exploration** to subagents (Anthropic
|
||||
recommendation: preserve main-context window on large refactors — quality assurance)
|
||||
- **R3** Real verification (DB round-trip, index dual-write, agent signal emission) — no
|
||||
"it compiles and runs"
|
||||
- **R4** The plan is authority — deviations require user sign-off
|
||||
- **R5** Out-of-scope items land in `tasks/todo.md`, not in memory
|
||||
- **R6** High test coverage is a prerequisite for future refactorability, not a nice-to-have
|
||||
- **R7** Code-structure discipline: small files (guideline ≤300 LoC), clear folders, partial
|
||||
classes by scenario, check in with the user on any uncertainty (placement, naming, scope)
|
||||
|
||||
---
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Bug report
|
||||
description: Something does not work as documented.
|
||||
labels: [bug]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for the report. Please redact tenant IDs, device names, serial numbers, and anything personal before posting.
|
||||
For a failed enrollment in your own tenant, the fastest path is the **Report Session** button in the portal — it attaches the session context for support without exposing it here.
|
||||
- type: dropdown
|
||||
id: component
|
||||
attributes:
|
||||
label: Component
|
||||
options:
|
||||
- Web portal
|
||||
- Agent (on device)
|
||||
- Bootstrap script / Intune deployment
|
||||
- MCP server / AI integration
|
||||
- Analysis rules / log patterns
|
||||
- Backend / API
|
||||
- Documentation
|
||||
- Not sure
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: what
|
||||
attributes:
|
||||
label: What happened?
|
||||
description: What did you do, what did you expect, what happened instead?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Steps to reproduce
|
||||
placeholder: |
|
||||
1. …
|
||||
2. …
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Agent version / portal date
|
||||
description: Agent version from the session details, or the date you saw it in the portal.
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs or screenshots
|
||||
description: Relevant excerpts only, redacted. Paste text rather than screenshots of text where possible.
|
||||
render: text
|
||||
@@ -0,0 +1,11 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Question or idea
|
||||
url: https://github.com/okieselbach/AutopilotMonitor/discussions
|
||||
about: Ask how something works or float a feature idea before it becomes a request.
|
||||
- name: Documentation
|
||||
url: https://docs.autopilotmonitor.com
|
||||
about: Setup guides, troubleshooting, and the security FAQ.
|
||||
- name: Security vulnerability
|
||||
url: https://github.com/okieselbach/AutopilotMonitor/security/advisories/new
|
||||
about: Report privately. Please never open a public issue for a security finding.
|
||||
@@ -0,0 +1,34 @@
|
||||
name: Feature request
|
||||
description: Propose something Autopilot Monitor should do.
|
||||
labels: [enhancement]
|
||||
body:
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What problem are you trying to solve?
|
||||
description: Describe the situation, not the solution. What are you doing today instead?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: proposal
|
||||
attributes:
|
||||
label: Proposed change
|
||||
description: What would you like to see? Rough is fine.
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: area
|
||||
attributes:
|
||||
label: Area
|
||||
options:
|
||||
- Web portal
|
||||
- Agent (on device)
|
||||
- Analysis rules / diagnostics
|
||||
- MCP server / AI integration
|
||||
- Notifications / integrations
|
||||
- Reporting
|
||||
- Other
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives considered
|
||||
|
After Width: | Height: | Size: 105 KiB |
|
After Width: | Height: | Size: 134 KiB |
|
After Width: | Height: | Size: 201 KiB |
|
After Width: | Height: | Size: 145 KiB |
|
Before Width: | Height: | Size: 2.2 KiB After Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 202 B After Width: | Height: | Size: 202 B |
|
After Width: | Height: | Size: 141 KiB |
@@ -0,0 +1,55 @@
|
||||
<!doctype html>
|
||||
<html><head><meta charset="utf-8">
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
html,body{margin:0;padding:0;}
|
||||
body{width:1280px;height:640px;overflow:hidden;font-family:Inter,system-ui,-apple-system,"Segoe UI",sans-serif;color:#0f172a;
|
||||
background:
|
||||
radial-gradient(900px 600px at 105% 110%, #dcfce7 0%, rgba(220,252,231,0) 60%),
|
||||
radial-gradient(700px 500px at -10% -10%, #e0f2fe 0%, rgba(224,242,254,0) 60%),
|
||||
linear-gradient(180deg,#ffffff 0%,#f6f8f7 100%);
|
||||
position:relative;}
|
||||
.grid{position:absolute;inset:0;background-image:linear-gradient(rgba(15,23,42,.045) 1px,transparent 1px),linear-gradient(90deg,rgba(15,23,42,.045) 1px,transparent 1px);background-size:40px 40px;mask-image:linear-gradient(180deg,rgba(0,0,0,.7),rgba(0,0,0,0) 85%);}
|
||||
.left{position:absolute;left:72px;top:0;height:640px;width:520px;display:flex;flex-direction:column;justify-content:center;z-index:2;}
|
||||
.brand{display:flex;align-items:center;gap:14px;margin-bottom:26px;}
|
||||
.brand svg{width:44px;height:40px;}
|
||||
.brand span{font-size:22px;font-weight:600;letter-spacing:-.01em;color:#0f172a;}
|
||||
h1{margin:0 0 18px;font-size:50px;line-height:1.06;font-weight:800;letter-spacing:-.035em;}
|
||||
h1 em{font-style:normal;color:#16a34a;}
|
||||
p.tag{margin:0 0 30px;font-size:20px;line-height:1.45;color:#475569;font-weight:400;max-width:480px;}
|
||||
.chips{display:flex;flex-wrap:wrap;gap:10px;margin-bottom:34px;}
|
||||
.chip{display:inline-flex;align-items:center;gap:8px;padding:8px 14px;border-radius:999px;background:#fff;border:1px solid #e2e8f0;box-shadow:0 1px 2px rgba(15,23,42,.05);font-size:15px;font-weight:500;color:#1e293b;}
|
||||
.chip i{width:8px;height:8px;border-radius:50%;background:#22c55e;display:inline-block;}
|
||||
.chip.b i{background:#3b82f6}.chip.p i{background:#8b5cf6}
|
||||
.foot{display:flex;align-items:center;gap:14px;font-size:15px;color:#64748b;font-weight:500;}
|
||||
.foot b{color:#0f172a;font-weight:600;}
|
||||
.foot .dot{width:4px;height:4px;border-radius:50%;background:#cbd5e1;}
|
||||
.shot{position:absolute;left:600px;top:76px;width:1000px;height:640px;border-radius:14px;background:#fff;
|
||||
box-shadow:0 30px 80px -20px rgba(15,23,42,.35),0 0 0 1px rgba(15,23,42,.08);overflow:hidden;z-index:1;}
|
||||
.bar{position:relative;z-index:2;height:34px;background:#f8fafc;border-bottom:1px solid #e2e8f0;display:flex;align-items:center;padding:0 14px;gap:7px;}
|
||||
.bar i{width:10px;height:10px;border-radius:50%;background:#e2e8f0;display:inline-block;}
|
||||
.bar .url{margin-left:12px;flex:0 0 auto;height:20px;padding:0 10px;border-radius:6px;background:#fff;border:1px solid #e2e8f0;font-size:11px;line-height:20px;color:#64748b;}
|
||||
.shot img{position:absolute;left:-214px;top:-16px;width:1440px;height:900px;transform-origin:top left;transform:scale(.94);}
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="grid"></div>
|
||||
<div class="left">
|
||||
<div class="brand">
|
||||
<svg viewBox="0 0 209 191" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M0 180.201L208.401 190.502L188.157 76.2438L5.48363e-06 0L0 180.201Z" fill="#33B161"/></svg>
|
||||
<span>Autopilot Monitor</span>
|
||||
</div>
|
||||
<h1>See every Windows<br>enrollment <em>as it happens</em></h1>
|
||||
<p class="tag">Live session timelines, fleet health, and on-demand diagnostics for Windows Autopilot and Intune.</p>
|
||||
<div class="chips">
|
||||
<span class="chip"><i></i>Live session timeline</span>
|
||||
<span class="chip b"><i></i>Fleet health</span>
|
||||
<span class="chip p"><i></i>MCP server for AI troubleshooting</span>
|
||||
</div>
|
||||
<div class="foot"><b>autopilotmonitor.com</b><span class="dot"></span>Open source<span class="dot"></span>Free Community plan</div>
|
||||
</div>
|
||||
<div class="shot">
|
||||
<div class="bar"><i></i><i></i><i></i><span class="url">portal.autopilotmonitor.com/dashboard</span></div>
|
||||
<img src="social-preview-dashboard.png" alt="">
|
||||
</div>
|
||||
</body></html>
|
||||
|
After Width: | Height: | Size: 210 KiB |
@@ -1,8 +1,8 @@
|
||||
# CodeQL configuration.
|
||||
#
|
||||
# The default query suites stay active — this file only filters out two C# queries
|
||||
# that fire structurally on this codebase, where the flagged behavior is the feature,
|
||||
# not a defect:
|
||||
# The default query suites stay active — this file only filters out queries that
|
||||
# fire structurally on this codebase, where the flagged behavior is the feature
|
||||
# or the protection exists in a form the query cannot see:
|
||||
#
|
||||
# cs/exposure-of-sensitive-information
|
||||
# Audit and operational logs record WHO performed an action (actor UPN / email)
|
||||
@@ -15,10 +15,20 @@
|
||||
# key. The IP is an identifier for throttling, not a secret at rest; storing it in
|
||||
# the in-memory / table rate-limit key is by design.
|
||||
#
|
||||
# js/missing-rate-limiting
|
||||
# Only the MCP server has Express routes. Every one that does work is throttled by
|
||||
# our own limiters (oauthRateLimit / oauthTokenRateLimit on /oauth/*, accessGuard
|
||||
# on /mcp); /health and the OAuth discovery documents return static JSON.
|
||||
# The query only recognizes limiters from npm packages (express-rate-limit and
|
||||
# similar) and flags a route handler as soon as it calls a function named
|
||||
# like *auth*, *login* or *verify*. So it reports throttled and unthrottled
|
||||
# routes the same way, and every alert it ever raised was a false positive. The
|
||||
# coverage check it stood in for is a test: every OAuth router route must carry
|
||||
# a limiter or be named as exempt (MCP oauth-rate-limit.test.ts).
|
||||
#
|
||||
# Genuine one-off false positives from OTHER queries (e.g. an open-redirect the taint
|
||||
# model can't see is allow-listed, a rate-limited route it doesn't recognize) are
|
||||
# dismissed individually in the Security tab so those queries stay sharp — they are
|
||||
# NOT suppressed here.
|
||||
# model can't see is allow-listed) are dismissed individually in the Security tab so
|
||||
# those queries stay sharp — they are NOT suppressed here.
|
||||
#
|
||||
# Log-injection (cs/log-forging) is deliberately NOT filtered: user-controlled values
|
||||
# are sanitized at the sink via Helpers/LogSanitizer, so the query stays useful for
|
||||
@@ -29,3 +39,5 @@ query-filters:
|
||||
id: cs/exposure-of-sensitive-information
|
||||
- exclude:
|
||||
id: cs/cleartext-storage-of-sensitive-information
|
||||
- exclude:
|
||||
id: js/missing-rate-limiting
|
||||
|
||||
@@ -1,25 +1,44 @@
|
||||
version: 2
|
||||
|
||||
# Cooldown: a version is proposed only once it has been public for 7 days, so a
|
||||
# release that is yanked or found malicious within its first week never reaches a
|
||||
# PR. Security updates are exempt from the cooldown by Dependabot's own rules.
|
||||
#
|
||||
# Grouping: every ecosystem gets one batched PR for minor/patch version
|
||||
# updates and one batched PR for security updates. Groups apply to version
|
||||
# updates only by default, so the security-updates groups are declared
|
||||
# explicitly — otherwise every advisory lands as its own PR.
|
||||
|
||||
updates:
|
||||
- package-ecosystem: nuget
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
dotnet:
|
||||
patterns: ['*']
|
||||
update-types: [minor, patch]
|
||||
dotnet-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
- package-ecosystem: npm
|
||||
directory: /src/Web/autopilot-monitor-web
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
web:
|
||||
patterns: ['*']
|
||||
update-types: [minor, patch]
|
||||
web-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
# Major bumps are breaking and reviewed deliberately, not auto-PR'd.
|
||||
# Security updates ignore this filter, so the Security tab stays clean.
|
||||
ignore:
|
||||
@@ -30,18 +49,51 @@ updates:
|
||||
directory: /src/McpServer/autopilot-monitor-mcp
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
mcp:
|
||||
patterns: ['*']
|
||||
update-types: [minor, patch]
|
||||
mcp-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
# Major bumps are breaking and reviewed deliberately, not auto-PR'd.
|
||||
# Security updates ignore this filter, so the Security tab stays clean.
|
||||
ignore:
|
||||
- dependency-name: '*'
|
||||
update-types: ['version-update:semver-major']
|
||||
|
||||
# Validator tooling for rules/ (ajv-cli); combine-rules.yml installs it from this
|
||||
# lockfile with contents:write in hand, so the pins must not go stale.
|
||||
- package-ecosystem: npm
|
||||
directory: /rules/scripts
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
rules-tooling:
|
||||
patterns: ['*']
|
||||
rules-tooling-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
# Every action is referenced by commit SHA with the version in a trailing comment;
|
||||
# action-pins.yml fails any workflow change that reintroduces a mutable tag. Dependabot
|
||||
# is what keeps those pins current: it rewrites the SHA and the comment together, so the
|
||||
# pins are reviewed in a PR instead of drifting silently or being bumped by hand.
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: monthly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
groups:
|
||||
actions:
|
||||
patterns: ['*']
|
||||
actions-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Returns the request headers that authorize an Azure Storage REST call with Microsoft Entra.
|
||||
|
||||
.DESCRIPTION
|
||||
One place for the storage OAuth contract shared by every publisher in this repo -- the
|
||||
agent ZIP and MSI, the customer bootstrap scripts, the version manifests and the build
|
||||
counters. Resource id, the az invocation, the service version and the failure message
|
||||
live here so they cannot drift between call sites.
|
||||
|
||||
Identity comes from whatever `az login` established: the federated GitHub OIDC principal
|
||||
in CI (azure/login, subject repo:okieselbach/AutopilotMonitor:ref:refs/heads/main), the
|
||||
operator's own account for the local build. No secret is read, stored or passed around.
|
||||
|
||||
x-ms-version is MANDATORY here, not decoration: Entra-authorized requests are rejected
|
||||
below 2017-11-09, and Table requests without a SAS must carry the header at all. 2021-08-06
|
||||
is old enough to be deployed in every region and new enough for everything we send.
|
||||
|
||||
Callers merge the result into their own headers -- `$auth + @{ ... }` yields a new
|
||||
hashtable per request, so nothing is shared across uploads:
|
||||
|
||||
$auth = & "$PSScriptRoot/Get-StorageAuthHeaders.ps1"
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($auth + @{ 'x-ms-blob-type' = 'BlockBlob' }) -Body $bytes
|
||||
|
||||
Acquire once per step and reuse: every call shells out to az (~1s) and the token is valid
|
||||
for the lifetime of a job.
|
||||
|
||||
.OUTPUTS
|
||||
[hashtable] Authorization + x-ms-version.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# The data-plane resource, not management.azure.com -- a management token is accepted by
|
||||
# neither the blob nor the table endpoint.
|
||||
$token = az account get-access-token --resource https://storage.azure.com --query accessToken -o tsv 2>$null
|
||||
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw ('Could not acquire an Azure Storage token. In CI this means the azure/login step ' +
|
||||
'is missing or its federated credential does not cover this ref; locally, run ' +
|
||||
'`az login`. A token alone is not enough: the principal also needs Storage Blob ' +
|
||||
'Data Contributor on the container it writes, and Storage Table Data Contributor ' +
|
||||
'on the table -- a missing role shows up later as HTTP 403 AuthorizationPermissionMismatch.')
|
||||
}
|
||||
|
||||
return @{
|
||||
'Authorization' = "Bearer $token"
|
||||
'x-ms-version' = '2021-08-06'
|
||||
}
|
||||
@@ -12,52 +12,78 @@
|
||||
lands a consistent set (ZIP + manifest + scripts).
|
||||
|
||||
Both callers run the same code. The scripts on the download alias are the live
|
||||
copy the WDP bootstrap MSI (Invoke-BootstrapDownload.ps1) downloads on every
|
||||
enrolling device, so "current" is a correctness property, not cosmetics.
|
||||
copy the stage-1 loader downloads on every enrolling device, whether it came from
|
||||
Intune or from the bootstrap MSI, so "current" is a correctness property, not
|
||||
cosmetics.
|
||||
|
||||
Steps, in order:
|
||||
1. Parse $ScriptVersion from the bootstrap source.
|
||||
2. Render the -Dev variant by literal substitution of the two URL/manifest
|
||||
1. Parse $ScriptVersion from the loader and the bootstrap source.
|
||||
2. Render the -Dev variants by literal substitution of the URL/manifest
|
||||
defaults. A missing anchor is a hard failure -- a silently un-substituted
|
||||
dev script would point the dev fleet at the stable agent.
|
||||
3. Version-bump guard: if the published bootstrap differs from the source
|
||||
but carries the same $ScriptVersion, abort. Otherwise the docs badge,
|
||||
3. Version-bump guard: if a published script differs from its source but
|
||||
carries the same $ScriptVersion, abort. Otherwise the docs badge,
|
||||
version.json.bootstrapVersion and the portal's "your script is outdated"
|
||||
hint would all keep asserting a version that is no longer what ships.
|
||||
4. Upload every script blob with Cache-Control: no-cache (they rotate in
|
||||
4. Build the publish set (loader, bootstrap, both dev renders, the extras).
|
||||
5. Authenticode-sign every script (see -SigningToken). Signing is the LAST
|
||||
transformation: it covers the exact CRLF/UTF-8 bytes that get uploaded,
|
||||
and nothing may touch them afterwards or the signature breaks.
|
||||
6. Upload every script blob with Cache-Control: no-cache (they rotate in
|
||||
place), mirrored fail-soft to the legacy account.
|
||||
5. Reconcile the two version oracles: version.json.bootstrapVersion
|
||||
(read-modify-write under If-Match, so a concurrent agent release cannot
|
||||
lose its agent fields) and AdminConfiguration.LatestBootstrapV2ScriptVersion.
|
||||
6. Verify through the alias -- re-download each blob and compare SHA-256.
|
||||
7. Reconcile the version oracles: version.json.bootstrapVersion and
|
||||
.loaderVersion (read-modify-write under If-Match, so a concurrent agent
|
||||
release cannot lose its agent fields) and
|
||||
AdminConfiguration.LatestBootstrapV2ScriptVersion.
|
||||
8. Verify through the alias -- re-download each blob and compare SHA-256.
|
||||
|
||||
.PARAMETER SasToken
|
||||
Container SAS for the eu storage account with write permission. Leading '?' tolerated.
|
||||
Comparisons against a published script always strip its signature block first
|
||||
(Get-ComparableText). A signature carries a timestamp, so signed bytes differ on
|
||||
every run; without stripping, the bump guard and the drift check would report a
|
||||
change for every unchanged script.
|
||||
|
||||
.PARAMETER LegacySasToken
|
||||
Container SAS for the legacy account. Optional -- a mirror failure warns, never fails.
|
||||
|
||||
.PARAMETER TableSasToken
|
||||
Table SAS for AdminConfiguration. Optional -- omitted or 'XXX' skips the oracle update.
|
||||
The legacy account lives in a different tenant, so the ambient Entra identity cannot
|
||||
reach it; this is the only credential the publisher still takes.
|
||||
|
||||
.PARAMETER RepoRoot
|
||||
Repository root. Defaults to the parent of .github/scripts.
|
||||
|
||||
.PARAMETER DryRun
|
||||
Run every check and render, upload nothing.
|
||||
Run every check and render, upload nothing. A dry run signs when a token is supplied
|
||||
and skips signing when it is not: the PR gate runs this without any Azure login.
|
||||
|
||||
.PARAMETER SigningToken
|
||||
Azure Key Vault access token for AzureSignTool. MANDATORY for a real publish -- an
|
||||
unsigned customer script is not something this repo publishes. Together with
|
||||
-KeyVaultUrl and -CertificateName.
|
||||
|
||||
.PARAMETER ExpectedPublisher
|
||||
Optional -like pattern the signer subject must match after signing. Pins the publisher
|
||||
instead of accepting any trusted signature; the loader on the device pins the same way.
|
||||
|
||||
.PARAMETER SignedOutputDir
|
||||
Optional directory that receives a copy of every published file exactly as uploaded.
|
||||
Used by CI to offer the signed scripts as a build artifact for local inspection.
|
||||
|
||||
.EXAMPLE
|
||||
./Publish-BootstrapScripts.ps1 -DryRun
|
||||
|
||||
.EXAMPLE
|
||||
./Publish-BootstrapScripts.ps1 -SasToken $env:SAS -TableSasToken $env:TABLE_SAS
|
||||
./Publish-BootstrapScripts.ps1 -LegacySasToken $env:LEGACY_SAS -SigningToken $env:AKV_TOKEN `
|
||||
-KeyVaultUrl $env:KV_URL -CertificateName $env:CERT
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$SasToken,
|
||||
[string]$LegacySasToken,
|
||||
[string]$TableSasToken,
|
||||
[string]$RepoRoot,
|
||||
[string]$SigningToken,
|
||||
[string]$KeyVaultUrl,
|
||||
[string]$CertificateName,
|
||||
[string]$TimestampUrl = 'http://timestamp.acs.microsoft.com',
|
||||
[string]$ExpectedPublisher,
|
||||
[string]$SignedOutputDir,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
@@ -66,9 +92,6 @@ $ErrorActionPreference = 'Stop'
|
||||
if (-not $RepoRoot) {
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
|
||||
}
|
||||
if (-not $DryRun -and [string]::IsNullOrWhiteSpace($SasToken)) {
|
||||
throw 'SasToken is required unless -DryRun is used.'
|
||||
}
|
||||
|
||||
$ContainerUrl = 'https://autopilotmonitoreu.blob.core.windows.net/agent'
|
||||
$LegacyContainerUrl = 'https://autopilotmonitor.blob.core.windows.net/agent'
|
||||
@@ -78,8 +101,23 @@ $TableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminCo
|
||||
$BootstrapSource = Join-Path $RepoRoot 'scripts/Bootstrap/Install-AutopilotMonitor.ps1'
|
||||
$BootstrapBlob = 'Install-AutopilotMonitor.ps1'
|
||||
$DevBlob = 'Install-AutopilotMonitor-Dev.ps1'
|
||||
# Stage 1. Assigned in Intune once, and embedded in the bootstrap MSI; it downloads the
|
||||
# bootstrap above on every device and verifies its publisher before running it.
|
||||
$LoaderSource = Join-Path $RepoRoot 'scripts/Bootstrap/Start-AutopilotMonitor.ps1'
|
||||
$LoaderBlob = 'Start-AutopilotMonitor.ps1'
|
||||
$LoaderDevBlob = 'Start-AutopilotMonitor-Dev.ps1'
|
||||
$ScriptContentType = 'text/plain; charset=utf-8'
|
||||
|
||||
$SigningEnabled = -not [string]::IsNullOrWhiteSpace($SigningToken)
|
||||
if (-not $DryRun -and -not $SigningEnabled) {
|
||||
throw ('Publishing requires -SigningToken (plus -KeyVaultUrl and -CertificateName): the ' +
|
||||
'loader on every device verifies the publisher of what it downloads, so an unsigned ' +
|
||||
'script would be rejected in the field. Only -DryRun may run without signing.')
|
||||
}
|
||||
if ($SigningEnabled -and ([string]::IsNullOrWhiteSpace($KeyVaultUrl) -or [string]::IsNullOrWhiteSpace($CertificateName))) {
|
||||
throw '-SigningToken needs -KeyVaultUrl and -CertificateName.'
|
||||
}
|
||||
|
||||
function Get-HttpStatus {
|
||||
param($ErrorRecord)
|
||||
if ($ErrorRecord.Exception.PSObject.Properties['Response'] -and $ErrorRecord.Exception.Response) {
|
||||
@@ -125,6 +163,132 @@ function Get-PublishBytes {
|
||||
return ,(New-Object System.Text.UTF8Encoding $false).GetBytes($normalised)
|
||||
}
|
||||
|
||||
# A PowerShell signature is appended as a comment block at the end of the file, so the
|
||||
# published bytes are "source + signature". Everything that compares a published script to
|
||||
# its source has to cut that block off first -- the signature carries a timestamp and is
|
||||
# therefore different on every publish, while the script above it is unchanged.
|
||||
function Remove-SignatureBlock {
|
||||
param([string]$Text)
|
||||
$marker = '# SIG # Begin signature block'
|
||||
$index = $Text.IndexOf($marker)
|
||||
if ($index -lt 0) { return $Text }
|
||||
return $Text.Substring(0, $index)
|
||||
}
|
||||
|
||||
# Comparison form: no signature, LF, no trailing blank lines. Used for "is the published
|
||||
# copy still this source" -- never for what gets uploaded.
|
||||
function Get-ComparableText {
|
||||
param([string]$Text)
|
||||
return (Remove-SignatureBlock $Text).Replace("`r`n", "`n").TrimEnd("`n")
|
||||
}
|
||||
|
||||
# Signs the publish set in place: each item's bytes are written to a work directory, signed
|
||||
# there, verified, and read back as the bytes that will be uploaded. Signing has to be the
|
||||
# last transformation -- the signature covers exactly these bytes, so nothing may normalise
|
||||
# or re-encode them afterwards.
|
||||
function Set-PublishSetSignature {
|
||||
param([object[]]$Items, [string]$WorkDir)
|
||||
|
||||
if (-not (Get-Command azuresigntool -ErrorAction SilentlyContinue)) {
|
||||
throw "azuresigntool is not on PATH. Install it with: dotnet tool install --global AzureSignTool --version 7.0.1"
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null
|
||||
|
||||
$paths = @()
|
||||
foreach ($item in $Items) {
|
||||
$path = Join-Path $WorkDir $item.BlobName
|
||||
[System.IO.File]::WriteAllBytes($path, [byte[]]$item.Bytes)
|
||||
$paths += $path
|
||||
}
|
||||
|
||||
Write-Host "Signing $($paths.Count) script(s) with certificate '$CertificateName'..."
|
||||
& azuresigntool.exe sign --verbose `
|
||||
--azure-key-vault-url $KeyVaultUrl `
|
||||
--azure-key-vault-accesstoken $SigningToken `
|
||||
--azure-key-vault-certificate $CertificateName `
|
||||
--timestamp-rfc3161 $TimestampUrl `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor bootstrap' `
|
||||
--description-url 'https://www.autopilotmonitor.com' `
|
||||
$paths
|
||||
|
||||
# Native tool: PowerShell does not throw on a non-zero exit, and an unnoticed signing
|
||||
# failure would publish unsigned scripts that every loader in the field then rejects.
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "azuresigntool failed with exit code $LASTEXITCODE -- nothing was uploaded."
|
||||
}
|
||||
|
||||
foreach ($item in $Items) {
|
||||
$path = Join-Path $WorkDir $item.BlobName
|
||||
$sig = Get-AuthenticodeSignature -FilePath $path
|
||||
if ($sig.Status -ne 'Valid') {
|
||||
throw "Signature on $($item.BlobName) is '$($sig.Status)', expected 'Valid'."
|
||||
}
|
||||
# Without a timestamp the signature dies with the certificate, and these scripts sit
|
||||
# on customer devices far longer than a certificate lifetime.
|
||||
if (-not $sig.TimeStamperCertificate) {
|
||||
throw "Signature on $($item.BlobName) carries no timestamp."
|
||||
}
|
||||
if ($ExpectedPublisher -and $sig.SignerCertificate.Subject -notlike $ExpectedPublisher) {
|
||||
throw "$($item.BlobName) was signed by '$($sig.SignerCertificate.Subject)', expected '$ExpectedPublisher'."
|
||||
}
|
||||
# Plain assignment, NO comma operator: the ',' elsewhere in this file exists because
|
||||
# PowerShell unrolls an array RETURNED from a function, which is not what a property
|
||||
# assignment does -- here it would wrap the byte[] in an Object[] of length 1 and the
|
||||
# upload would fail converting it back.
|
||||
$signedBytes = [System.IO.File]::ReadAllBytes($path)
|
||||
if ($signedBytes.Length -le $item.Bytes.Length) {
|
||||
throw ("Signed $($item.BlobName) is $($signedBytes.Length) bytes but the unsigned form " +
|
||||
"was $($item.Bytes.Length) -- a signature only ever adds bytes, so this is a " +
|
||||
'read-back bug, not a signing result.')
|
||||
}
|
||||
$item.Bytes = $signedBytes
|
||||
Write-Host " signed $($item.BlobName) ($($signedBytes.Length) bytes, signer $($sig.SignerCertificate.Subject.Split(',')[0]))"
|
||||
}
|
||||
}
|
||||
|
||||
# Aborts when a published script changed behaviour without a version bump. Consumers act on
|
||||
# the version (docs badge, version.json, the portal's outdated-script hint), so an unchanged
|
||||
# version has to mean unchanged behaviour.
|
||||
function Assert-VersionBump {
|
||||
param(
|
||||
[string]$BlobName,
|
||||
[string]$SourceContent,
|
||||
[string]$SourceOrigin,
|
||||
[string]$SourceVersion
|
||||
)
|
||||
|
||||
$publishedBytes = Get-PublishedBytes "$AliasUrl/$BlobName"
|
||||
if ($null -eq $publishedBytes) {
|
||||
Write-Host "No published $BlobName yet -- version-bump guard skipped (first publish)"
|
||||
return
|
||||
}
|
||||
|
||||
$publishedText = [System.Text.Encoding]::UTF8.GetString($publishedBytes)
|
||||
if ((Get-ComparableText $publishedText) -eq (Get-ComparableText $SourceContent)) {
|
||||
Write-Host "Published $BlobName is already this source (signature block aside)"
|
||||
return
|
||||
}
|
||||
|
||||
$publishedVersion = Get-BootstrapScriptVersion -Content $publishedText -Origin "$AliasUrl/$BlobName" -Optional
|
||||
|
||||
if ($null -eq $publishedVersion) {
|
||||
Write-Host "::warning::The published $BlobName is not a readable script (no ScriptVersion found). Republishing over it."
|
||||
} elseif ($publishedVersion -ne $SourceVersion) {
|
||||
Write-Host "Version bump: $publishedVersion -> $SourceVersion ($BlobName)"
|
||||
} elseif ((Get-CodeFingerprint -Text (Remove-SignatureBlock $publishedText) -Origin "$AliasUrl/$BlobName") -eq
|
||||
(Get-CodeFingerprint -Text $SourceContent -Origin $SourceOrigin)) {
|
||||
# Comments, typos in comments, reflow -- nothing a consumer of the version can act on.
|
||||
Write-Host "::warning::$BlobName changed in comments or formatting only; publishing under the unchanged version $SourceVersion."
|
||||
} else {
|
||||
throw ("Version-bump guard: $BlobName changed behaviour but ScriptVersion is still $SourceVersion. " +
|
||||
"Bump it in $SourceOrigin -- the docs badge, version.json and the " +
|
||||
'portal outdated-script hint all read that value, and customers decide from it whether ' +
|
||||
'to re-upload their Intune copy. Comment-only and formatting changes do not need a bump.')
|
||||
}
|
||||
}
|
||||
|
||||
function Get-BootstrapScriptVersion {
|
||||
param([string]$Content, [string]$Origin, [switch]$Optional)
|
||||
if ($Content -match '\$ScriptVersion\s*=\s*"([\d\.\-a-zA-Z]+)"') { return $Matches[1] }
|
||||
@@ -152,12 +316,16 @@ function Get-CodeFingerprint {
|
||||
return (($tokens | Where-Object { $ignored -notcontains $_.Kind.ToString() } | ForEach-Object { $_.Text }) -join "`n")
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 1. source
|
||||
# ------------------------------------------------------------------ 1. sources
|
||||
$sourceContent = Get-Content $BootstrapSource -Raw
|
||||
$sourceBytes = Get-PublishBytes $sourceContent
|
||||
$scriptVersion = Get-BootstrapScriptVersion -Content $sourceContent -Origin $BootstrapSource
|
||||
Write-Host "Bootstrap script version: $scriptVersion"
|
||||
|
||||
$loaderContent = Get-Content $LoaderSource -Raw
|
||||
$loaderVersion = Get-BootstrapScriptVersion -Content $loaderContent -Origin $LoaderSource
|
||||
Write-Host "Loader script version: $loaderVersion"
|
||||
|
||||
# ------------------------------------------------------------------ 2. dev render
|
||||
$stableUrlLiteral = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent.zip"'
|
||||
$devUrlLiteral = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent-dev.zip"'
|
||||
@@ -170,33 +338,21 @@ if ($sourceContent.IndexOf($stableUrlLiteral) -lt 0 -or $sourceContent.IndexOf($
|
||||
$devContent = $sourceContent.Replace($stableUrlLiteral, $devUrlLiteral).Replace($stableManLiteral, $devManLiteral)
|
||||
Write-Host "Rendered $DevBlob (dev agent URL + version-dev.json)"
|
||||
|
||||
# ------------------------------------------------------------------ 3. bump guard
|
||||
$publishedBytes = Get-PublishedBytes "$AliasUrl/$BootstrapBlob"
|
||||
if ($null -eq $publishedBytes) {
|
||||
Write-Host 'No published bootstrap yet -- version-bump guard skipped (first publish)'
|
||||
} elseif ((Get-Sha256 $publishedBytes) -eq (Get-Sha256 $sourceBytes)) {
|
||||
Write-Host 'Published bootstrap is already byte-identical to the source'
|
||||
} else {
|
||||
$publishedText = [System.Text.Encoding]::UTF8.GetString($publishedBytes)
|
||||
$publishedVersion = Get-BootstrapScriptVersion -Content $publishedText -Origin "$AliasUrl/$BootstrapBlob" -Optional
|
||||
# The dev loader must fetch the dev bootstrap, or a dev device would run the stable chain.
|
||||
$stableLoaderLiteral = '$BootstrapUrl = "https://download.autopilotmonitor.com/agent/Install-AutopilotMonitor.ps1"'
|
||||
$devLoaderLiteral = '$BootstrapUrl = "https://download.autopilotmonitor.com/agent/Install-AutopilotMonitor-Dev.ps1"'
|
||||
|
||||
if ($null -eq $publishedVersion) {
|
||||
Write-Host "::warning::The published $BootstrapBlob is not a readable bootstrap script (no ScriptVersion found). Republishing over it."
|
||||
} elseif ($publishedVersion -ne $scriptVersion) {
|
||||
Write-Host "Version bump: $publishedVersion -> $scriptVersion"
|
||||
} elseif ((Get-CodeFingerprint -Text $publishedText -Origin "$AliasUrl/$BootstrapBlob") -eq
|
||||
(Get-CodeFingerprint -Text $sourceContent -Origin $BootstrapSource)) {
|
||||
# Comments, typos in comments, reflow -- nothing a consumer of the version can act on.
|
||||
Write-Host "::warning::$BootstrapBlob changed in comments or formatting only; publishing under the unchanged version $scriptVersion."
|
||||
} else {
|
||||
throw ("Version-bump guard: $BootstrapBlob changed behaviour but ScriptVersion is still $scriptVersion. " +
|
||||
"Bump it in $BootstrapSource -- the docs badge, version.json.bootstrapVersion and the " +
|
||||
'portal outdated-script hint all read that value, and customers decide from it whether ' +
|
||||
'to re-upload their Intune copy. Comment-only and formatting changes do not need a bump.')
|
||||
}
|
||||
if ($loaderContent.IndexOf($stableLoaderLiteral) -lt 0) {
|
||||
throw "Loader dev-render: anchor literal missing in $LoaderSource"
|
||||
}
|
||||
$devLoaderContent = $loaderContent.Replace($stableLoaderLiteral, $devLoaderLiteral)
|
||||
Write-Host "Rendered $LoaderDevBlob (points at $DevBlob)"
|
||||
|
||||
# ------------------------------------------------------------------ 4. upload
|
||||
# ------------------------------------------------------------------ 3. bump guards
|
||||
Assert-VersionBump -BlobName $BootstrapBlob -SourceContent $sourceContent -SourceOrigin $BootstrapSource -SourceVersion $scriptVersion
|
||||
Assert-VersionBump -BlobName $LoaderBlob -SourceContent $loaderContent -SourceOrigin $LoaderSource -SourceVersion $loaderVersion
|
||||
|
||||
# ------------------------------------------------------------------ 4. publish set
|
||||
# Sources beyond the bootstrap pair, read straight from the repo.
|
||||
$extraSources = @(
|
||||
@{ Path = 'scripts/Bootstrap/Test-ShouldBootstrapAgent.ps1'; BlobName = 'Test-ShouldBootstrapAgent.ps1' }
|
||||
@@ -206,20 +362,38 @@ $extraSources = @(
|
||||
# Uninstall-AutopilotMonitor.ps1 is deliberately NOT published: nothing links it, and
|
||||
# an unauthenticated uninstall script on the public download host is not a feature.
|
||||
|
||||
# Text is the unsigned publish form and stays the reference for every comparison against a
|
||||
# published copy; Bytes is what goes on the wire and carries the signature after step 4.
|
||||
$publishSet = [System.Collections.Generic.List[object]]::new()
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $BootstrapBlob; Bytes = $sourceBytes })
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $DevBlob; Bytes = (Get-PublishBytes $devContent) })
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $LoaderBlob; Text = $loaderContent; Bytes = (Get-PublishBytes $loaderContent) })
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $LoaderDevBlob; Text = $devLoaderContent; Bytes = (Get-PublishBytes $devLoaderContent) })
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $BootstrapBlob; Text = $sourceContent; Bytes = $sourceBytes })
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $DevBlob; Text = $devContent; Bytes = (Get-PublishBytes $devContent) })
|
||||
foreach ($extra in $extraSources) {
|
||||
$extraPath = Join-Path $RepoRoot $extra.Path
|
||||
if (-not (Test-Path $extraPath)) {
|
||||
throw "Publish source missing: $extraPath"
|
||||
}
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $extra.BlobName; Bytes = (Get-PublishBytes (Get-Content $extraPath -Raw)) })
|
||||
$extraText = Get-Content $extraPath -Raw
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $extra.BlobName; Text = $extraText; Bytes = (Get-PublishBytes $extraText) })
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 5. sign
|
||||
# After this point the bytes are final. Normalising, re-encoding or rewriting a line ending
|
||||
# would invalidate the signature, and the device-side loader would refuse the script.
|
||||
if ($SigningEnabled) {
|
||||
Set-PublishSetSignature -Items $publishSet -WorkDir (Join-Path ([System.IO.Path]::GetTempPath()) "apm-publish-$PID")
|
||||
} else {
|
||||
Write-Host '::warning::No signing token supplied -- dry run continues with unsigned scripts.'
|
||||
}
|
||||
|
||||
$legacySas = if ($LegacySasToken) { $LegacySasToken.TrimStart('?') } else { '' }
|
||||
$writeSas = if ($SasToken) { $SasToken.TrimStart('?') } else { '' }
|
||||
|
||||
# Acquired lazily: a dry run must stay runnable without any Azure login -- the PR gate
|
||||
# (bootstrap-script-gates.yml) executes this script with -DryRun and never logs in.
|
||||
$authHeaders = if ($DryRun) { $null } else { & (Join-Path $PSScriptRoot 'Get-StorageAuthHeaders.ps1') }
|
||||
|
||||
# ------------------------------------------------------------------ 6. upload
|
||||
foreach ($item in $publishSet) {
|
||||
# Explicit type, not a convenience: an Object[] body is uploaded as space-separated
|
||||
# decimals rather than raw bytes, and every hash check still passes because [byte[]]
|
||||
@@ -227,11 +401,21 @@ foreach ($item in $publishSet) {
|
||||
[byte[]]$bytes = $item.Bytes
|
||||
$item | Add-Member -NotePropertyName Sha256 -NotePropertyValue (Get-Sha256 $bytes) -Force
|
||||
|
||||
# Exactly the bytes that go on the wire, for the CI artifact. Written in a dry run too:
|
||||
# that is the run an operator uses to inspect a signed script before it is published.
|
||||
if ($SignedOutputDir) {
|
||||
New-Item -ItemType Directory -Force -Path $SignedOutputDir | Out-Null
|
||||
[System.IO.File]::WriteAllBytes((Join-Path $SignedOutputDir $item.BlobName), $bytes)
|
||||
}
|
||||
|
||||
if ($DryRun) {
|
||||
# Doubles as a drift check: says per blob whether the alias already serves this source.
|
||||
# Doubles as a drift check: says per blob whether the alias already serves this
|
||||
# source. Compared as text without the signature block -- signed bytes differ on
|
||||
# every run, so a byte comparison would report drift for every unchanged script.
|
||||
$served = Get-PublishedBytes "$AliasUrl/$($item.BlobName)"
|
||||
$state = if ($null -eq $served) { 'MISSING on the alias' }
|
||||
elseif ((Get-Sha256 $served) -eq $item.Sha256) { 'already current' }
|
||||
elseif ((Get-ComparableText ([System.Text.Encoding]::UTF8.GetString($served))) -eq
|
||||
(Get-ComparableText $item.Text)) { 'already current' }
|
||||
else { 'STALE on the alias' }
|
||||
Write-Host " [dry-run] $($item.BlobName) ($($bytes.Length) bytes, sha256 $($item.Sha256)) -- $state"
|
||||
continue
|
||||
@@ -241,8 +425,18 @@ foreach ($item in $publishSet) {
|
||||
# with Front Door. Route caching is disabled there, but a stale script paired with a
|
||||
# fresh manifest would fail the bootstrap SHA check -- this keeps that class of bug
|
||||
# impossible even if caching is ever re-enabled.
|
||||
$headers = @{ 'x-ms-blob-type' = 'BlockBlob'; 'Content-Type' = $ScriptContentType; 'x-ms-blob-cache-control' = 'no-cache' }
|
||||
Invoke-RestMethod -Uri "$ContainerUrl/$($item.BlobName)?$writeSas" -Method Put -Headers $headers -Body $bytes | Out-Null
|
||||
#
|
||||
# Content-Disposition: without it a text/plain blob opens as a web page in every
|
||||
# browser, and the documented "download the script" step silently becomes "copy this
|
||||
# text". The header makes the link a download everywhere, and command-line clients
|
||||
# (the loader, the MSI, curl) ignore it.
|
||||
$headers = @{
|
||||
'x-ms-blob-type' = 'BlockBlob'
|
||||
'Content-Type' = $ScriptContentType
|
||||
'x-ms-blob-cache-control' = 'no-cache'
|
||||
'x-ms-blob-content-disposition' = ('attachment; filename="{0}"' -f $item.BlobName)
|
||||
}
|
||||
Invoke-RestMethod -Uri "$ContainerUrl/$($item.BlobName)" -Method Put -Headers ($authHeaders + $headers) -Body $bytes | Out-Null
|
||||
|
||||
# Read straight back from the blob (authoritative, no CDN in the way) before touching the
|
||||
# mirror or the next file. The alias verification at the end would catch a bad body too,
|
||||
@@ -266,7 +460,7 @@ foreach ($item in $publishSet) {
|
||||
}
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 5. version oracles
|
||||
# ------------------------------------------------------------------ 7. version oracles
|
||||
# version.json is the agent manifest; only bootstrapVersion belongs to us. Read from the
|
||||
# blob (not the alias) for an authoritative ETag, write back under If-Match so a concurrent
|
||||
# agent release cannot lose its version/sha256 fields.
|
||||
@@ -278,11 +472,15 @@ if (-not $DryRun) {
|
||||
} else { [string]$manifestResp.Content }
|
||||
$manifest = $manifestRaw | ConvertFrom-Json
|
||||
|
||||
if ($manifest.bootstrapVersion -eq $scriptVersion) {
|
||||
Write-Host "version.json already reports bootstrapVersion $scriptVersion"
|
||||
if ($manifest.bootstrapVersion -eq $scriptVersion -and $manifest.loaderVersion -eq $loaderVersion) {
|
||||
Write-Host "version.json already reports bootstrapVersion $scriptVersion and loaderVersion $loaderVersion"
|
||||
} else {
|
||||
$etag = '"' + ([string]($manifestResp.Headers['ETag'] | Select-Object -First 1)).Trim('"') + '"'
|
||||
$manifest | Add-Member -NotePropertyName 'bootstrapVersion' -NotePropertyValue $scriptVersion -Force
|
||||
# Additive: no consumer reads loaderVersion yet. It exists so support can tell
|
||||
# which stage-1 file a device was assigned -- stage 2 is always current by
|
||||
# construction, so its version says nothing about the customer's Intune copy.
|
||||
$manifest | Add-Member -NotePropertyName 'loaderVersion' -NotePropertyValue $loaderVersion -Force
|
||||
$body = [System.Text.Encoding]::UTF8.GetBytes(($manifest | ConvertTo-Json -Compress))
|
||||
$manifestHeaders = @{
|
||||
'x-ms-blob-type' = 'BlockBlob'
|
||||
@@ -290,8 +488,8 @@ if (-not $DryRun) {
|
||||
'x-ms-blob-cache-control' = 'no-cache'
|
||||
'If-Match' = $etag
|
||||
}
|
||||
Invoke-RestMethod -Uri "$ContainerUrl/version.json?$writeSas" -Method Put -Headers $manifestHeaders -Body $body | Out-Null
|
||||
Write-Host "version.json bootstrapVersion -> $scriptVersion (agent fields untouched)"
|
||||
Invoke-RestMethod -Uri "$ContainerUrl/version.json" -Method Put -Headers ($authHeaders + $manifestHeaders) -Body $body | Out-Null
|
||||
Write-Host "version.json bootstrapVersion -> $scriptVersion, loaderVersion -> $loaderVersion (agent fields untouched)"
|
||||
|
||||
if ($legacySas) {
|
||||
try {
|
||||
@@ -310,11 +508,8 @@ if (-not $DryRun) {
|
||||
}
|
||||
}
|
||||
|
||||
$tableSas = if ($TableSasToken) { $TableSasToken.TrimStart('?') } else { '' }
|
||||
if ($DryRun) {
|
||||
Write-Host " [dry-run] would set AdminConfiguration.LatestBootstrapV2ScriptVersion = $scriptVersion"
|
||||
} elseif ([string]::IsNullOrEmpty($tableSas) -or $tableSas -eq 'XXX') {
|
||||
Write-Host 'SKIPPED: AdminConfiguration update -- set AZURE_TABLE_ADMIN_SAS_TOKEN to enable the portal version oracle'
|
||||
} else {
|
||||
$tableHeaders = @{
|
||||
'Content-Type' = 'application/json'
|
||||
@@ -322,11 +517,11 @@ if ($DryRun) {
|
||||
'If-Match' = '*'
|
||||
}
|
||||
$tableBody = @{ LatestBootstrapV2ScriptVersion = $scriptVersion } | ConvertTo-Json
|
||||
Invoke-RestMethod -Uri "$TableUrl`?$tableSas" -Method Merge -Headers $tableHeaders -Body $tableBody | Out-Null
|
||||
Invoke-RestMethod -Uri $TableUrl -Method Merge -Headers ($authHeaders + $tableHeaders) -Body $tableBody | Out-Null
|
||||
Write-Host "AdminConfiguration.LatestBootstrapV2ScriptVersion = $scriptVersion"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 6. verify via alias
|
||||
# ------------------------------------------------------------------ 8. verify via alias
|
||||
if (-not $DryRun) {
|
||||
$pending = [System.Collections.ArrayList]::new()
|
||||
$publishSet | ForEach-Object { [void]$pending.Add($_) }
|
||||
@@ -358,7 +553,8 @@ if ($env:GITHUB_STEP_SUMMARY) {
|
||||
$summary = @(
|
||||
"### Bootstrap scripts: $mode",
|
||||
'',
|
||||
"Bootstrap script version: **$scriptVersion**",
|
||||
"Loader script version: **$loaderVersion** | Bootstrap script version: **$scriptVersion**",
|
||||
"Signed: **$(if ($SigningEnabled) { "yes, certificate '$CertificateName'" } else { 'NO (unsigned dry run)' })**",
|
||||
''
|
||||
) + ($publishSet | ForEach-Object { "- $($_.BlobName) -- sha256 $($_.Sha256)" })
|
||||
($summary -join [Environment]::NewLine) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
A failed build burns a number, which is harmless -- uniqueness matters,
|
||||
density does not.
|
||||
|
||||
The blob is public-read; only the write needs the container SAS.
|
||||
The blob is public-read; only the write needs authorization, and that comes from the
|
||||
ambient Entra identity (see Get-StorageAuthHeaders.ps1) -- no secret is passed in.
|
||||
|
||||
Why a counter and not the published version manifest: manifests are written
|
||||
on publish, not on build (the agent writes version.json only for stable
|
||||
@@ -20,10 +21,7 @@
|
||||
overwrite each other's versioned artifact.
|
||||
|
||||
.PARAMETER CounterUrl
|
||||
Full URL of the counter blob, without SAS.
|
||||
|
||||
.PARAMETER SasToken
|
||||
Container SAS with create+write permission. A leading '?' is tolerated.
|
||||
Full URL of the counter blob.
|
||||
|
||||
.PARAMETER Override
|
||||
Explicit build number instead of counter+1. If it is not greater than the
|
||||
@@ -37,20 +35,19 @@
|
||||
[int] the reserved build number. Progress goes to the host, not the pipeline.
|
||||
|
||||
.EXAMPLE
|
||||
$n = ./Request-BuildNumber.ps1 -CounterUrl $url -SasToken $sas
|
||||
$n = ./Request-BuildNumber.ps1 -CounterUrl $url
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$CounterUrl,
|
||||
[Parameter(Mandatory = $true)][string]$SasToken,
|
||||
[int]$Override = 0,
|
||||
[int]$MaxAttempts = 5
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Tolerate both secret formats ("sp=cw&..." and "?sp=cw&...").
|
||||
$writeUrl = $CounterUrl + '?' + $SasToken.TrimStart('?')
|
||||
# The write is Entra-authorized; the read below stays anonymous (public container).
|
||||
$authHeaders = & (Join-Path $PSScriptRoot 'Get-StorageAuthHeaders.ps1')
|
||||
|
||||
function Get-HttpStatus {
|
||||
param($ErrorRecord)
|
||||
@@ -65,7 +62,7 @@ function Read-Counter {
|
||||
$resp = Invoke-WebRequest -Uri $CounterUrl -Method Get -UseBasicParsing
|
||||
} catch {
|
||||
if ((Get-HttpStatus $_) -eq 404) {
|
||||
throw "Counter blob not found at $CounterUrl. Seed it once with the last used build number: PUT the plain integer with 'x-ms-blob-type: BlockBlob' and 'If-None-Match: *'. See docs/versioning.md."
|
||||
throw "Counter blob not found at $CounterUrl. Seed it once with the last used build number: PUT the plain integer with 'x-ms-blob-type: BlockBlob' and 'If-None-Match: *'. See internal/docs/versioning.md."
|
||||
}
|
||||
throw
|
||||
}
|
||||
@@ -85,8 +82,8 @@ for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) {
|
||||
$target = if ($Override -gt 0) { $Override } else { $counter.Value + 1 }
|
||||
|
||||
try {
|
||||
$headers = @{ 'x-ms-blob-type' = 'BlockBlob'; 'If-Match' = $counter.ETag }
|
||||
Invoke-RestMethod -Uri $writeUrl -Method Put -Headers $headers -Body "$target" -ContentType 'text/plain' | Out-Null
|
||||
$headers = $authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'If-Match' = $counter.ETag }
|
||||
Invoke-RestMethod -Uri $CounterUrl -Method Put -Headers $headers -Body "$target" -ContentType 'text/plain' | Out-Null
|
||||
Write-Host " Reserved build number $target (CAS write ok, attempt $attempt)"
|
||||
return $target
|
||||
} catch {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
name: Workflow gates
|
||||
|
||||
# Two checks over the workflow files themselves:
|
||||
# * verify-pins -- every third-party action is referenced by a full commit SHA plus a
|
||||
# version comment (stricter than zizmor's unpinned-uses audit).
|
||||
# * zizmor -- static analysis of every workflow (permissions, credential
|
||||
# persistence, template injection, ...). Configuration and the
|
||||
# reasoned ignores live in zizmor.yml at the repository root; the
|
||||
# same run is expected locally before a workflow change is committed.
|
||||
#
|
||||
# Every third-party action must be referenced by a full commit SHA, never by a tag.
|
||||
# Tags are mutable: whoever controls an action repository can repoint v4 at new code,
|
||||
# and that code runs inside jobs that hold the code-signing identity, the storage
|
||||
# credentials and a token that can push to main. A SHA is the only reference an
|
||||
# upstream owner cannot change under us.
|
||||
#
|
||||
# Lives in its own workflow rather than in ci.yml: CI deliberately ignores changes to
|
||||
# the other workflow files, so a guard placed there would stay silent for exactly the
|
||||
# commits it needs to inspect. Dependabot's action-pin PRs are the main customer: they
|
||||
# rewrite workflow files without anyone running zizmor locally.
|
||||
#
|
||||
# Like CI, this reports rather than blocks: main is deliberately unprotected, so a red
|
||||
# run is a signal to act on, not a merge barrier. Making it binding means adding
|
||||
# 'Verify action pins' as a required status check -- a branch-protection decision, not
|
||||
# something this file can assert.
|
||||
#
|
||||
# The pins are not maintained by hand — .github/dependabot.yml opens a monthly grouped
|
||||
# PR that rewrites both the SHA and the trailing version comment. The comment is
|
||||
# therefore required too: it is what makes a pinned line reviewable.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['.github/workflows/**', '.github/dependabot.yml', 'zizmor.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['.github/workflows/**', '.github/dependabot.yml', 'zizmor.yml']
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
verify-pins:
|
||||
name: Verify action pins
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Every third-party action is pinned to a SHA
|
||||
run: |
|
||||
set -uo pipefail
|
||||
failed=0
|
||||
while IFS=: read -r file line ref; do
|
||||
# Reusable workflows from this repository (workspace-relative ./ or the
|
||||
# self-repository form $/) move with the commit that calls them, so there is
|
||||
# nothing to pin.
|
||||
case "$ref" in ./*|\$/*) continue ;; esac
|
||||
|
||||
if [[ "$ref" =~ ^[^@]+@[0-9a-f]{40}[[:space:]]+#[[:space:]]*[^[:space:]] ]]; then
|
||||
printf 'ok %s:%s %s\n' "$file" "$line" "$ref"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$ref" =~ ^[^@]+@[0-9a-f]{40}[[:space:]]*$ ]]; then
|
||||
echo "::error file=$file,line=$line::Pinned but the version comment is missing: '$ref'. Write 'owner/action@<sha> # vN' so the line stays reviewable."
|
||||
else
|
||||
echo "::error file=$file,line=$line::Action '$ref' is not pinned to a commit SHA. Tags are mutable -- resolve it with: gh api repos/<owner>/<action>/git/ref/tags/<tag>"
|
||||
fi
|
||||
failed=1
|
||||
# -H is load-bearing: with a single matching file grep omits the name, the
|
||||
# fields shift by one and every diagnostic points at nothing.
|
||||
done < <(grep -nHE '^[[:space:]]*(-[[:space:]]+)?uses:' .github/workflows/*.yml \
|
||||
| sed -E 's/^([^:]+):([0-9]+):[[:space:]]*(-[[:space:]]+)?uses:[[:space:]]*/\1:\2:/')
|
||||
|
||||
if [ "$failed" -ne 0 ]; then
|
||||
echo "::error::Unpinned action references found -- fix them before this reaches main."
|
||||
exit 1
|
||||
fi
|
||||
echo "All action references are pinned to commit SHAs."
|
||||
|
||||
zizmor:
|
||||
name: zizmor audit
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write # SARIF upload: findings land in the Code scanning tab next to CodeQL
|
||||
actions: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# zizmor itself is version-pinned like every other tool here; bump it together with
|
||||
# the local install so the gate and the pre-commit run agree on the finding set.
|
||||
# Informational findings (template expansion of our own step outputs in pwsh) are
|
||||
# visible locally but do not fail the gate.
|
||||
- uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
|
||||
with:
|
||||
version: 1.30.1
|
||||
min-severity: low
|
||||
@@ -18,7 +18,9 @@ jobs:
|
||||
name: Bootstrap guard tests (PS 5.1)
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Pester 5+
|
||||
shell: powershell
|
||||
|
||||
@@ -27,6 +27,10 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
release_notes:
|
||||
description: 'Customer-facing notes for the GitHub Release (Markdown bullets). Empty = generic fallback linking to the docs changelog'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -53,6 +57,13 @@ env:
|
||||
VERSIONED_ZIP: AutopilotMonitor-Agent-v2.zip
|
||||
STABLE_ZIP: AutopilotMonitor-Agent.zip
|
||||
STABLE_MANIFEST: version.json
|
||||
# Authenticode via Azure Key Vault; identity and vault come from repository variables.
|
||||
# CODE_SIGN_EXPECTED_SUBJECT is optional and pins the publisher in the verify step.
|
||||
CODE_SIGN_CLIENT_ID: ${{ vars.CODE_SIGN_CLIENT_ID }}
|
||||
CODE_SIGN_CERTIFICATE_NAME: ${{ vars.CODE_SIGN_CERTIFICATE_NAME }}
|
||||
CODE_SIGN_KEY_VAULT_URL: ${{ vars.CODE_SIGN_KEY_VAULT_URL }}
|
||||
CODE_SIGN_EXPECTED_SUBJECT: ${{ vars.CODE_SIGN_EXPECTED_SUBJECT }}
|
||||
CODE_SIGN_TIMESTAMP_URL: http://timestamp.acs.microsoft.com
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -61,14 +72,68 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Every released build is signed; a missing variable stops the run before a build
|
||||
# number is burnt.
|
||||
- name: Check code-signing configuration
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($name in 'CODE_SIGN_CLIENT_ID', 'CODE_SIGN_CERTIFICATE_NAME', 'CODE_SIGN_KEY_VAULT_URL') {
|
||||
if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($name))) {
|
||||
throw "Repository variable $name is not set. Every released build must be Authenticode-signed."
|
||||
}
|
||||
}
|
||||
Write-Host "Code-signing configuration present (certificate '$env:CODE_SIGN_CERTIFICATE_NAME')"
|
||||
|
||||
# Logs in FIRST and only to fetch the Key Vault token: azure/login sets the default az
|
||||
# account, so the storage login below has to be the last one. Both federated credentials
|
||||
# are bound to the ref, so this job must never be given an `environment:` -- that would
|
||||
# change the OIDC subject claim and break both logins.
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ env.CODE_SIGN_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
|
||||
- name: Acquire Key Vault token for signing
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$token = az account get-access-token --scope https://vault.azure.net/.default --query accessToken --output tsv
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw "Could not acquire a Key Vault token for the code-signing identity."
|
||||
}
|
||||
Write-Host "::add-mask::$token"
|
||||
"CODE_SIGN_AKV_ACCESS_TOKEN=$token" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Every storage write below (counter blob, agent ZIP, manifests, AdminConfiguration)
|
||||
# is authorized by this federated identity -- no SAS secret. One login covers the whole
|
||||
# job: the federated credential is bound to the ref, and a release run takes ~2 minutes,
|
||||
# well inside the token lifetime. Only the legacy mirror still needs a SAS: that account
|
||||
# lives in the pre-migration tenant, where this principal has no identity.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# SDK 10 = single build toolchain (parity with local build.ps1 / global.json).
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v6
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
# Version pinned; roll-forward because the tool targets an older runtime than the SDK above.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
# Build numbers come from the shared counter blob -- one counter and one
|
||||
# implementation for CI and the local build (.github/scripts/Request-BuildNumber.ps1).
|
||||
# Reserve-before-build with ETag-CAS: concurrent builds can never mint the same
|
||||
@@ -81,7 +146,6 @@ jobs:
|
||||
# build_number could otherwise break out of the quotes and run arbitrary code.
|
||||
env:
|
||||
BUILD_NUMBER_OVERRIDE: ${{ inputs.build_number }}
|
||||
AGENT_SAS_TOKEN: ${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}
|
||||
run: |
|
||||
$override = 0
|
||||
if ($env:BUILD_NUMBER_OVERRIDE -ne '') {
|
||||
@@ -92,7 +156,7 @@ jobs:
|
||||
}
|
||||
|
||||
$counterUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent/buildcounter-v$($env:LINE_MAJOR).txt"
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl $counterUrl -SasToken $env:AGENT_SAS_TOKEN -Override $override
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl $counterUrl -Override $override
|
||||
|
||||
$csproj = Get-Content $env:CSPROJ_PATH -Raw
|
||||
if ($csproj -match '<VersionPrefix>(\d+\.\d+)</VersionPrefix>') {
|
||||
@@ -125,6 +189,47 @@ jobs:
|
||||
- name: Run Agent unit tests
|
||||
run: dotnet test ${{ env.TEST_PROJECT }} --configuration Release --no-restore --verbosity quiet
|
||||
|
||||
# Authenticode over everything we author; third-party assemblies keep their own signature.
|
||||
#
|
||||
# THE POSITION IS LOAD-BEARING: signing rewrites the PE file, so it must run before the
|
||||
# ZIP and every SHA-256 in the next step -- those hashes are the bootstrap's integrity
|
||||
# oracle, and hashes taken before signing match no downloaded file.
|
||||
#
|
||||
# $LASTEXITCODE is checked explicitly: PowerShell does not throw on a native non-zero exit.
|
||||
- name: Sign agent binaries (Authenticode)
|
||||
shell: pwsh
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$targets = @()
|
||||
$targets += Get-ChildItem -Path "$env:AGENT_OUTPUT/*" -Include 'AutopilotMonitor.*.exe', 'AutopilotMonitor.*.dll' -File
|
||||
$targets += Get-ChildItem -Path "$env:DIALOG_OUTPUT/AutopilotMonitor.SummaryDialog.exe" -File
|
||||
|
||||
if (-not ($targets | Where-Object Name -eq 'AutopilotMonitor.Agent.exe')) {
|
||||
throw "AutopilotMonitor.Agent.exe not found in $env:AGENT_OUTPUT -- nothing to sign."
|
||||
}
|
||||
if (-not ($targets | Where-Object Name -eq 'AutopilotMonitor.SummaryDialog.exe')) {
|
||||
throw "AutopilotMonitor.SummaryDialog.exe not found in $env:DIALOG_OUTPUT -- nothing to sign."
|
||||
}
|
||||
|
||||
$targets | ForEach-Object { Write-Host "Signing $($_.Name)" }
|
||||
|
||||
& azuresigntool.exe sign --verbose `
|
||||
--azure-key-vault-url "$env:CODE_SIGN_KEY_VAULT_URL" `
|
||||
--azure-key-vault-accesstoken "$env:CODE_SIGN_AKV_ACCESS_TOKEN" `
|
||||
--azure-key-vault-certificate "$env:CODE_SIGN_CERTIFICATE_NAME" `
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description "Autopilot Monitor Agent" `
|
||||
--description-url "https://www.autopilotmonitor.com" `
|
||||
($targets | ForEach-Object { $_.FullName })
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "azuresigntool failed with exit code $LASTEXITCODE -- aborting before anything is published."
|
||||
}
|
||||
|
||||
- name: Create release artifact
|
||||
id: artifact
|
||||
shell: pwsh
|
||||
@@ -215,12 +320,57 @@ jobs:
|
||||
"sha256=$sha256Hash" >> $env:GITHUB_OUTPUT
|
||||
"exe_sha256=$exeSha256Hash" >> $env:GITHUB_OUTPUT
|
||||
|
||||
# Verified against the STAGING copy -- the exact bytes that ship. The invariant is "every
|
||||
# AutopilotMonitor PE in the ZIP is signed", so a project added later cannot ride along
|
||||
# unsigned. A missing timestamp fails too: without it the signature dies with the cert.
|
||||
- name: Verify Authenticode signatures
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$files = Get-ChildItem -Path "staging/agent/*" -Include 'AutopilotMonitor.*.exe', 'AutopilotMonitor.*.dll' -File
|
||||
if ($files.Count -lt 2) {
|
||||
throw "Expected signed binaries in staging/agent, found $($files.Count)."
|
||||
}
|
||||
|
||||
$failed = $false
|
||||
$thumbprints = @{}
|
||||
foreach ($file in $files) {
|
||||
$sig = Get-AuthenticodeSignature -FilePath $file.FullName
|
||||
if ($sig.Status -ne 'Valid') {
|
||||
Write-Host "::error file=$($file.Name)::Signature status is '$($sig.Status)' -- expected 'Valid'."
|
||||
$failed = $true
|
||||
continue
|
||||
}
|
||||
if (-not $sig.TimeStamperCertificate) {
|
||||
Write-Host "::error file=$($file.Name)::Signature carries no RFC-3161 timestamp -- it would expire with the certificate."
|
||||
$failed = $true
|
||||
continue
|
||||
}
|
||||
if (-not [string]::IsNullOrWhiteSpace($env:CODE_SIGN_EXPECTED_SUBJECT) -and
|
||||
$sig.SignerCertificate.Subject -notlike $env:CODE_SIGN_EXPECTED_SUBJECT) {
|
||||
Write-Host "::error file=$($file.Name)::Signed by '$($sig.SignerCertificate.Subject)', expected '$env:CODE_SIGN_EXPECTED_SUBJECT'."
|
||||
$failed = $true
|
||||
continue
|
||||
}
|
||||
$thumbprints[$sig.SignerCertificate.Thumbprint] = $true
|
||||
Write-Host "$($file.Name): Valid, timestamped $($sig.TimeStamperCertificate.Subject.Split(',')[0]), signer $($sig.SignerCertificate.Subject)"
|
||||
}
|
||||
|
||||
if ($thumbprints.Count -gt 1) {
|
||||
Write-Host "::error::Staged binaries carry $($thumbprints.Count) different signing certificates -- expected exactly one."
|
||||
$failed = $true
|
||||
}
|
||||
if ($failed) {
|
||||
throw "Signature verification failed. Nothing was published."
|
||||
}
|
||||
Write-Host "All $($files.Count) binaries signed with certificate $($thumbprints.Keys -join ', ')"
|
||||
|
||||
- name: Upload to Azure Blob Storage
|
||||
if: inputs.upload_blob
|
||||
shell: pwsh
|
||||
run: |
|
||||
$containerUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent"
|
||||
$sasToken = "${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}".TrimStart('?')
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
# Legacy mirror (transition until customers moved to the download.-alias):
|
||||
# blobs are ALSO pushed to the old account so the bootstrap URL baked into
|
||||
# customers' Intune scripts keeps serving current bits. Fail-soft -- a
|
||||
@@ -230,7 +380,6 @@ jobs:
|
||||
|
||||
function Send-Blob {
|
||||
param([string]$Local, [string]$BlobName, [string]$ContentType)
|
||||
$url = "$containerUrl/$BlobName`?$sasToken"
|
||||
$bytes = [System.IO.File]::ReadAllBytes($Local)
|
||||
# no-cache on EVERY blob: they all rotate in place (even the per-line ZIP is
|
||||
# overwritten each build), and the download alias fronts them with Front Door.
|
||||
@@ -244,7 +393,7 @@ jobs:
|
||||
$md5.Dispose()
|
||||
}
|
||||
Write-Host "Uploading $BlobName ($([math]::Round($bytes.Length / 1MB, 2)) MB)..."
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers $headers -Body $bytes
|
||||
Invoke-RestMethod -Uri "$containerUrl/$BlobName" -Method Put -Headers ($authHeaders + $headers) -Body $bytes
|
||||
Write-Host " uploaded $BlobName"
|
||||
if (-not [string]::IsNullOrEmpty($legacySasToken)) {
|
||||
try {
|
||||
@@ -270,63 +419,79 @@ jobs:
|
||||
# to main, so a release lands a set that is consistent with what is already live
|
||||
# (dev render, version-bump guard, bootstrapVersion oracles, alias verification).
|
||||
# Runs AFTER version.json so its bootstrapVersion read sees this build's manifest.
|
||||
# Signed with the same identity: the device-side loader verifies the publisher
|
||||
# of what it downloads, and the publisher refuses to upload unsigned scripts.
|
||||
./.github/scripts/Publish-BootstrapScripts.ps1 `
|
||||
-SasToken '${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}' `
|
||||
-LegacySasToken '${{ secrets.AZURE_BLOB_AGENT_LEGACY_SAS_TOKEN }}' `
|
||||
-TableSasToken '${{ secrets.AZURE_TABLE_ADMIN_SAS_TOKEN }}'
|
||||
-SigningToken $env:CODE_SIGN_AKV_ACCESS_TOKEN `
|
||||
-KeyVaultUrl $env:CODE_SIGN_KEY_VAULT_URL `
|
||||
-CertificateName $env:CODE_SIGN_CERTIFICATE_NAME `
|
||||
-TimestampUrl $env:CODE_SIGN_TIMESTAMP_URL `
|
||||
-ExpectedPublisher $env:CODE_SIGN_EXPECTED_SUBJECT
|
||||
}
|
||||
|
||||
# Update AdminConfiguration LatestAgentV2* -- ONLY on stable cutover. The hash-oracle
|
||||
# must never point at a build that is not actually in the stable namespace.
|
||||
# LatestBootstrapV2ScriptVersion is NOT set here: it belongs to the script publisher
|
||||
# above, which also owns it on script-only pushes. One writer per field.
|
||||
$tableSasToken = "${{ secrets.AZURE_TABLE_ADMIN_SAS_TOKEN }}".TrimStart('?')
|
||||
if ('${{ inputs.publish_as_stable }}' -ne 'true') {
|
||||
Write-Host "SKIPPED: AdminConfiguration update (versioned-only run -- stable namespace unchanged)"
|
||||
} elseif ([string]::IsNullOrEmpty($tableSasToken) -or $tableSasToken -eq "XXX") {
|
||||
Write-Host "SKIPPED: Table SAS token not configured -- set AZURE_TABLE_ADMIN_SAS_TOKEN secret to enable backend hash-oracle"
|
||||
} else {
|
||||
$sha256 = "${{ steps.artifact.outputs.sha256 }}"
|
||||
$exeSha256 = "${{ steps.artifact.outputs.exe_sha256 }}"
|
||||
$agentVersion = "${{ steps.version.outputs.version }}"
|
||||
$tableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')?$tableSasToken"
|
||||
$tableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')"
|
||||
$tableBody = @{
|
||||
LatestAgentV2Version = $agentVersion
|
||||
LatestAgentV2Sha256 = $sha256
|
||||
LatestAgentV2ExeSha256 = $exeSha256
|
||||
}
|
||||
Invoke-RestMethod -Uri $tableUrl -Method Merge -Headers @{
|
||||
Invoke-RestMethod -Uri $tableUrl -Method Merge -Headers ($authHeaders + @{
|
||||
"Content-Type" = "application/json"
|
||||
"Accept" = "application/json;odata=nometadata"
|
||||
"If-Match" = "*"
|
||||
} -Body ($tableBody | ConvertTo-Json)
|
||||
}) -Body ($tableBody | ConvertTo-Json)
|
||||
Write-Host "AdminConfiguration updated: V2 version=$agentVersion, sha256=$sha256, exeSha256=$exeSha256"
|
||||
}
|
||||
|
||||
# Sigstore keyless provenance: binds the ZIP digest to this workflow run (repo, ref,
|
||||
# commit, workflow file) in a public transparency log. The digest covers the identical
|
||||
# Sigstore keyless provenance: binds the artifact digests to this workflow run (repo, ref,
|
||||
# commit, workflow file) in a public transparency log. The digests cover the identical
|
||||
# bytes uploaded to blob (versioned AND stable), so one attestation covers all copies.
|
||||
# Verify: gh attestation verify <zip> --repo okieselbach/AutopilotMonitor
|
||||
- name: Attest build provenance (ZIP)
|
||||
uses: actions/attest-build-provenance@v4
|
||||
# Complements Authenticode rather than replacing it: the signature says who published a
|
||||
# binary, the attestation which build produced these exact bytes. Both EXEs are subjects
|
||||
# of their own so an extracted binary can be verified without the ZIP.
|
||||
# Verify: gh attestation verify <file> --repo okieselbach/AutopilotMonitor
|
||||
# Skipped on runs that publish nothing -- no log entry for bytes nobody receives.
|
||||
- name: Attest build provenance
|
||||
if: inputs.upload_blob || inputs.create_release
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
with:
|
||||
subject-path: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
subject-path: |
|
||||
${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
staging/agent/AutopilotMonitor.Agent.exe
|
||||
staging/agent/AutopilotMonitor.SummaryDialog.exe
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: AutopilotMonitor-Agent-v${{ steps.version.outputs.version }}
|
||||
path: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
|
||||
- name: Upload version metadata
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: version-metadata
|
||||
path: ${{ env.MANIFEST_NAME }}
|
||||
|
||||
# First-party `gh` instead of a third-party action: this job holds contents:write,
|
||||
# id-token:write and attestations:write, so no code we do not control runs here.
|
||||
# --generate-notes appends the auto-generated notes below the body.
|
||||
# The "What's new" section comes ONLY from the release_notes input -- CI never
|
||||
# composes customer-facing text from commits. An empty input writes a generic
|
||||
# fallback that links to the docs changelog and carries the HTML-comment marker
|
||||
# below; the operator tooling scans release bodies for that marker to find the
|
||||
# last release with real notes, so the next noted release covers the gap.
|
||||
# Never rename the marker. --generate-notes is not used: it lists pull requests
|
||||
# only (Dependabot noise); the compare link to the previous tag replaces it.
|
||||
- name: Create GitHub Release
|
||||
if: inputs.create_release
|
||||
shell: pwsh
|
||||
@@ -336,19 +501,8 @@ jobs:
|
||||
RELEASE_NAME: Autopilot Monitor Agent ${{ steps.version.outputs.version }}
|
||||
RELEASE_PRERELEASE: ${{ inputs.prerelease }}
|
||||
RELEASE_ZIP: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
RELEASE_BODY: |
|
||||
## Autopilot Monitor Agent ${{ steps.version.outputs.version }}
|
||||
|
||||
### Installation
|
||||
1. Download `AutopilotMonitor-Agent-v${{ steps.version.outputs.version }}.zip`
|
||||
2. Extract to `C:\ProgramData\AutopilotMonitor\Agent\`
|
||||
3. Run `AutopilotMonitor.Agent.exe --install` to register the Scheduled Task
|
||||
|
||||
### Included Components
|
||||
- **AutopilotMonitor.Agent.exe** - Monitoring agent (V2 release line)
|
||||
- **AutopilotMonitor.SummaryDialog.exe** - Enrollment summary dialog (WPF)
|
||||
- **version-v2.json** - Per-line version manifest
|
||||
|
||||
RELEASE_NOTES: ${{ inputs.release_notes }}
|
||||
RELEASE_BODY_TAIL: |
|
||||
### Version Info
|
||||
- **Version:** ${{ steps.version.outputs.version }}
|
||||
- **Build Number:** ${{ steps.version.outputs.build_number }}
|
||||
@@ -357,14 +511,35 @@ jobs:
|
||||
- **Published as stable:** ${{ inputs.publish_as_stable }}
|
||||
|
||||
### Provenance
|
||||
This ZIP carries a [GitHub Artifact Attestation](https://github.com/okieselbach/AutopilotMonitor/attestations) (Sigstore keyless). Verify:
|
||||
The executables are Authenticode-signed and timestamped. The ZIP and both executables carry a [GitHub Artifact Attestation](https://github.com/okieselbach/AutopilotMonitor/attestations) (Sigstore keyless). Verify:
|
||||
```
|
||||
gh attestation verify AutopilotMonitor-Agent-v${{ steps.version.outputs.version }}.zip --repo okieselbach/AutopilotMonitor
|
||||
```
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$notes = "$env:RELEASE_NOTES".Trim()
|
||||
if ($notes -eq '') {
|
||||
$notes = "Minor enhancements and bug fixes. See the [Agent Changelog](https://docs.autopilotmonitor.com/changelog/agent-changelog) for details.`n<!-- release-notes: none -->"
|
||||
}
|
||||
|
||||
# Previous release for the compare link. Fail-soft: the first release has none,
|
||||
# and a listing hiccup must never fail a release.
|
||||
$prevTag = ''
|
||||
$eap = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
$prevTag = "$(gh release list --repo $env:GITHUB_REPOSITORY --limit 1 --exclude-drafts --json tagName -q '.[0].tagName' 2>$null)".Trim()
|
||||
} catch { $prevTag = '' }
|
||||
$ErrorActionPreference = $eap
|
||||
$compare = ''
|
||||
if ($prevTag -ne '' -and $prevTag -ne $env:RELEASE_TAG) {
|
||||
$compare = "`n`n**Full changelog:** https://github.com/$env:GITHUB_REPOSITORY/compare/$prevTag...$env:RELEASE_TAG`n"
|
||||
}
|
||||
|
||||
$body = "### What's new`n$notes`n`n" + "$env:RELEASE_BODY_TAIL".TrimEnd() + $compare
|
||||
$notesFile = Join-Path $env:RUNNER_TEMP 'release-notes.md'
|
||||
Set-Content -Path $notesFile -Value $env:RELEASE_BODY -Encoding utf8 -NoNewline
|
||||
Set-Content -Path $notesFile -Value $body -Encoding utf8 -NoNewline
|
||||
|
||||
$ghArgs = @(
|
||||
'release', 'create', $env:RELEASE_TAG,
|
||||
@@ -372,8 +547,7 @@ jobs:
|
||||
'--repo', $env:GITHUB_REPOSITORY,
|
||||
'--target', $env:GITHUB_SHA,
|
||||
'--title', $env:RELEASE_NAME,
|
||||
'--notes-file', $notesFile,
|
||||
'--generate-notes'
|
||||
'--notes-file', $notesFile
|
||||
)
|
||||
if ($env:RELEASE_PRERELEASE -eq 'true') { $ghArgs += '--prerelease' }
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
name: Build & Publish Bootstrap MSI
|
||||
|
||||
# Thin MDM LOB delivery vehicle for WDP (Autopilot v2) — see
|
||||
# src/Agent/AutopilotMonitor.BootstrapMsi/Package.wxs. The MSI carries no bootstrap
|
||||
# logic (only the runner script that downloads the CURRENT server-hosted
|
||||
# Install-AutopilotMonitor.ps1), so it is rebuilt rarely: only when the runner script
|
||||
# or the WiX package changes. Version lives in Package.wxs — bump it there before
|
||||
# dispatching so Intune LOB update detection sees a new ProductVersion.
|
||||
# src/Agent/AutopilotMonitor.BootstrapMsi/Package.wxs. The MSI is packaging only: it
|
||||
# carries the SAME stage-1 loader that is assigned directly in Intune, so it is rebuilt
|
||||
# rarely — only when that loader or the WiX package changes. Version lives in Package.wxs
|
||||
# — bump it there before dispatching so Intune LOB update detection sees a new
|
||||
# ProductVersion.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -23,8 +23,15 @@ permissions:
|
||||
|
||||
env:
|
||||
MSI_PROJECT_DIR: src/Agent/AutopilotMonitor.BootstrapMsi
|
||||
RUNNER_SCRIPT: src/Agent/AutopilotMonitor.BootstrapMsi/Invoke-BootstrapDownload.ps1
|
||||
RUNNER_SCRIPT: scripts/Bootstrap/Start-AutopilotMonitor.ps1
|
||||
STABLE_MSI: AutopilotMonitor-Bootstrap.msi
|
||||
# Authenticode via Azure Key Vault; identity and vault come from repository variables.
|
||||
# CODE_SIGN_EXPECTED_SUBJECT is optional and pins the publisher in the verify steps.
|
||||
CODE_SIGN_CLIENT_ID: ${{ vars.CODE_SIGN_CLIENT_ID }}
|
||||
CODE_SIGN_CERTIFICATE_NAME: ${{ vars.CODE_SIGN_CERTIFICATE_NAME }}
|
||||
CODE_SIGN_KEY_VAULT_URL: ${{ vars.CODE_SIGN_KEY_VAULT_URL }}
|
||||
CODE_SIGN_EXPECTED_SUBJECT: ${{ vars.CODE_SIGN_EXPECTED_SUBJECT }}
|
||||
CODE_SIGN_TIMESTAMP_URL: http://timestamp.acs.microsoft.com
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -33,10 +40,14 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# The runner executes as SYSTEM on every enrolling device — a parse error or
|
||||
# The loader executes as SYSTEM on every enrolling device — a parse error or
|
||||
# non-ASCII mojibake (PS 5.1 codepage roulette) must fail the build, not the fleet.
|
||||
# bootstrap-script-gates.yml runs the same checks plus Pester on pushes; this job
|
||||
# stands on its own because it can be dispatched at any time.
|
||||
- name: Validate runner script (parse + ASCII-only)
|
||||
shell: pwsh
|
||||
run: |
|
||||
@@ -62,8 +73,77 @@ jobs:
|
||||
Write-Host "MSI version (from Package.wxs): $version"
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
|
||||
# Every published MSI is signed; a missing variable stops the run before the build.
|
||||
- name: Check code-signing configuration
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($name in 'CODE_SIGN_CLIENT_ID', 'CODE_SIGN_CERTIFICATE_NAME', 'CODE_SIGN_KEY_VAULT_URL') {
|
||||
if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($name))) {
|
||||
throw "Repository variable $name is not set. Every published MSI must be Authenticode-signed."
|
||||
}
|
||||
}
|
||||
|
||||
# Logs in FIRST and only to fetch the Key Vault token: azure/login sets the default az
|
||||
# account, so the storage login below has to be the last one. Both federated credentials
|
||||
# are bound to the ref, so this job must never be given an `environment:` -- that would
|
||||
# change the OIDC subject claim and break both logins.
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ env.CODE_SIGN_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
|
||||
- name: Acquire Key Vault token for signing
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$token = az account get-access-token --scope https://vault.azure.net/.default --query accessToken --output tsv
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw "Could not acquire a Key Vault token for the code-signing identity."
|
||||
}
|
||||
Write-Host "::add-mask::$token"
|
||||
"CODE_SIGN_AKV_ACCESS_TOKEN=$token" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# The blob upload below is authorized by this federated identity, not by a SAS.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# Exact version, not a "5.*" range: wix writes the bytes of the MSI that customers
|
||||
# install, so a compromised or simply broken point release must not flow into a build
|
||||
# unnoticed. Bumping is a reviewed commit.
|
||||
- name: Install WiX toolset
|
||||
run: dotnet tool install --global wix --version "5.*"
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
|
||||
# Version pinned; roll-forward because the tool targets an older runtime than the SDK.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
# BEFORE the build: wix embeds the file as it finds it, so a loader signed afterwards
|
||||
# would leave the copy inside the MSI unsigned. Devices under an AllSigned policy need
|
||||
# the embedded loader to carry a signature of its own.
|
||||
- name: Sign bootstrap loader (Authenticode)
|
||||
shell: pwsh
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
& azuresigntool.exe sign --verbose `
|
||||
--azure-key-vault-url "$env:CODE_SIGN_KEY_VAULT_URL" `
|
||||
--azure-key-vault-accesstoken "$env:CODE_SIGN_AKV_ACCESS_TOKEN" `
|
||||
--azure-key-vault-certificate "$env:CODE_SIGN_CERTIFICATE_NAME" `
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor bootstrap runner' `
|
||||
--description-url 'https://www.autopilotmonitor.com' `
|
||||
$env:RUNNER_SCRIPT
|
||||
if ($LASTEXITCODE -ne 0) { throw "azuresigntool failed with exit code $LASTEXITCODE" }
|
||||
|
||||
- name: Build MSI
|
||||
shell: pwsh
|
||||
@@ -75,16 +155,46 @@ jobs:
|
||||
wix build Package.wxs -arch x64 -o $out
|
||||
Pop-Location
|
||||
if (-not (Test-Path $env:STABLE_MSI)) { Write-Error "wix build produced no MSI"; exit 1 }
|
||||
$sha256 = (Get-FileHash $env:STABLE_MSI -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
Write-Host "Built $env:STABLE_MSI ($([math]::Round((Get-Item $env:STABLE_MSI).Length / 1KB)) KB)"
|
||||
Write-Host "SHA-256: $sha256"
|
||||
|
||||
# Signing rewrites the MSI, so it runs before the hash and the upload. A missing
|
||||
# timestamp fails the run: without it the signature dies with the certificate.
|
||||
- name: Sign MSI (Authenticode)
|
||||
shell: pwsh
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
& azuresigntool.exe sign --verbose `
|
||||
--azure-key-vault-url "$env:CODE_SIGN_KEY_VAULT_URL" `
|
||||
--azure-key-vault-accesstoken "$env:CODE_SIGN_AKV_ACCESS_TOKEN" `
|
||||
--azure-key-vault-certificate "$env:CODE_SIGN_CERTIFICATE_NAME" `
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor Bootstrap' `
|
||||
--description-url 'https://www.autopilotmonitor.com' `
|
||||
$env:STABLE_MSI
|
||||
if ($LASTEXITCODE -ne 0) { throw "azuresigntool failed with exit code $LASTEXITCODE" }
|
||||
|
||||
# The MSI and the loader it carries -- verified here, before anything is published.
|
||||
foreach ($file in @($env:STABLE_MSI, $env:RUNNER_SCRIPT)) {
|
||||
$sig = Get-AuthenticodeSignature -FilePath $file
|
||||
if ($sig.Status -ne 'Valid') { throw "Signature on $file is '$($sig.Status)', expected 'Valid'." }
|
||||
if (-not $sig.TimeStamperCertificate) { throw "Signature on $file carries no timestamp." }
|
||||
if ($env:CODE_SIGN_EXPECTED_SUBJECT -and $sig.SignerCertificate.Subject -notlike $env:CODE_SIGN_EXPECTED_SUBJECT) {
|
||||
throw "$file was signed by '$($sig.SignerCertificate.Subject)', expected '$env:CODE_SIGN_EXPECTED_SUBJECT'."
|
||||
}
|
||||
Write-Host "$file : Valid, timestamped, signer $($sig.SignerCertificate.Subject.Split(',')[0])"
|
||||
}
|
||||
|
||||
Write-Host "SHA-256: $((Get-FileHash $env:STABLE_MSI -Algorithm SHA256).Hash.ToLowerInvariant())"
|
||||
|
||||
- name: Upload to Azure Blob Storage
|
||||
if: inputs.upload_blob
|
||||
shell: pwsh
|
||||
run: |
|
||||
$containerUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent"
|
||||
$sasToken = "${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}".TrimStart('?')
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$version = "${{ steps.version.outputs.version }}"
|
||||
|
||||
# Same Send-Blob contract as build-agent.yml: no-cache on every blob because the
|
||||
@@ -102,7 +212,7 @@ jobs:
|
||||
$headers["Content-MD5"] = [Convert]::ToBase64String($md5.ComputeHash($bytes))
|
||||
$md5.Dispose()
|
||||
Write-Host "Uploading $BlobName ($([math]::Round($bytes.Length / 1KB)) KB)..."
|
||||
Invoke-RestMethod -Uri "$containerUrl/$BlobName`?$sasToken" -Method Put -Headers $headers -Body $bytes
|
||||
Invoke-RestMethod -Uri "$containerUrl/$BlobName" -Method Put -Headers ($authHeaders + $headers) -Body $bytes
|
||||
Write-Host " uploaded $BlobName"
|
||||
}
|
||||
|
||||
@@ -125,15 +235,17 @@ jobs:
|
||||
}
|
||||
Write-Host "Alias verified: https://download.autopilotmonitor.com/agent/$env:STABLE_MSI ($remoteSize bytes)"
|
||||
|
||||
# Sigstore keyless provenance, same as the agent ZIP. Verify:
|
||||
# Sigstore keyless provenance, same as the agent ZIP; covers the signed bytes. Skipped
|
||||
# on runs that publish nothing -- no log entry for bytes nobody receives. Verify:
|
||||
# gh attestation verify AutopilotMonitor-Bootstrap.msi --repo okieselbach/AutopilotMonitor
|
||||
- name: Attest build provenance (MSI)
|
||||
uses: actions/attest-build-provenance@v4
|
||||
if: inputs.upload_blob
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
with:
|
||||
subject-path: ${{ env.STABLE_MSI }}
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: AutopilotMonitor-Bootstrap-v${{ steps.version.outputs.version }}
|
||||
path: ${{ env.STABLE_MSI }}
|
||||
|
||||
@@ -7,12 +7,82 @@ name: CI
|
||||
# The deploy workflows build but never test, so this is the only place the suites
|
||||
# actually run. It does not gate anything — main is deliberately unprotected, so a
|
||||
# red run reports rather than blocks. Read the result and follow up.
|
||||
|
||||
#
|
||||
# Path filters: commits that only bump a private submodule gitlink (internal/, infra/,
|
||||
# .claude/commands), touch prose or edit another workflow cannot change anything the
|
||||
# suites cover, so they are skipped instead of burning ~15 runner-minutes each. A gitlink bump reports the bare
|
||||
# path ('internal', no trailing slash), which 'internal/**' does NOT match — both forms
|
||||
# are listed. GitHub skips only when EVERY changed path matches; a commit that touches
|
||||
# docs AND code still runs.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'internal'
|
||||
- 'internal/**'
|
||||
- 'infra'
|
||||
- 'infra/**'
|
||||
- '.claude/**'
|
||||
- '.gitmodules'
|
||||
- 'zizmor.yml'
|
||||
- '**/*.md'
|
||||
- 'LICENSE'
|
||||
- '.github/ISSUE_TEMPLATE/**'
|
||||
- '.github/FUNDING.yml'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/assets/**'
|
||||
- '.github/codeql/**'
|
||||
- '.github/scripts/**'
|
||||
# Every other workflow: nothing here builds or tests anything CI covers. paths-ignore
|
||||
# cannot negate, so they are listed one by one; ci.yml itself stays out of the list
|
||||
# so a change to CI runs CI. A new workflow file is opt-in — add it here too.
|
||||
- '.github/workflows/action-pins.yml'
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
- '.github/workflows/build-agent.yml'
|
||||
- '.github/workflows/build-msi.yml'
|
||||
- '.github/workflows/codeql.yml'
|
||||
- '.github/workflows/combine-rules.yml'
|
||||
- '.github/workflows/deploy-backend.yml'
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'internal'
|
||||
- 'internal/**'
|
||||
- 'infra'
|
||||
- 'infra/**'
|
||||
- '.claude/**'
|
||||
- '.gitmodules'
|
||||
- 'zizmor.yml'
|
||||
- '**/*.md'
|
||||
- 'LICENSE'
|
||||
- '.github/ISSUE_TEMPLATE/**'
|
||||
- '.github/FUNDING.yml'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/assets/**'
|
||||
- '.github/codeql/**'
|
||||
- '.github/scripts/**'
|
||||
# Every other workflow: nothing here builds or tests anything CI covers. paths-ignore
|
||||
# cannot negate, so they are listed one by one; ci.yml itself stays out of the list
|
||||
# so a change to CI runs CI. A new workflow file is opt-in — add it here too.
|
||||
- '.github/workflows/action-pins.yml'
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
- '.github/workflows/build-agent.yml'
|
||||
- '.github/workflows/build-msi.yml'
|
||||
- '.github/workflows/codeql.yml'
|
||||
- '.github/workflows/combine-rules.yml'
|
||||
- '.github/workflows/deploy-backend.yml'
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -28,9 +98,11 @@ jobs:
|
||||
# windows-latest is required: the agent test projects target net48.
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-dotnet@v6
|
||||
- uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
@@ -41,16 +113,16 @@ jobs:
|
||||
run: dotnet build AutopilotMonitor.sln --configuration Release --no-restore
|
||||
|
||||
- name: Backend tests
|
||||
run: dotnet test src/Backend/AutopilotMonitor.Functions.Tests --configuration Release --no-build --verbosity quiet
|
||||
run: dotnet test src/Backend/AutopilotMonitor.Functions.Tests --configuration Release --no-build --verbosity quiet --logger "console;verbosity=minimal"
|
||||
|
||||
- name: Agent V2 Core tests
|
||||
run: dotnet test src/Agent/AutopilotMonitor.Agent.V2.Core.Tests --configuration Release --no-build --verbosity quiet
|
||||
run: dotnet test src/Agent/AutopilotMonitor.Agent.V2.Core.Tests --configuration Release --no-build --verbosity quiet --logger "console;verbosity=minimal"
|
||||
|
||||
- name: Decision Core tests
|
||||
run: dotnet test src/Agent/AutopilotMonitor.DecisionCore.Tests --configuration Release --no-build --verbosity quiet
|
||||
run: dotnet test src/Agent/AutopilotMonitor.DecisionCore.Tests --configuration Release --no-build --verbosity quiet --logger "console;verbosity=minimal"
|
||||
|
||||
- name: Summary Dialog tests
|
||||
run: dotnet test src/Agent/AutopilotMonitor.SummaryDialog.Tests --configuration Release --no-build --verbosity quiet
|
||||
run: dotnet test src/Agent/AutopilotMonitor.SummaryDialog.Tests --configuration Release --no-build --verbosity quiet --logger "console;verbosity=minimal"
|
||||
|
||||
web:
|
||||
name: Web (typecheck + tests)
|
||||
@@ -59,9 +131,11 @@ jobs:
|
||||
run:
|
||||
working-directory: src/Web/autopilot-monitor-web
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -93,9 +167,11 @@ jobs:
|
||||
run:
|
||||
working-directory: src/McpServer/autopilot-monitor-mcp
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -113,7 +189,7 @@ jobs:
|
||||
# (run 31252869254). Persist the model across runs instead; HF_CACHE_DIR
|
||||
# is honored by vector-search-provider.ts.
|
||||
- name: Cache embedding model
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: ${{ github.workspace }}/.hf-cache
|
||||
key: hf-model-Xenova-all-MiniLM-L6-v2
|
||||
@@ -127,9 +203,11 @@ jobs:
|
||||
name: Generated guardrails match rules/guardrails.json
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -148,9 +226,11 @@ jobs:
|
||||
name: Generated wire types match shared-manifests.json
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
|
||||
@@ -1,22 +1,77 @@
|
||||
name: CodeQL
|
||||
|
||||
# Documentation-only changes cannot affect what CodeQL analyzes (C# and
|
||||
# TypeScript), so they are filtered out rather than spending ~10 minutes to
|
||||
# re-report an unchanged result. A commit that touches docs AND code still runs:
|
||||
# GitHub skips only when EVERY changed path matches the filter.
|
||||
# Documentation-only changes and private-submodule gitlink bumps (internal/, infra/,
|
||||
# .claude/commands) cannot affect what CodeQL analyzes (C# and TypeScript), so they
|
||||
# are filtered out rather than spending ~10 minutes to re-report an unchanged result.
|
||||
# A gitlink bump reports the bare path ('internal', no trailing slash), which
|
||||
# 'internal/**' does NOT match — both forms are listed. A commit that touches docs
|
||||
# AND code still runs: GitHub skips only when EVERY changed path matches the filter.
|
||||
# Same list as ci.yml except that each file leaves itself out; keep them in step.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'internal'
|
||||
- 'internal/**'
|
||||
- 'infra'
|
||||
- 'infra/**'
|
||||
- '.claude/**'
|
||||
- '.gitmodules'
|
||||
- 'zizmor.yml'
|
||||
- '**/*.md'
|
||||
- 'LICENSE'
|
||||
- '.github/ISSUE_TEMPLATE/**'
|
||||
- '.github/FUNDING.yml'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/assets/**'
|
||||
- '.github/scripts/**'
|
||||
# Every other workflow: none of them changes the C# or TypeScript CodeQL analyzes.
|
||||
# paths-ignore cannot negate, so they are listed one by one; codeql.yml itself and
|
||||
# .github/codeql/ stay out of the list. A new workflow file is opt-in — add it here too.
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
- '.github/workflows/build-agent.yml'
|
||||
- '.github/workflows/build-msi.yml'
|
||||
- '.github/workflows/ci.yml'
|
||||
- '.github/workflows/combine-rules.yml'
|
||||
- '.github/workflows/deploy-backend.yml'
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'internal'
|
||||
- 'internal/**'
|
||||
- 'infra'
|
||||
- 'infra/**'
|
||||
- '.claude/**'
|
||||
- '.gitmodules'
|
||||
- 'zizmor.yml'
|
||||
- '**/*.md'
|
||||
- 'LICENSE'
|
||||
- '.github/ISSUE_TEMPLATE/**'
|
||||
- '.github/FUNDING.yml'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/assets/**'
|
||||
- '.github/scripts/**'
|
||||
# Every other workflow: none of them changes the C# or TypeScript CodeQL analyzes.
|
||||
# paths-ignore cannot negate, so they are listed one by one; codeql.yml itself and
|
||||
# .github/codeql/ stay out of the list. A new workflow file is opt-in — add it here too.
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
- '.github/workflows/build-agent.yml'
|
||||
- '.github/workflows/build-msi.yml'
|
||||
- '.github/workflows/ci.yml'
|
||||
- '.github/workflows/combine-rules.yml'
|
||||
- '.github/workflows/deploy-backend.yml'
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
schedule:
|
||||
# Weekly, so advisories published after a merge still surface. Deliberately
|
||||
# unfiltered — it re-analyzes the whole tree regardless of what changed, which
|
||||
@@ -49,15 +104,17 @@ jobs:
|
||||
build-mode: none
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-dotnet@v6
|
||||
- uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
|
||||
if: matrix.language == 'csharp'
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.37.9
|
||||
uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
@@ -68,6 +125,6 @@ jobs:
|
||||
run: dotnet build AutopilotMonitor.sln --configuration Release
|
||||
|
||||
- name: Analyze
|
||||
uses: github/codeql-action/analyze@v4.37.9
|
||||
uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
|
||||
with:
|
||||
category: /language:${{ matrix.language }}
|
||||
|
||||
@@ -24,14 +24,22 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- name: Install ajv-cli
|
||||
run: npm install -g ajv-cli ajv-formats
|
||||
# Lockfile and no lifecycle scripts: this job holds contents:write and pushes to
|
||||
# main, so an install-time script from a hijacked package would run with a token that
|
||||
# can rewrite the repository. rules/scripts/package-lock.json pins ajv-cli, ajv-formats
|
||||
# and every transitive package (Dependabot keeps it current); ajv is pure JS and
|
||||
# needs no install hooks.
|
||||
- name: Install ajv-cli from the lockfile
|
||||
working-directory: rules/scripts
|
||||
run: |
|
||||
npm ci --ignore-scripts
|
||||
echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Validate Gather Rules
|
||||
run: |
|
||||
|
||||
@@ -19,17 +19,19 @@ jobs:
|
||||
name: Build and Deploy Backend
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Setup .NET 10
|
||||
uses: actions/setup-dotnet@v6
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
@@ -41,7 +43,7 @@ jobs:
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/backend.txt" -SasToken '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/backend.txt"
|
||||
|
||||
$csproj = Get-Content ${{ env.CSPROJ_PATH }} -Raw
|
||||
if ($csproj -notmatch '<VersionPrefix>(\d+\.\d+)</VersionPrefix>') {
|
||||
@@ -115,9 +117,9 @@ jobs:
|
||||
runId = '${{ github.run_id }}'
|
||||
} | ConvertTo-Json
|
||||
|
||||
$sas = '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'.TrimStart('?')
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/backend.json?$sas"
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/backend.json"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' } -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' }) -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/backend.json"
|
||||
Write-Host $manifest
|
||||
|
||||
@@ -26,7 +26,9 @@ jobs:
|
||||
name: Build and Deploy MCP Server
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# The published customer documentation lives in its own PUBLIC repo and is
|
||||
# baked into the image as the search_docs corpus (see the Dockerfile). No
|
||||
@@ -36,8 +38,9 @@ jobs:
|
||||
# re-run actually rolls a new revision). /health reports the SHA captured
|
||||
# below so that staleness is checkable.
|
||||
- name: Check out documentation bundle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
repository: autopilotmonitor/autopilotmonitor-docs
|
||||
path: mcp-docs
|
||||
|
||||
@@ -46,7 +49,7 @@ jobs:
|
||||
run: echo "sha=$(git -C mcp-docs rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -59,7 +62,7 @@ jobs:
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/mcp.txt" -SasToken '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/mcp.txt"
|
||||
|
||||
$pkgVersion = (Get-Content ${{ env.PACKAGE_JSON }} -Raw | ConvertFrom-Json).version
|
||||
$prefix = ($pkgVersion -split '\.')[0..1] -join '.'
|
||||
@@ -77,7 +80,7 @@ jobs:
|
||||
"short_sha=$shortSha" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- name: Login to ACR
|
||||
run: az acr login --name ${{ env.ACR_NAME }}
|
||||
run: az acr login --name "$ACR_NAME"
|
||||
|
||||
- name: Build and push container image
|
||||
run: |
|
||||
@@ -87,17 +90,17 @@ jobs:
|
||||
--build-arg BUILD_NUMBER=${{ steps.version.outputs.build_number }} \
|
||||
--build-arg BUILD_COMMIT=${{ steps.version.outputs.short_sha }} \
|
||||
--build-arg BUILD_UTC=${{ steps.version.outputs.build_utc }} \
|
||||
-t ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }}:${{ env.IMAGE_TAG }} \
|
||||
-t ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }}:latest \
|
||||
-t "$ACR_NAME.azurecr.io/$CONTAINER_APP_NAME:$IMAGE_TAG" \
|
||||
-t "$ACR_NAME.azurecr.io/$CONTAINER_APP_NAME:latest" \
|
||||
.
|
||||
docker push ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }} --all-tags
|
||||
docker push "$ACR_NAME.azurecr.io/$CONTAINER_APP_NAME" --all-tags
|
||||
|
||||
- name: Deploy to Container App
|
||||
run: |
|
||||
az containerapp update \
|
||||
--name ${{ env.CONTAINER_APP_NAME }} \
|
||||
--resource-group ${{ env.RESOURCE_GROUP }} \
|
||||
--image ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }}:${{ env.IMAGE_TAG }}
|
||||
--name "$CONTAINER_APP_NAME" \
|
||||
--resource-group "$RESOURCE_GROUP" \
|
||||
--image "$ACR_NAME.azurecr.io/$CONTAINER_APP_NAME:$IMAGE_TAG"
|
||||
|
||||
# `az containerapp update` returns once the revision is provisioned, not once
|
||||
# it serves traffic — and with minReplicas=0 the first request also pays a cold
|
||||
@@ -144,9 +147,9 @@ jobs:
|
||||
docsCommit = '${{ steps.docs.outputs.sha }}'.Substring(0, 7)
|
||||
} | ConvertTo-Json
|
||||
|
||||
$sas = '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'.TrimStart('?')
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/mcp.json?$sas"
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/mcp.json"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' } -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' }) -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/mcp.json"
|
||||
Write-Host $manifest
|
||||
|
||||
@@ -35,12 +35,40 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy Job
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
lfs: false
|
||||
fetch-depth: 0 # Full history needed for git-based sitemap lastmod dates
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
# Only the version manifest at the very end needs Azure; logging in here rather than
|
||||
# next to that step keeps a credential problem from surfacing after a live deploy.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# The customer docs repo is the single source of the "What's new" panel: the prebuild
|
||||
# step (scripts/whats-new/build-whats-new.js) turns changelog/*.md into
|
||||
# public/whats-new.json and dates every bullet via git blame — hence the full history.
|
||||
# Public repo, no token. Editing the docs does NOT update the live panel until this
|
||||
# workflow runs again (push under src/Web/** or a manual dispatch); versions/web.json
|
||||
# records which docs commit is live.
|
||||
- name: Check out documentation bundle
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
repository: autopilotmonitor/autopilotmonitor-docs
|
||||
path: whats-new-docs
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve docs commit
|
||||
id: docs
|
||||
run: echo "sha=$(git -C whats-new-docs rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -51,7 +79,7 @@ jobs:
|
||||
run: npm ci
|
||||
|
||||
# next.config output:'export' -- `next build` writes the static site to out/
|
||||
# (prebuild: sync-error-codes, generate-lastmod, generate-version).
|
||||
# (prebuild: sync-error-codes, generate-lastmod, generate-version, whats-new).
|
||||
- name: Build
|
||||
working-directory: ${{ env.WEB_DIR }}
|
||||
env:
|
||||
@@ -68,9 +96,12 @@ jobs:
|
||||
# Build stamp for public/version.json (scripts/generate-version.js in prebuild);
|
||||
# the verify step below polls the deployed file for exactly this commit.
|
||||
GITHUB_SHA: ${{ github.sha }}
|
||||
# Docs checkout for public/whats-new.json; the prebuild fails in CI without it.
|
||||
WHATS_NEW_DOCS_DIR: ${{ github.workspace }}/whats-new-docs
|
||||
run: |
|
||||
npm run build
|
||||
test -f out/version.json || { echo "out/version.json missing - export did not run"; exit 1; }
|
||||
test -f out/whats-new.json || { echo "out/whats-new.json missing - whats-new prebuild did not run"; exit 1; }
|
||||
|
||||
# One archive of the exact bytes that get uploaded, so the attestation has a single subject.
|
||||
- name: Package export
|
||||
@@ -85,12 +116,12 @@ jobs:
|
||||
# commit, workflow file) in a public transparency log.
|
||||
# Verify: gh attestation verify portal-<sha>.tar.gz --repo okieselbach/AutopilotMonitor
|
||||
- name: Attest build provenance (portal export)
|
||||
uses: actions/attest-build-provenance@v4
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
with:
|
||||
subject-path: ${{ steps.package.outputs.archive }}
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ steps.package.outputs.archive }}
|
||||
@@ -136,7 +167,8 @@ jobs:
|
||||
throw "Portal did not serve commit $expectedCommit within 5 minutes -- deployment not verified, version manifest NOT published"
|
||||
|
||||
# Cross-component deploy record, written last and only on a verified deploy --
|
||||
# same contract as versions/backend.json and versions/mcp.json.
|
||||
# same contract as versions/backend.json and versions/mcp.json. docsCommit says which
|
||||
# docs revision the live What's new panel was generated from (as for the MCP corpus).
|
||||
- name: Publish version manifest
|
||||
shell: pwsh
|
||||
run: |
|
||||
@@ -145,11 +177,12 @@ jobs:
|
||||
commit = '${{ github.sha }}'.Substring(0, 7)
|
||||
deployedUtc = [DateTime]::UtcNow.ToString('o')
|
||||
runId = '${{ github.run_id }}'
|
||||
docsCommit = '${{ steps.docs.outputs.sha }}'.Substring(0, 7)
|
||||
} | ConvertTo-Json
|
||||
|
||||
$sas = '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'.TrimStart('?')
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/web.json?$sas"
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/web.json"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' } -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' }) -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/web.json"
|
||||
Write-Host $manifest
|
||||
|
||||
@@ -3,6 +3,9 @@ name: IndexNow Notify (Bing/Yandex)
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
# The job only talks to public endpoints; it never needs a repository token.
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
name: Federated login + read check
|
||||
steps:
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
|
||||
@@ -25,5 +25,6 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# $/ is the self-repository form: this repository at the running commit.
|
||||
gates:
|
||||
uses: ./.github/workflows/bootstrap-script-gates.yml
|
||||
uses: $/.github/workflows/bootstrap-script-gates.yml
|
||||
|
||||
@@ -34,23 +34,109 @@ concurrency:
|
||||
group: agent-blob-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
# Authenticode for the customer scripts. Same identity and same repository variables as
|
||||
# build-agent.yml -- the loader on every device pins this publisher, so the two must not
|
||||
# drift apart.
|
||||
env:
|
||||
CODE_SIGN_CLIENT_ID: ${{ vars.CODE_SIGN_CLIENT_ID }}
|
||||
CODE_SIGN_CERTIFICATE_NAME: ${{ vars.CODE_SIGN_CERTIFICATE_NAME }}
|
||||
CODE_SIGN_KEY_VAULT_URL: ${{ vars.CODE_SIGN_KEY_VAULT_URL }}
|
||||
CODE_SIGN_EXPECTED_SUBJECT: ${{ vars.CODE_SIGN_EXPECTED_SUBJECT }}
|
||||
CODE_SIGN_TIMESTAMP_URL: http://timestamp.acs.microsoft.com
|
||||
|
||||
jobs:
|
||||
# $/ is the self-repository form: this repository at the running commit, never a
|
||||
# path some earlier step could have written to the workspace.
|
||||
gates:
|
||||
uses: ./.github/workflows/bootstrap-script-gates.yml
|
||||
uses: $/.github/workflows/bootstrap-script-gates.yml
|
||||
|
||||
publish:
|
||||
name: Publish to the download alias
|
||||
needs: gates
|
||||
runs-on: windows-latest
|
||||
# Job-scoped: only the publish job federates to Azure, the gates job never holds an
|
||||
# OIDC token it does not need.
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # OIDC federated login for the storage writes (secret-less)
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check code-signing configuration
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($name in 'CODE_SIGN_CLIENT_ID', 'CODE_SIGN_CERTIFICATE_NAME', 'CODE_SIGN_KEY_VAULT_URL') {
|
||||
if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($name))) {
|
||||
throw "Repository variable $name is not set. Published customer scripts must be signed."
|
||||
}
|
||||
}
|
||||
|
||||
# First, and only to fetch the Key Vault token: azure/login sets the default az
|
||||
# account and the storage login below has to be the one that stays. See D-223 --
|
||||
# both federated credentials are bound to the ref, so this job must never carry a
|
||||
# GitHub `environment:`.
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ env.CODE_SIGN_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
|
||||
- name: Acquire Key Vault token for signing
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$token = az account get-access-token --scope https://vault.azure.net/.default --query accessToken --output tsv
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw "Could not acquire a Key Vault token for the code-signing identity."
|
||||
}
|
||||
Write-Host "::add-mask::$token"
|
||||
"CODE_SIGN_AKV_ACCESS_TOKEN=$token" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Authorizes the eu blob writes and the AdminConfiguration merge. The gates job above
|
||||
# runs the same publisher with -DryRun and deliberately without a login -- a dry run
|
||||
# must never need Azure credentials.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# Version pinned so a new or compromised NuGet release cannot flow into signing
|
||||
# unreviewed; roll-forward because the tool targets an older runtime than the SDK
|
||||
# preinstalled on the runner.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
- name: Publish bootstrap + customer scripts
|
||||
shell: pwsh
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: |
|
||||
$dryRun = '${{ inputs.dry_run }}' -eq 'true'
|
||||
./.github/scripts/Publish-BootstrapScripts.ps1 `
|
||||
-SasToken '${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}' `
|
||||
-LegacySasToken '${{ secrets.AZURE_BLOB_AGENT_LEGACY_SAS_TOKEN }}' `
|
||||
-TableSasToken '${{ secrets.AZURE_TABLE_ADMIN_SAS_TOKEN }}' `
|
||||
-SigningToken $env:CODE_SIGN_AKV_ACCESS_TOKEN `
|
||||
-KeyVaultUrl $env:CODE_SIGN_KEY_VAULT_URL `
|
||||
-CertificateName $env:CODE_SIGN_CERTIFICATE_NAME `
|
||||
-TimestampUrl $env:CODE_SIGN_TIMESTAMP_URL `
|
||||
-ExpectedPublisher $env:CODE_SIGN_EXPECTED_SUBJECT `
|
||||
-SignedOutputDir "$env:RUNNER_TEMP/signed-scripts" `
|
||||
-DryRun:$dryRun
|
||||
|
||||
# The exact bytes that were (or, on a dry run, would have been) uploaded. A dry-run
|
||||
# dispatch is therefore the way to inspect a signed script locally before it reaches
|
||||
# a single customer. Kept on failure too: a failed alias verification is precisely
|
||||
# when someone wants to look at the file.
|
||||
- name: Upload signed scripts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: signed-scripts
|
||||
path: ${{ runner.temp }}/signed-scripts
|
||||
if-no-files-found: ignore
|
||||
|
||||
@@ -15,7 +15,7 @@ bld/
|
||||
[Ll]ogs/
|
||||
|
||||
# Build counter files: the authoritative counter is the shared blob
|
||||
# (agent container, buildcounter-v{N}.txt, ETag-CAS — see docs/agent/build-counter-blob.md).
|
||||
# (agent container, buildcounter-v{N}.txt, ETag-CAS — see internal/docs/agent/build-counter-blob.md).
|
||||
# The local file is only an offline fallback for -SkipUpload/bare-dotnet smoke builds.
|
||||
buildcounter.txt
|
||||
|
||||
@@ -114,17 +114,10 @@ tmpclaude-*-cwd
|
||||
# Claude skills and scripts are now in a private submodule (.claude/commands/)
|
||||
# No individual file ignores needed — the submodule handles everything
|
||||
|
||||
# Deployment scripts (local only — contain tokens + SAS URLs)
|
||||
scripts/Deployment/
|
||||
|
||||
# Migration tooling (local only, vorerst) — cross-tenant storage/cert migration scripts
|
||||
scripts/Migration/
|
||||
|
||||
# Infra: tracked. The Bicep templates carry no credentials — every secret is a
|
||||
# @secure() parameter with an empty default, injected at deploy time, and image
|
||||
# pull runs on managed identity. Keep it that way: never commit a resolved secret
|
||||
# value or a *.parameters.json holding one.
|
||||
infra/*.parameters.json
|
||||
|
||||
# M2 — local raw session fixtures before anonymization (plan §4 M2)
|
||||
tests/fixtures/enrollment-sessions-local/
|
||||
@@ -135,22 +128,15 @@ tests/fixtures/enrollment-sessions-local/
|
||||
TODO.txt
|
||||
tools/orphan-analysis.py
|
||||
tools/export-session-fixture.mjs
|
||||
tasks/
|
||||
|
||||
# Azure Table cleanup backups (dry-run exports + delete manifests)
|
||||
# Produced by /cleanup-table skill — never commit deletion payloads
|
||||
# Produced by operator tooling — never commit deletion payloads
|
||||
db-backups/
|
||||
plans/REFACTOR_AGENT_V2.md
|
||||
.claude/scheduled_tasks.lock
|
||||
.claude/plans/health-script-remediation-fix.md
|
||||
|
||||
# Operator-local Azure setup helper — applies lifecycle policy + container
|
||||
# creation against a live storage account. Not portable across environments
|
||||
# (storage account name, resource group are operator-specific) so it stays
|
||||
# out of the repo. Each operator keeps their own copy.
|
||||
scripts/Backup/setup-backup-container.ps1
|
||||
docs/Introduction-AutopilotMonitor.pptx
|
||||
docs/~$Introduction-AutopilotMonitor.pptx
|
||||
internal/docs/Introduction-AutopilotMonitor.pptx
|
||||
internal/docs/~$Introduction-AutopilotMonitor.pptx
|
||||
|
||||
# Playwright MCP browser artifacts (snapshots, screenshots, console logs)
|
||||
.playwright-mcp/
|
||||
@@ -159,8 +145,11 @@ docs/~$Introduction-AutopilotMonitor.pptx
|
||||
# changes each run) and served at /version.json from the static export.
|
||||
src/Web/autopilot-monitor-web/public/version.json
|
||||
|
||||
# What's new payload: generated by scripts/whats-new/build-whats-new.js from the docs
|
||||
# checkout on every build and served at /whats-new.json from the static export.
|
||||
src/Web/autopilot-monitor-web/public/whats-new.json
|
||||
|
||||
# Local IME binaries scratch used by the /ime-decompile skill (MSI + msiexec-
|
||||
# extracted payload per version, for grepping in binaries). Decompilates live in
|
||||
# the private ime-decompiles repo, MSIs in the ime-archive blob - this stays local.
|
||||
|
||||
# Local IME installer scratch (MSI + extracted payload per version, for grepping
|
||||
# in binaries). Operator-only, never committed.
|
||||
ime-files/
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
[submodule ".claude/commands"]
|
||||
path = .claude/commands
|
||||
url = https://github.com/okieselbach/Autopilot-Monitor-dotclaude.git
|
||||
[submodule "docs"]
|
||||
path = internal
|
||||
url = https://github.com/okieselbach/Autopilot-Monitor-internal.git
|
||||
[submodule "infra"]
|
||||
path = infra
|
||||
url = https://github.com/okieselbach/Autopilot-Monitor-infra.git
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# Code of Conduct
|
||||
|
||||
This applies to all project spaces: issues, pull requests, discussions, and the documentation repository.
|
||||
|
||||
## Expectations
|
||||
|
||||
- Be respectful and constructive. Disagree with ideas, not with people.
|
||||
- Assume good intent. Most confusion is a documentation gap, not malice.
|
||||
- Keep discussions on topic and searchable: clear titles, reproducible steps, text instead of screenshots of text where possible.
|
||||
- Never post customer data, tenant IDs, device identifiers, tokens, or log excerpts containing personal data in public issues or discussions. Redact first.
|
||||
|
||||
## Not acceptable
|
||||
|
||||
Harassment, personal attacks, discriminatory language, publishing others' private information, and sustained disruption of discussions.
|
||||
|
||||
## Enforcement
|
||||
|
||||
The maintainers may edit, hide, or delete content and may block accounts that violate this code. To raise a concern, contact the maintainers privately through
|
||||
[GitHub](https://github.com/okieselbach) rather than in a public thread.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Feedback and Issues
|
||||
|
||||
Autopilot Monitor is developed and operated by its maintainers. The source is published so that customers and security reviewers can see what runs on their devices and with their data. This page explains where feedback goes.
|
||||
|
||||
| I want to… | Go to |
|
||||
|---|---|
|
||||
| Report a bug | [New issue → Bug report](https://github.com/okieselbach/AutopilotMonitor/issues/new/choose) |
|
||||
| Suggest a feature | [New issue → Feature request](https://github.com/okieselbach/AutopilotMonitor/issues/new/choose) |
|
||||
| Ask a question or discuss an idea | [Discussions](https://github.com/okieselbach/AutopilotMonitor/discussions) |
|
||||
| Report a security issue | [SECURITY.md](SECURITY.md) — privately, never as a public issue |
|
||||
| Read how the product works | [docs.autopilotmonitor.com](https://docs.autopilotmonitor.com) |
|
||||
|
||||
## Reporting a problem in your own tenant
|
||||
|
||||
For a failed or odd enrollment, the **Report Session** button in the portal is the fastest path. It hands the session context to support without exposing any of it publicly. Use a GitHub issue for problems that are reproducible and not tied to your data.
|
||||
|
||||
Please redact tenant IDs, device names, serial numbers, and anything personal before posting in a public issue or discussion.
|
||||
@@ -1,39 +1,673 @@
|
||||
Autopilot Monitor - Split License
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (c) 2026 Autopilot Monitor Contributors
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
|
||||
This project uses a split licensing model. Different components are
|
||||
licensed under different terms:
|
||||
Everyone is permitted to copy and distribute verbatim copies of this
|
||||
license document, but changing it is not allowed.
|
||||
|
||||
## MIT License
|
||||
Preamble
|
||||
|
||||
The following components are licensed under the MIT License:
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
- src/Agent/ - The Autopilot Monitor Agent (deployed to devices)
|
||||
- src/Shared/ - Shared DTOs, enums, and constants
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains
|
||||
free software for all its users.
|
||||
|
||||
See src/Agent/LICENSE and src/Shared/LICENSE for the full MIT license text.
|
||||
When we speak of free software, we are referring to freedom, not price.
|
||||
Our General Public Licenses are designed to make sure that you have the
|
||||
freedom to distribute copies of free software (and charge for them if
|
||||
you wish), that you receive source code or can get it if you want it,
|
||||
that you can change the software or use pieces of it in new free
|
||||
programs, and that you know you can do these things.
|
||||
|
||||
## GNU Affero General Public License v3.0 (AGPL-3.0)
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
The following components are licensed under the AGPL-3.0:
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
- src/Backend/ - Azure Functions backend
|
||||
- src/Web/ - Next.js web frontend
|
||||
- src/McpServer/ - MCP Server
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
See src/Backend/LICENSE for the full AGPL-3.0 license text.
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing
|
||||
under this license.
|
||||
|
||||
## Why Split Licensing?
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
The Agent runs on end-user devices and is licensed under MIT to allow
|
||||
unrestricted use, modification, and distribution without copyleft
|
||||
obligations.
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
The server-side components (Backend, Web, McpServer) are licensed under
|
||||
AGPL-3.0 to ensure that modifications to the server infrastructure
|
||||
remain open source, especially when deployed as a network service.
|
||||
0. Definitions.
|
||||
|
||||
The Shared library is MIT-licensed because it is a dependency of the
|
||||
MIT-licensed Agent. Using AGPL for Shared would effectively make the
|
||||
Agent AGPL as well.
|
||||
"This License" refers to version 3 of the GNU Affero General Public
|
||||
License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds
|
||||
of works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of
|
||||
an exact copy. The resulting work is called a "modified version" of
|
||||
the earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user
|
||||
through a computer network, with no transfer of a copy, is not
|
||||
conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices" to
|
||||
the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work for
|
||||
making modifications to it. "Object code" means any non-source form
|
||||
of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that is
|
||||
widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users can
|
||||
regenerate automatically from other parts of the Corresponding Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that same
|
||||
work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not convey,
|
||||
without conditions so long as your license otherwise remains in force.
|
||||
You may convey covered works to others for the sole purpose of having
|
||||
them make modifications exclusively for you, or provide you with
|
||||
facilities for running those works, provided that you comply with the
|
||||
terms of this License in conveying all material for which you do not
|
||||
control copyright. Those thus making or running the covered works for
|
||||
you must do so exclusively on your behalf, under your direction and
|
||||
control, on terms that prohibit them from making any copies of your
|
||||
copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under the
|
||||
conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such
|
||||
circumvention is effected by exercising rights under this License with
|
||||
respect to the covered work, and you disclaim any intention to limit
|
||||
operation or modification of the work as a means of enforcing, against
|
||||
the work's users, your or third parties' legal rights to forbid
|
||||
circumvention of technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the
|
||||
code; keep intact all notices of the absence of any warranty; and give
|
||||
all recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these
|
||||
conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under
|
||||
section 7. This requirement modifies the requirement in section
|
||||
4 to "keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not used
|
||||
to limit the access or legal rights of the compilation's users beyond
|
||||
what the individual works permit. Inclusion of a covered work in an
|
||||
aggregate does not cause this License to apply to the other parts of
|
||||
the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms of
|
||||
sections 4 and 5, provided that you also convey the machine-readable
|
||||
Corresponding Source under the terms of this License, in one of these
|
||||
ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the Corresponding
|
||||
Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in
|
||||
accord with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to
|
||||
the Corresponding Source in the same way through the same place
|
||||
at no further charge. You need not require recipients to copy
|
||||
the Corresponding Source along with the object code. If the
|
||||
place to copy the object code is a network server, the
|
||||
Corresponding Source may be on a different server (operated by
|
||||
you or a third party) that supports equivalent copying
|
||||
facilities, provided you maintain clear directions next to the
|
||||
object code saying where to find the Corresponding Source.
|
||||
Regardless of what server hosts the Corresponding Source, you
|
||||
remain obligated to ensure that it is available for as long as
|
||||
needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission,
|
||||
provided you inform other peers where the object code and
|
||||
Corresponding Source of the work are being offered to the general
|
||||
public at no charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for
|
||||
incorporation into a dwelling. In determining whether a product is a
|
||||
consumer product, doubtful cases shall be resolved in favor of
|
||||
coverage. For a particular product received by a particular user,
|
||||
"normally used" refers to a typical or common use of that class of
|
||||
product, regardless of the status of the particular user or of the way
|
||||
in which the particular user actually uses, or expects or is expected
|
||||
to use, the product. A product is a consumer product regardless of
|
||||
whether the product has substantial commercial, industrial or
|
||||
non-consumer uses, unless such uses represent the only significant
|
||||
mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to
|
||||
install and execute modified versions of a covered work in that User
|
||||
Product from a modified version of its Corresponding Source. The
|
||||
information must suffice to ensure that the continued functioning of
|
||||
the modified object code is in no case prevented or interfered with
|
||||
solely because modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or
|
||||
updates for a work that has been modified or installed by the
|
||||
recipient, or for the User Product in which it has been modified or
|
||||
installed. Access to a network may be denied when the modification
|
||||
itself materially and adversely affects the operation of the network
|
||||
or violates the rules and protocols for communication across the
|
||||
network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright
|
||||
holders of that material) supplement the terms of this License with
|
||||
terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material,
|
||||
or requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors
|
||||
or authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions
|
||||
of it) with contractual assumptions of liability to the recipient,
|
||||
for any liability that these contractual assumptions directly
|
||||
impose on those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions; the
|
||||
above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your license
|
||||
from a particular copyright holder is reinstated (a) provisionally,
|
||||
unless and until the copyright holder explicitly and finally terminates
|
||||
your license, and (b) permanently, if the copyright holder fails to
|
||||
notify you of the violation by some reasonable means prior to 60 days
|
||||
after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or run
|
||||
a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims owned
|
||||
or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the
|
||||
patent license to downstream recipients. "Knowingly relying" means
|
||||
you have actual knowledge that, but for the patent license, your
|
||||
conveying the covered work in a country, or your recipient's use of
|
||||
the covered work in a country, would infringe one or more identifiable
|
||||
patents in that country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within the
|
||||
scope of its coverage, prohibits the exercise of, or is conditioned on
|
||||
the non-exercise of one or more of the rights that are specifically
|
||||
granted under this License. You may not convey a covered work if you
|
||||
are a party to an arrangement with a third party that is in the
|
||||
business of distributing software, under which you make payment to the
|
||||
third party based on the extent of your activity of conveying the
|
||||
work, and under which the third party grants, to any of the parties
|
||||
who would receive the covered work from you, a discriminatory patent
|
||||
license (a) in connection with copies of the covered work conveyed by
|
||||
you (or copies made from those copies), or (b) primarily for and in
|
||||
connection with specific products or compilations that contain the
|
||||
covered work, unless you entered into that arrangement, or that patent
|
||||
license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey
|
||||
a covered work so as to satisfy simultaneously your obligations under
|
||||
this License and any other pertinent obligations, then as a
|
||||
consequence you may not convey it at all. For example, if you agree
|
||||
to terms that obligate you to collect a royalty for further conveying
|
||||
from those to whom you convey the Program, the only way you could
|
||||
satisfy both those terms and this License would be to refrain entirely
|
||||
from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public
|
||||
License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your
|
||||
version supports such interaction) an opportunity to receive the
|
||||
Corresponding Source of your version by providing access to the
|
||||
Corresponding Source from a network server at no charge, through some
|
||||
standard or customary means of facilitating copying of software. This
|
||||
Corresponding Source shall include the Corresponding Source for any
|
||||
work covered by version 3 of the GNU General Public License that is
|
||||
incorporated pursuant to the following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions
|
||||
of the GNU Affero General Public License from time to time. Such new
|
||||
versions will be similar in spirit to the present version, but may
|
||||
differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever
|
||||
published by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future versions
|
||||
of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
|
||||
WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
|
||||
PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
|
||||
DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
|
||||
CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
|
||||
CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
|
||||
ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
|
||||
NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
|
||||
LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
|
||||
TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
|
||||
PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these
|
||||
terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public
|
||||
License along with this program. If not, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper
|
||||
mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application,
|
||||
its interface could display a "Source" link that leads users to an
|
||||
archive of the code. There are many ways you could offer source, and
|
||||
different solutions will be better for different programs; see section
|
||||
13 for the specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. For more information on this, and how to apply and follow
|
||||
the GNU AGPL, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
@@ -1,46 +1,93 @@
|
||||
# <img src="docs/images/logo.svg" height="28" alt="" /> AutopilotMonitor
|
||||
# <img src=".github/assets/logo.svg" height="28" alt="Autopilot Monitor logo" /> Autopilot Monitor
|
||||
|
||||
[](https://www.autopilotmonitor.com)
|
||||
[](LICENSE)
|
||||
[](https://www.autopilotmonitor.com)
|
||||
|
||||
Advanced monitoring and troubleshooting solution for Windows Autopilot deployments. Gain full visibility into every enrollment session with a detailed event timeline, fleet health dashboards, and session reporting. Define custom analysis rules to automatically detect issues and gather rules to collect targeted evidence. Retrieve diagnostics packages on demand, configure agent settings like auto-reboot behavior and automatic timezone adjustment — all managed centrally from the web dashboard.
|
||||
Autopilot Monitor is an open-source, real-time monitoring and troubleshooting platform for **Windows Autopilot** and **Microsoft Intune** enrollment. It records enrollment events, application installations, policies, device state, and diagnostics while provisioning is still running, instead of collecting logs by hand after a failure. Administrators get live enrollment tracking, Enrollment Status Page (ESP) visibility, detailed session timelines, automated issue detection, on-demand diagnostics, fleet health analytics, and AI-assisted troubleshooting through a hosted Model Context Protocol (MCP) server.
|
||||
|
||||
The platform supports classic Windows Autopilot and modern Intune provisioning scenarios, including **Windows Autopilot Device Preparation**.
|
||||
|
||||
## Availability
|
||||
|
||||
Autopilot Monitor is **publicly available** — the Community plan is free. Sign in with your work account at **[autopilotmonitor.com](https://www.autopilotmonitor.com)**; new organizations complete a short activation step after first sign-in.
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/images/SessionList.png" width="45%" />
|
||||
<img src="docs/images/FleetHealth.png" width="45%" />
|
||||
<img src=".github/assets/SessionList.png" width="45%" alt="Autopilot Monitor dashboard showing live Windows Autopilot and Intune enrollment sessions" />
|
||||
<img src=".github/assets/FleetHealth.png" width="45%" alt="Autopilot Monitor fleet health dashboard with Windows Autopilot enrollment analytics and success metrics" />
|
||||
</p>
|
||||
<p align="center">
|
||||
<img src="docs/images/SessionDetails.png" width="45%" />
|
||||
<img src="docs/images/SessionTimeline.png" width="45%" />
|
||||
<img src=".github/assets/SessionDetails.png" width="45%" alt="Autopilot Monitor session details showing device information, diagnostics, apps, policies, and enrollment status" />
|
||||
<img src=".github/assets/SessionTimeline.png" width="45%" alt="Windows Autopilot enrollment timeline showing ESP phases, application installs, policies, events, and errors" />
|
||||
</p>
|
||||
|
||||
## Overview
|
||||
## Key Features
|
||||
|
||||
Autopilot Monitor provides real-time tracking, intelligent diagnostics, and automated troubleshooting for Windows Autopilot enrollment processes. It consists of:
|
||||
- **Real-time Windows Autopilot monitoring** — follow device provisioning while enrollment is still running
|
||||
- **Detailed enrollment timelines** — correlate Autopilot, ESP, Intune Management Extension, apps, policies, device state, and system events
|
||||
- **Enrollment Status Page visibility** — see which phase, application, or policy is delaying provisioning
|
||||
- **Automated issue detection** — define custom analysis rules that identify known failure patterns
|
||||
- **Targeted evidence collection** — gather registry values, event log entries, files, log-parser matches, and allow-listed commands when specific conditions are detected
|
||||
- **On-demand diagnostics** — request diagnostics packages from enrolled devices remotely
|
||||
- **Fleet health analytics** — success rates, deployment duration, device models, applications, errors, and recurring issues
|
||||
- **Session reporting** — inspect and share detailed enrollment information for individual devices
|
||||
- **Windows Autopilot Device Preparation support** — monitor Device Preparation deployments alongside classic Autopilot scenarios
|
||||
- **Centralized agent configuration** — control diagnostic behavior, automatic reboots, timezone adjustment, and other monitoring settings from the web dashboard
|
||||
- **AI-assisted troubleshooting** — query enrollment data through the built-in Model Context Protocol server
|
||||
|
||||
- **Bootstrap Script** — PowerShell script deployed via Intune that starts monitoring early in the enrollment process
|
||||
- **Monitoring Agent** — Lightweight .NET application that collects telemetry and evidence during enrollment
|
||||
## How it works
|
||||
|
||||
Autopilot Monitor consists of several components working together throughout Windows enrollment:
|
||||
|
||||
- **Bootstrap Script** — PowerShell script deployed through Microsoft Intune that starts monitoring early in the provisioning process
|
||||
- **Monitoring Agent** — lightweight .NET application that collects telemetry and evidence during enrollment
|
||||
- **Backend API** — Azure Functions-based ingestion and processing pipeline
|
||||
- **Web Dashboard** — Next.js application for real-time monitoring and fleet analytics
|
||||
- **Web Dashboard** — Next.js application for real-time monitoring, troubleshooting, and fleet analytics
|
||||
- **MCP Server** — Model Context Protocol server that lets AI assistants query Autopilot Monitor data in natural language
|
||||
|
||||
## Architecture
|
||||
## Windows Autopilot and Intune Troubleshooting
|
||||
|
||||
For detailed information about the system architecture, components, and data flow, see [Architecture Documentation](docs/architecture.md).
|
||||
Autopilot Monitor helps investigate common Windows provisioning and enrollment problems such as:
|
||||
|
||||
- Windows Autopilot deployments that fail or appear stuck
|
||||
- Enrollment Status Page (ESP) delays and unusually long deployment phases
|
||||
- Win32 application installation failures and slow application downloads
|
||||
- Intune Management Extension problems
|
||||
- Microsoft Entra join, Hybrid Join, and device registration issues
|
||||
- policies that block or delay provisioning
|
||||
- inconsistent behavior across hardware models and recurring failures affecting multiple devices
|
||||
|
||||
Because telemetry is collected during enrollment, many problems can be investigated without reproducing the issue or manually collecting logs from the affected device.
|
||||
|
||||
## AI Integration (MCP)
|
||||
|
||||
Autopilot Monitor ships a hosted **[Model Context Protocol](https://modelcontextprotocol.io)** server. Connect Claude Desktop, VS Code, or any MCP client that speaks Streamable HTTP with OAuth, and ask questions like:
|
||||
|
||||
- *"Show me all failed enrollments from the last 24 hours and group them by failure reason."*
|
||||
- *"Why did session X fail, and which app install caused the delay?"*
|
||||
- *"Which devices had unusually long ESP application phases this week?"*
|
||||
- *"Which device models have the highest Autopilot failure rate?"*
|
||||
- *"Which devices in my fleet are affected by CVE-2024-30078?"*
|
||||
|
||||
Server URL: `https://mcp.autopilotmonitor.com/mcp` — sign-in runs through your existing work account, access is scoped to your tenant exactly like in the portal, and the MCP server stores no credentials. Setup guide: **[AI Integration (MCP)](https://docs.autopilotmonitor.com/integrations/ai-integration-mcp)**.
|
||||
|
||||
## Documentation
|
||||
|
||||
Full admin documentation is available at **[docs.autopilotmonitor.com](https://docs.autopilotmonitor.com)**
|
||||
Full admin documentation is available at **[docs.autopilotmonitor.com](https://docs.autopilotmonitor.com)** — deployment, configuration, monitoring, diagnostics, troubleshooting, analysis and gather rules, integrations, and AI-assisted troubleshooting.
|
||||
|
||||
## Open Source
|
||||
|
||||
Autopilot Monitor is developed openly on GitHub. The source is published so that customers and security reviewers can see what runs on their devices and with their data: the repository contains the device agent, shared library, backend, web dashboard, and MCP server.
|
||||
|
||||
## Feedback
|
||||
|
||||
Bug reports and feature requests go through [Issues](https://github.com/okieselbach/AutopilotMonitor/issues/new/choose), questions and ideas through [Discussions](https://github.com/okieselbach/AutopilotMonitor/discussions). Security issues are reported privately, see [SECURITY.md](SECURITY.md). For a problem in your own tenant, the **Report Session** button in the portal is the fastest path.
|
||||
|
||||
## License
|
||||
|
||||
This project uses a **split licensing model**:
|
||||
This project uses a **split licensing model**. The root [LICENSE](LICENSE) is AGPL-3.0, which covers the server-side components; the device-side components carry their own MIT license file:
|
||||
|
||||
- **MIT License** — Agent (`src/Agent/`) and Shared library (`src/Shared/`) — unrestricted use on end-user devices
|
||||
- **AGPL-3.0** — Backend (`src/Backend/`), Web Dashboard (`src/Web/`), and MCP Server (`src/McpServer/`) — server-side components remain open source
|
||||
- **MIT License** — Agent ([`src/Agent/`](src/Agent/LICENSE)) and Shared library ([`src/Shared/`](src/Shared/LICENSE)) — unrestricted use on end-user devices, no copyleft obligations
|
||||
- **AGPL-3.0** — Backend ([`src/Backend/`](src/Backend/LICENSE)), Web Dashboard ([`src/Web/`](src/Web/LICENSE)), and MCP Server ([`src/McpServer/`](src/McpServer/LICENSE)) — modifications to server-side components remain open source, especially when deployed as a network service
|
||||
|
||||
See [LICENSE](LICENSE) for full details.
|
||||
The Shared library is MIT because it is a dependency of the MIT-licensed Agent. The license file inside each component directory is authoritative for that component.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
Please report security issues **privately** through
|
||||
[GitHub Security Advisories](https://github.com/okieselbach/AutopilotMonitor/security/advisories/new).
|
||||
Private vulnerability reporting is enabled, so a report reaches the maintainers without ever being public.
|
||||
|
||||
Please do not open a public issue, post on social media, or send email for security findings.
|
||||
|
||||
## What to expect
|
||||
|
||||
- An acknowledgement and an initial severity assessment.
|
||||
- A fix or a mitigation plan, with a shared view of timing before anything is disclosed.
|
||||
- Credit in the advisory if you want it, and none if you prefer to stay anonymous.
|
||||
|
||||
Reports are read by the maintainers directly. There is no guaranteed response time and no bug bounty.
|
||||
|
||||
## Scope and rules of engagement
|
||||
|
||||
Security research against the hosted service is welcome and is not a violation of the
|
||||
[Terms of Use](https://www.autopilotmonitor.com/terms). While testing, please:
|
||||
|
||||
- do not access data of tenants you do not own,
|
||||
- do not degrade the service for others,
|
||||
- do not run automated scanners against production.
|
||||
|
||||
## Supported versions
|
||||
|
||||
The hosted service always runs the current `main`. For the on-device agent, only the
|
||||
[latest release](https://github.com/okieselbach/AutopilotMonitor/releases/latest) receives fixes.
|
||||
|
||||
## More
|
||||
|
||||
The [Security FAQ](https://docs.autopilotmonitor.com/trust-and-security/security-faq) and
|
||||
[Data Flows](https://docs.autopilotmonitor.com/trust-and-security/data-flows) pages describe the security model,
|
||||
data handling, and isolation in detail.
|
||||
@@ -1,166 +0,0 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Build script for Autopilot Monitor solution.
|
||||
|
||||
.DESCRIPTION
|
||||
Builds all .NET projects and optionally the web UI.
|
||||
Run this after cloning or when you want to rebuild everything.
|
||||
|
||||
.PARAMETER Configuration
|
||||
Build configuration: Debug or Release (default: Debug)
|
||||
|
||||
.PARAMETER IncludeWeb
|
||||
Also build the web UI (requires Node.js)
|
||||
|
||||
.PARAMETER Clean
|
||||
Clean before building
|
||||
|
||||
.EXAMPLE
|
||||
.\build.ps1
|
||||
Builds all .NET projects in Debug mode
|
||||
|
||||
.EXAMPLE
|
||||
.\build.ps1 -Configuration Release -IncludeWeb
|
||||
Builds everything in Release mode including web UI
|
||||
#>
|
||||
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Debug", "Release")]
|
||||
[string]$Configuration = "Debug",
|
||||
|
||||
[switch]$IncludeWeb,
|
||||
|
||||
[switch]$Clean
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
Write-Host "============================================" -ForegroundColor Cyan
|
||||
Write-Host " Autopilot Monitor Build Script" -ForegroundColor Cyan
|
||||
Write-Host "============================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
|
||||
$rootPath = $PSScriptRoot
|
||||
$slnPath = Join-Path $rootPath "AutopilotMonitor.sln"
|
||||
|
||||
# Check prerequisites
|
||||
Write-Host "[1/4] Checking prerequisites..." -ForegroundColor Yellow
|
||||
|
||||
# Check .NET SDK
|
||||
try {
|
||||
$dotnetVersion = & dotnet --version
|
||||
Write-Host " ✓ .NET SDK version: $dotnetVersion" -ForegroundColor Green
|
||||
}
|
||||
catch {
|
||||
Write-Host " ✗ .NET SDK not found. Please install .NET 10 SDK." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check Node.js if building web
|
||||
if ($IncludeWeb) {
|
||||
try {
|
||||
$nodeVersion = & node --version
|
||||
Write-Host " ✓ Node.js version: $nodeVersion" -ForegroundColor Green
|
||||
}
|
||||
catch {
|
||||
Write-Host " ✗ Node.js not found. Please install Node.js 18+." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
|
||||
# Clean if requested
|
||||
if ($Clean) {
|
||||
Write-Host "[2/4] Cleaning solution..." -ForegroundColor Yellow
|
||||
& dotnet clean $slnPath -c $Configuration
|
||||
Write-Host " ✓ Clean completed" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
}
|
||||
else {
|
||||
Write-Host "[2/4] Skipping clean (use -Clean to clean first)" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
# Restore packages
|
||||
Write-Host "[3/4] Restoring NuGet packages..." -ForegroundColor Yellow
|
||||
& dotnet restore $slnPath
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " ✗ Restore failed" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host " ✓ Restore completed" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Build solution
|
||||
Write-Host "[4/4] Building solution ($Configuration)..." -ForegroundColor Yellow
|
||||
& dotnet build $slnPath -c $Configuration --no-restore
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " ✗ Build failed" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host " ✓ Build completed" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Build web if requested
|
||||
if ($IncludeWeb) {
|
||||
Write-Host "[BONUS] Building web UI..." -ForegroundColor Yellow
|
||||
$webPath = Join-Path $rootPath "src\Web\autopilot-monitor-web"
|
||||
|
||||
Push-Location $webPath
|
||||
try {
|
||||
Write-Host " Installing npm packages..." -ForegroundColor Gray
|
||||
& npm install --silent
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " ✗ npm install failed" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host " Building Next.js app..." -ForegroundColor Gray
|
||||
& npm run build
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " ✗ Web build failed" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host " ✓ Web build completed" -ForegroundColor Green
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
# Summary
|
||||
Write-Host "============================================" -ForegroundColor Cyan
|
||||
Write-Host " Build Completed Successfully!" -ForegroundColor Green
|
||||
Write-Host "============================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
Write-Host "Output locations:" -ForegroundColor White
|
||||
Write-Host " Agent: src\Agent\AutopilotMonitor.Agent.V2\bin\$Configuration\net48\" -ForegroundColor Gray
|
||||
Write-Host " Functions: src\Backend\AutopilotMonitor.Functions\bin\$Configuration\net10.0\" -ForegroundColor Gray
|
||||
|
||||
if ($IncludeWeb) {
|
||||
Write-Host " Web UI: src\Web\autopilot-monitor-web\.next\" -ForegroundColor Gray
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Next steps:" -ForegroundColor White
|
||||
Write-Host " 1. Start Azurite: azurite --silent --location c:\azurite" -ForegroundColor Gray
|
||||
Write-Host " 2. Start Functions: cd src\Backend\AutopilotMonitor.Functions && func start" -ForegroundColor Gray
|
||||
|
||||
if ($IncludeWeb) {
|
||||
Write-Host " 3. Start Web UI: cd src\Web\autopilot-monitor-web && npm run dev" -ForegroundColor Gray
|
||||
Write-Host " 4. Test Agent: cd src\Agent\AutopilotMonitor.Agent.V2\bin\$Configuration\net48 && .\AutopilotMonitor.Agent.exe --console" -ForegroundColor Gray
|
||||
}
|
||||
else {
|
||||
Write-Host " 3. Test Agent: cd src\Agent\AutopilotMonitor.Agent.V2\bin\$Configuration\net48 && .\AutopilotMonitor.Agent.exe --console" -ForegroundColor Gray
|
||||
}
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
---
|
||||
type: Reference
|
||||
title: Autopilot ZTD Diagnostics — Event IDs, Registry Surfaces, Endpoints, Error Codes
|
||||
description: Windows' own diagnostic surfaces for the Autopilot profile-download (ZTD) flow and known deployment error codes — what the agent reads, what each identifier means, and where the knowledge comes from.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/DeviceInfo/ZtdEvidence.cs
|
||||
tags:
|
||||
- agent
|
||||
- autopilot
|
||||
- ztd
|
||||
- diagnostics
|
||||
- event-log
|
||||
- troubleshooting
|
||||
timestamp: 2026-07-16T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Autopilot ZTD Diagnostics
|
||||
|
||||
Windows documents its own diagnostic surfaces for the Autopilot profile-download flow
|
||||
(the Zero-Touch Deployment / "ZTD" service round-trip during OOBE). The agent consumes
|
||||
three of them to turn the `autopilot_profile_missing` warning from a guess into an
|
||||
evidence-backed verdict, and the backend rule engine uses the error-code map for
|
||||
known-issue enrichment.
|
||||
|
||||
Windows 365 Cloud PC exception: on Cloud PCs (`CloudPcDetector.DetectIsCloudPc()`)
|
||||
`ProfileAvailable=0` is the expected state — Cloud PCs are provisioned by the Windows 365
|
||||
service, not by Autopilot. The event is still emitted with the full evidence payload plus
|
||||
an `isCloudPc` marker, but as severity Info with a "this is expected" message instead of
|
||||
a Warning (`DeviceInfoCollector.ResolveAutopilotProfileMissingPresentation`). The portal's
|
||||
Autopilot Profile card mirrors this: a neutral note for Cloud PC sessions
|
||||
(`session.isCloudPc` or the event marker), the amber warning otherwise.
|
||||
|
||||
Consumers in this repo:
|
||||
|
||||
* `ZtdEvidence.cs` (agent) — one-shot event-log query + `Diagnostics\Autopilot` registry
|
||||
dump + deployment-service reachability probe on the profile-missing path.
|
||||
* `ModernDeploymentTracker.cs` (agent) — continuous watcher of the same channel,
|
||||
Level ≤ 3 only (Warning/Error/Critical) plus targeted Info IDs.
|
||||
* Analyze rules (backend) — HRESULT/known-issue map (see [Error codes](#schema--error-codes)).
|
||||
|
||||
# Schema — ModernDeployment Autopilot event channel
|
||||
|
||||
Channel: `Microsoft-Windows-ModernDeployment-Diagnostics-Provider/Autopilot`
|
||||
(Event Viewer: Application and Services Logs → Microsoft → Windows →
|
||||
ModernDeployment-Diagnostics-Provider → Autopilot).
|
||||
|
||||
| Event ID | Level | Meaning | Agent verdict mapping (`ztdVerdict`) |
|
||||
| --- | --- | --- | --- |
|
||||
| 100 | Warning | "Autopilot policy [name] not found" — temporary, device is waiting for the profile download. Windows can log hundreds of these per minute (harmless-rollup in `ModernDeploymentTracker`). | `waiting_for_profile_no_internet_confirmation` (when nothing else was logged) |
|
||||
| 101/103/109/111 | Info | Profile settings retrieval/processing details. | not queried |
|
||||
| 153 | Info | `ProfileState_Unknown → ProfileState_Available` — a profile was downloaded and is ready. | `profile_downloaded` |
|
||||
| 160 | Info | Profile settings acquisition beginning. | context only |
|
||||
| 161 | Info | Profile download succeeded. | `profile_downloaded` |
|
||||
| 163 | Info | Download not required — device already provisioned (typically after `Sysprep /Generalize`, i.e. Autopilot-for-existing-devices images). | `already_provisioned` |
|
||||
| 164 | Info | Internet available to attempt policy download — the authoritative witness that the ZTD round-trip could run at OOBE time. | with no 161: `download_attempted_no_profile_returned` (the assignment-gap signature, session 423b5360) |
|
||||
| 171 | Error | TPM identity confirmation failed (HRESULT in payload) — self-deploying/pre-prov attestation. | context only (`inconclusive` if alone) |
|
||||
| 172 | Error | Failed to set profile as available — usually follows 171. | context only |
|
||||
| 807 | Error | `ZtdDeviceIsNotRegistered` — hardware hash not registered in Autopilot. | `device_not_registered` |
|
||||
| 809 | Error | `ZtdDeviceHasNoAssignedProfile` — the assigned profile was deleted without cleanup. | `assigned_profile_deleted` |
|
||||
| 815 | Error | `ZtdDeviceHasNoAssignedProfile` — no profile assigned and no tenant default profile. | `no_profile_assigned` |
|
||||
| 908 | Error | `SerialNumberMismatch` / `ProductKeyIdMismatch` between Autopilot registration and physical hardware. | `serial_or_product_key_mismatch` |
|
||||
|
||||
Verdict priority: error IDs are authoritative (807 → 908 → 809 → 815), then the Info-flow
|
||||
reconstruction (161/153 → 163 → 164 → 100). Implemented in
|
||||
`ZtdEvidence.ComputeZtdVerdict`, pinned by `ZtdEvidenceVerdictTests`.
|
||||
|
||||
# Schema — Diagnostics registry key
|
||||
|
||||
`HKLM\SOFTWARE\Microsoft\Provisioning\Diagnostics\Autopilot` — written by the ZTD client,
|
||||
dumped verbatim by the agent into the `autopilot_profile` event (`diagnosticsRegistry`):
|
||||
|
||||
| Value | Meaning |
|
||||
| --- | --- |
|
||||
| `AadTenantId` | GUID of the tenant the user signed into. |
|
||||
| `CloudAssignedTenantDomain` | Tenant the device is registered with; **blank = not registered with Autopilot**. |
|
||||
| `CloudAssignedTenantId` | GUID matching `CloudAssignedTenantDomain`; blank when not registered. |
|
||||
| `IsAutopilotDisabled` | 1 = device not registered **or** profile download failed (network/firewall/timeout) — Microsoft's own catch-all indicator. **Casing caveat:** Windows actually writes `IsAutoPilotDisabled` (capital P; observed session e7dbac7a, 24H2) while the FAQ documents `IsAutopilotDisabled` — all agent lookups are case-insensitive for this reason. |
|
||||
| `TenantMatched` | 0 = user's tenant ≠ registration tenant → the "username belongs to another organization" OOBE error. |
|
||||
| `CloudAssignedOobeConfig` | Bitmap of configured OOBE settings (SkipCortanaOptIn=1, OobeUserNotLocalAdmin=2, SkipExpressSettings=4, SkipOemRegistration=8, SkipEula=16). |
|
||||
|
||||
The profile cache itself lives in `HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotPolicyCache`
|
||||
(`PolicyJsonCache`, `ProfileAvailable`) — `ProfileAvailable=0` plus an empty
|
||||
`ZeroTouchConfig` is the "blank profile cached" state the troubleshooting FAQ describes
|
||||
when no profile exists at check time. A reboot re-runs the download and can replace the
|
||||
blank cache.
|
||||
|
||||
# Schema — Endpoints
|
||||
|
||||
| Endpoint | Purpose | Agent behavior |
|
||||
| --- | --- | --- |
|
||||
| `https://ztd.dds.microsoft.com` | Autopilot Deployment Service — profile delivery during OOBE. | Reachability probe on profile-missing (any HTTP status = reachable). **Note:** several third-party blogs cite `ztd.ssd.microsoft.com`; the official requirements page says `ztd.dds.microsoft.com`. |
|
||||
| `https://login.live.com` | Second Deployment-Service URL; also needed by the Microsoft Sign-in Assistant (`wlidsvc`) — a policy that disables wlidsvc breaks profile download. | not probed |
|
||||
| `*.microsoftaik.azure.net` (+ per-vendor EK cert URLs) | TPM attestation for self-deploying/pre-provisioning. | not probed |
|
||||
| `lgmsapeweu.blob.core.windows.net` | Intune automatic diagnostics upload. | deliberately not probed (decision 2026-07-16: not mission-critical) |
|
||||
|
||||
# Observability — when the agent can actually see these errors
|
||||
|
||||
The agent's install timing bounds what the known-issue rules can ever observe. In the
|
||||
normal deployment model the agent is installed by an **Intune platform script via IME —
|
||||
i.e. only AFTER MDM enrollment succeeded**. Everything that happens before that point
|
||||
(profile download, TPM attestation in self-deploying/pre-provisioning, the MDM enrollment
|
||||
handshake itself) is over before our process exists.
|
||||
|
||||
| Failure class | Normal flow (IME install) | Bootstrap token mode | Await-enrollment mode |
|
||||
| --- | --- | --- | --- |
|
||||
| Enrollment blocked (0x80180014/18, 0xC1036501, 0x801C03F3 → ANALYZE-ENRL-004) | **Invisible** — no enrollment ⇒ no agent, no session. | **Visible** — the bootstrap token exists precisely to report pre-MDM (as of 2026-07: one customer piloting). | **Invisible to the backend** — the agent binary is present but only polls for the MDM certificate; without the cert it cannot authenticate, and monitoring starts only after the cert appears. A hard blocker ends in the await timeout (default 480 min) with nothing uploaded. |
|
||||
| TPM attestation (ANALYZE-ENRL-003) | **Visible via error backfill** — attestation precedes enrollment, but the startup error backfill (below) replays Autopilot-channel Level 1-2 records from the last 4h, so retry-then-success attempts surface as `backfilled=true` events. | Visible. | Visible via error backfill once monitoring starts. |
|
||||
| Hybrid-join 0x80004005 (ANALYZE-ENRL-005) | **Visible** — the ODJ/hybrid phase runs during device ESP while the agent is active. | Visible. | Visible after cert arrival. |
|
||||
| Clock skew (ANALYZE-DEV-007) | **Always** — ntp_time_check is our own event. | Always. | Always. |
|
||||
|
||||
The rules stay enabled despite the narrow normal-flow windows: they are evidence-gated
|
||||
(zero cost when the codes never appear) and they future-proof the bootstrap and
|
||||
await-enrollment paths.
|
||||
|
||||
## Startup error backfill (ModernDeploymentTracker)
|
||||
|
||||
At agent start (backfill enabled), the tracker replays Autopilot-channel **Level 1-2**
|
||||
records from the last **4 hours** through the normal emission pipeline — this is what makes
|
||||
pre-install errors (TPM attestation retries, ZTD errors 807/809/815/908) visible in the
|
||||
normal IME deployment flow. Spam containment, in order:
|
||||
|
||||
1. **Errors only** — the Warning-level 100 heartbeat storm is excluded by the XPath.
|
||||
2. **Harmless-ID downgrade + burst rollup** still applies to backfilled records.
|
||||
3. **Cap** of 25 records per scan (newest first; the tail is logged, not emitted).
|
||||
4. **RecordId watermark** (`autopilot-error-backfill.json`, advanced by both the backfill
|
||||
scan and the live watcher) — restarts never re-emit records a previous run forwarded.
|
||||
5. Backfilled events carry `backfilled: true` plus the original `timeCreated` in their
|
||||
payload; the event's own timestamp is the emission time, so the session's `StartedAt`
|
||||
(which the backend aligns to the earliest event) is not dragged backwards.
|
||||
|
||||
Presentation (2026-07-28, customer feedback — an unannounced wall of pre-agent Error
|
||||
events read like a live failure):
|
||||
|
||||
- The scan buffers the (capped) record set, first emits an **Info summary event**
|
||||
(`modern_deployment_log`, payload `backfillSummary: true` + `replayedCount` /
|
||||
`droppedOverCap` / `lookbackMinutes`) announcing the replay block, then replays the
|
||||
records **oldest-first** so the timeline sequence follows event-log chronology.
|
||||
- The web timeline (`EventTimeline.tsx`) badges `backfilled: true` events („⟲ Backfilled",
|
||||
slate, explanatory tooltip) and renders the original `timeCreated` as the row time with a
|
||||
"(reported …)" suffix; the expanded metadata block gains a `Recorded` line. Severity is
|
||||
deliberately **not** downgraded — unresolved pre-agent errors must still surface in the
|
||||
Error filter.
|
||||
|
||||
# Schema — Error codes (backend known-issue map)
|
||||
|
||||
Codes documented in the known-issues / troubleshooting-FAQ pages that appear in ESP/enrollment
|
||||
failures we can observe:
|
||||
|
||||
| Code | Meaning / context | Actionable guidance |
|
||||
| --- | --- | --- |
|
||||
| `0x800705B4` | Timeout; in self-deploying mode usually "device is not TPM 2.0 capable" (e.g. a VM). | Don't use self-deploying on VMs / non-TPM-2.0 hardware. |
|
||||
| `0x801C03EA` | TPM attestation failed → Entra join with device token failed. | TPM/firmware issue; check attestation endpoints. |
|
||||
| `0x81039001` | `E_AUTOPILOT_CLIENT_TPM_MAX_ATTESTATION_RETRY_EXCEEDED` (intermittent, "Securing your hardware"). | Retry provisioning; persistent → firmware. |
|
||||
| `0x81039023` | TPM attestation failure on Windows 11 (pre-prov/self-deploy). | Fixed by KB5013943 (21H2) or later CU. |
|
||||
| `0x81039024` | TPM has known vulnerabilities → attestation refused. | Update TPM firmware from OEM. |
|
||||
| `0x80070490` | TPM attestation failure on AMD ASP fTPM platforms. | Update AMD firmware. |
|
||||
| `0xC1036501` | Automatic MDM enrollment impossible — multiple MDM configurations in Entra ID. | Consolidate MDM configs. |
|
||||
| `0x801C03F3` | Entra device object for the Autopilot device was deleted (logged in `Microsoft-Windows-User Device Registration/Admin`). | Deregister + re-register the device (recreates the object). |
|
||||
| `0x80180014` | Device-record reuse in self-deploying/pre-prov mode, or Windows MDM enrollment disabled ("Enrollment blocked for AP device by SDM One Time Limit Check"). | Intune → Autopilot device → **Unblock device**, or allow Windows (MDM) in enrollment restrictions. |
|
||||
| `0x80180018` | MDM enrollment refused — licensing (missing Intune/EMS license) or device cap. | Check user license + device limits. |
|
||||
| `0x80070774` | Hybrid-join ESP failure — domain mismatch between Intune Connector for AD and device targeting. | Install/configure the connector in the matching domain. |
|
||||
| `0x80004005` | Hybrid-join deployment timeout — **build-dependent known issue**, fixed in KB5065789 (25H2, ≥ 26200.6725), KB5065426 (24H2, ≥ 26100.6584), KB5070312 (23H2, ≥ 22631.6276). | Update the image / install the KB. |
|
||||
|
||||
Related non-code known issues worth remembering: clock skew of minutes causes TPM
|
||||
attestation errors and ESP timeouts (`w32tm /resync /force`; the agent's `ntp_time_check`
|
||||
event carries the measured offset); multiple/unexpected OOBE reboots can delete kiosk
|
||||
autologon registry entries (fixed in 24H2 KB5058411); ESP does not support mixing LOB and
|
||||
Win32 apps ("Another installation is in progress").
|
||||
|
||||
# Citations
|
||||
|
||||
* [Windows Autopilot troubleshooting FAQ](https://learn.microsoft.com/en-us/autopilot/troubleshooting-faq) — event IDs, registry values, process flow. (Checked 2026-07-16.)
|
||||
* [Windows Autopilot known issues](https://learn.microsoft.com/en-us/autopilot/known-issues) — error codes, KB fixes. (Checked 2026-07-16.)
|
||||
* [Windows Autopilot requirements — Networking](https://learn.microsoft.com/en-us/autopilot/requirements?tabs=networking) — endpoint list incl. `ztd.dds.microsoft.com`. (Checked 2026-07-16.)
|
||||
* Re-check cadence: both Learn pages change regularly (known-issues carries an RSS feed:
|
||||
`https://learn.microsoft.com/api/search/rss?search=%22Be+informed+about+known+issues+that+might+occur+during+Windows+Autopilot+deployment.%22&locale=en-us&%24filter=`).
|
||||
When new event IDs / error codes appear, update `ZtdEvidence.cs`, the backend rule map,
|
||||
and this document together.
|
||||
@@ -1,65 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Agent Build Numbers — Shared Counter Blob with ETag-CAS
|
||||
description: Why agent build numbers come from a public counter blob reserved before the build, how the compare-and-swap protocol works, and what the repo's buildcounter.txt still does.
|
||||
resource: /.github/workflows/build-agent.yml
|
||||
tags:
|
||||
- agent
|
||||
- build
|
||||
- ci
|
||||
- release
|
||||
- versioning
|
||||
timestamp: 2026-07-17T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Agent Build Numbers — Shared Counter Blob
|
||||
|
||||
Agent (V2 line) build numbers — the `<n>` in `2.0.<n>` — are minted from a single
|
||||
shared counter blob so that **local builds (`scripts/Deployment/build.ps1`) and CI
|
||||
builds (`.github/workflows/build-agent.yml`) can never collide**. Both build paths
|
||||
stay first-class: local daily-ops builds remain possible, CI builds add provenance
|
||||
attestation on top.
|
||||
|
||||
# Schema
|
||||
|
||||
The reserve-before-build CAS protocol itself is shared with the backend and the MCP
|
||||
server and is documented once in [Version Contract](../versioning.md); the single
|
||||
implementation is `.github/scripts/Request-BuildNumber.ps1`. Agent-specific:
|
||||
|
||||
- **Counter blob**: `https://autopilotmonitoreu.blob.core.windows.net/agent/buildcounter-v{LINE}.txt`
|
||||
— one per release line, analogous to `version-v{LINE}.json`, and kept in the `agent`
|
||||
container next to the artifacts it numbers rather than in `versions/`. Content: the
|
||||
**last used** build number as plain text. Public-read like the rest of the container
|
||||
(the number is visible in `version-v{LINE}.json` anyway); writes require the SAS.
|
||||
- The reserved number reaches the build as `BUILD_NUMBER` in the environment. The
|
||||
csproj honors it and then skips its `PersistBuildCounter` target, so the repo file
|
||||
is not touched.
|
||||
- **CI manual override** (`build_number` input): numbers `<=` counter are re-builds
|
||||
and leave the blob unchanged; numbers `>` counter advance it (monotonic guard).
|
||||
- V2 was seeded with `1356` on 2026-07-17.
|
||||
|
||||
# Consequences
|
||||
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt` is now only an **offline
|
||||
fallback** for `-SkipUpload` smoke builds and bare `dotnet build` (where the version
|
||||
does not matter). It is never committed anymore; the CI step that pushed counter
|
||||
bumps to `main` was removed.
|
||||
- `AdminConfiguration.LatestAgentV2*` (backend hash-oracle) is updated **only on
|
||||
stable cutover** (`publish_as_stable` / default `build.ps1` run without
|
||||
`-VersionedOnly`/`-Dev`) — it must never point at a build that is not actually in
|
||||
the stable namespace.
|
||||
- CI builds additionally attest build provenance for the release ZIP via
|
||||
`actions/attest-build-provenance` (Sigstore keyless, bound to the workflow OIDC
|
||||
identity). The digest covers the identical bytes uploaded to blob, so one
|
||||
attestation covers all copies. Verify:
|
||||
`gh attestation verify <zip> --repo okieselbach/AutopilotMonitor`. Local builds
|
||||
cannot produce this attestation — provenance is a CI-only property.
|
||||
|
||||
# Citations
|
||||
|
||||
- [Version Contract](../versioning.md) — the shared CAS protocol and manifest format
|
||||
- `.github/workflows/build-agent.yml` — "Reserve build number (counter blob CAS)" step
|
||||
- `.github/scripts/Request-BuildNumber.ps1` — the shared implementation
|
||||
- `scripts/Deployment/build.ps1` (gitignored, contains SAS) — local build path
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2/AutopilotMonitor.Agent.V2.csproj` —
|
||||
`BUILD_NUMBER` property + `PersistBuildCounter` target
|
||||
@@ -1,184 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: CMTrace Time Resolution — Per-Line Self-Anchoring
|
||||
description: How the agent converts bias-less CMTrace local times to UTC without trusting any process's timezone belief — each provably fresh line anchors its own offset against the agent clock on the 15-minute grid; everything not provably fresh falls back uniformly and says so.
|
||||
resource: src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime
|
||||
tags: [agent, ime, cmtrace, timestamps, timezone, calibration, provenance]
|
||||
timestamp: 2026-09-01
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
A CMTrace line carries local time and no offset — IME 1.104 writes `DateTime.Now.TimeOfDay`
|
||||
in both of its trace listeners. Converting such a line with the reader's own
|
||||
`TimeZoneInfo.Local` silently assumes the writing and reading processes agree on the zone.
|
||||
They do not: each caches its zone at process start and follows neither a later `tzutil` nor
|
||||
a Windows auto-timezone change. The error is `Offset_writer − Offset_reader` — zero only
|
||||
when both believe the same thing, right or wrong. Field measurement over 11,068 sessions
|
||||
found 26 sessions with a real error, from +1 h to −17 h.
|
||||
|
||||
Two designs failed before this one:
|
||||
|
||||
* **Fix the reader's belief** (`TimeZoneInfo.ClearCachedData`, live OS offset): makes the
|
||||
agent isolated-correct and thereby breaks the majority of sessions, where two equally
|
||||
stale beliefs cancel.
|
||||
* **One measured offset per file** (agent 2.0.1410, reverted in `04b1a7c6`): a single log
|
||||
file holds lines from multiple writer *eras*. `AgentExecutor.log` is written by
|
||||
short-lived child processes whose zone belief flips per process — interleaved in one
|
||||
file, and flipping *back* (fixture:
|
||||
`tests/fixtures/cmtrace-logs/agentexecutor-two-writer-eras-v1.cmtrace`). Any cross-line
|
||||
anchor — per file, or nearest-in-write-order — applies one era's offset to the other
|
||||
era's lines. In the field this shifted `script_started` events by −9 h against their
|
||||
correct completions.
|
||||
|
||||
# Schema
|
||||
|
||||
The resolution rule, in order:
|
||||
|
||||
1. **Writer-declared bias** (`+480` suffix): authoritative, used as-is. Never a calibration
|
||||
anchor — it needs no measurement and must not overwrite one.
|
||||
2. **Per-line self-anchoring** — only for lines read in a *provably fresh* pass:
|
||||
|
||||
```
|
||||
offset_line = round((local − agentUtcNow) / 15 min) · 15 min
|
||||
lineUtc = local − offset_line
|
||||
```
|
||||
|
||||
Every real UTC offset is a whole multiple of 15 minutes, so the rounding absorbs poll
|
||||
and flush latency. No cross-line state exists, so interleaved eras resolve line by
|
||||
line — the property the fixture test pins. The line's own timestamp still contributes
|
||||
sub-poll precision and ordering (two lines 6 ms apart stay 6 ms apart; a plain
|
||||
`occurredAt = now` could not do that). Guards: grid residual ≤ 2 min, offset within
|
||||
UTC−12 … UTC+14.
|
||||
3. **Uniform reader-zone fallback** for everything else: wrong in absolute terms whenever
|
||||
the writer held a different belief, but *self-consistent* — both ends of a duration are
|
||||
wrong by the same amount, so derived durations stay right. The revert's core lesson:
|
||||
partially corrected is strictly worse than uniformly wrong.
|
||||
|
||||
**Freshness** is what makes rule 2 sound, and it is strict:
|
||||
|
||||
* Every poll look at a file stamps `LastCheckedUtc` (`LogFilePositionTracker.MarkChecked`),
|
||||
including "no new data" and an empty first sight — so a file first seen empty
|
||||
(`AgentExecutor.log` appearing mid-enrollment) anchors from its very first content.
|
||||
* A pass's lines are fresh only if the file was previously observed *in this process* and
|
||||
the gap since the last look is ≤ `FreshLineMaxAge` (30 s — far above the 100 ms poll,
|
||||
far below the 13-minute zone where a line's *age* could round onto the offset grid).
|
||||
* A restart-restored position bookmark deliberately carries **no** freshness: the first
|
||||
pass after a restart reads downtime backlog and must never anchor.
|
||||
|
||||
**Provenance** (`sourceOffsetOrigin` in event DataJson): `bias` | `line-anchored` |
|
||||
`reader-zone-fallback` (plus retired `calibrated` from 2.0.1410 events). The applied
|
||||
offset is always recorded (`sourceOffsetMinutes`), the raw local time too
|
||||
(`sourceLocalTs`), so any event can be re-derived later. The per-*file* measurement still
|
||||
runs, purely observationally (`measuredWriterOffsetMinutes`, Info log when it disagrees
|
||||
with the process zone).
|
||||
|
||||
# Constraints
|
||||
|
||||
* An anchored line resolves to `now ± 2 min` *by construction*. The accepted edge: a
|
||||
freshly written line carrying a **replayed** old timestamp whose age is exactly
|
||||
N×15 min ± 2 min anchors to ≈now instead of its replayed past — bounded error, and
|
||||
sub-24 h replays are treated as current by the historic-replay guard anyway.
|
||||
* Snapshot readers (`StallProbeCollector`, `LogParserCollector`) cannot tail and therefore
|
||||
cannot prove freshness — they stay on the marked fallback. This is a documented limit,
|
||||
not an omission; sharing the per-file measurement with them would re-apply exactly what
|
||||
the revert removed.
|
||||
* Mixed pairs remain possible across a restart (start resolved from backlog fallback,
|
||||
completion line-anchored) and are recognizable via provenance.
|
||||
|
||||
# Line parsing (bounded cost)
|
||||
|
||||
`CmTraceLogParser.TryParseLine` (Shared) is the single CMTrace line parser for the agent
|
||||
(`ImeLogTracker`, `LogParserCollector`, `StallProbeCollector`) and the backend
|
||||
(`TestLogPatternFunction`). Both feed it input an outsider influences: tenant-supplied
|
||||
sample lines (200 x 8 KB per request) on the backend, and on the agent every tailed log
|
||||
file, including entries that IME assembles from installer or script output. It is
|
||||
therefore not one regex. The former shape — a greedy `(?<message>.*)` in front of the
|
||||
`]LOG]!><time="` literal, unanchored, no match timeout — re-tried every `<![LOG[`
|
||||
occurrence as a fresh start and backtracked the message group across every later character
|
||||
for each: quadratic in line length (an 8 KB line of repeated prefixes and `]` filler cost
|
||||
~7 million engine steps; a 1 MB assembled entry would have spun the SYSTEM agent for
|
||||
hours and blacked out IME tracking for the rest of the enrollment).
|
||||
|
||||
The parser reproduces exactly what that regex returned, in linear time: the message begins
|
||||
after the first `<![LOG[` (a later start could only match when the first one does, since
|
||||
`.*` spans the difference) and ends at the last `]LOG]!><time="` occurrence whose trailer
|
||||
parses — earlier occurrences are tried in turn, which is the order greedy backtracking
|
||||
visited them. The trailer is checked with a `\G`-anchored regex that has no ambiguous
|
||||
quantifier and a 1-second match timeout as a backstop; a timeout means "not a CMTrace
|
||||
line" (callers already have that path: raw-text matching in the tracker, `parse_failed`
|
||||
on the backend), never an exception. `CmTraceLogParserTests` pins the equivalence against
|
||||
the legacy regex over the shapes that matter (multiline messages, bias suffix, trailer
|
||||
literal inside the message, a last trailer that fails to parse, BOM prefix) and the
|
||||
reported worst case at 200 lines.
|
||||
|
||||
On the agent, `ImeLogTracker` additionally caps an assembled multiline entry at 100 raw
|
||||
lines **and** 1 MB of characters (`MaxMultiLineBufferChars`). A capped entry is dropped
|
||||
and logged at **Warning** — visible at the default `Info` level, so a real IME entry that
|
||||
large shows up in the client log and can be acted on. The remaining physical lines of a
|
||||
dropped entry are skipped until it closes or a new `<![LOG[` entry begins; previously they
|
||||
fell through to raw-text pattern matching, so fragments of a discarded entry could fire
|
||||
patterns without CMTrace context.
|
||||
|
||||
# Backend regression tripwire
|
||||
|
||||
The backend watches for the failure mode this design eliminates. On every terminal ingest
|
||||
batch, the counter reconcile's existing event-partition scan additionally collects
|
||||
`Δ = ReceivedAt − OccurredUtc` per event, split by `Source == "ImeLogTracker"` versus the
|
||||
rest (`CmTraceSkewTripwire`, hooked in `EventIngestProcessor`). If
|
||||
`median(Δ_IME) − median(Δ_other)` is a clean 15-minute multiple (residual < 2 min — the
|
||||
same constants as the anchoring grid guard above) with ≥ 20 samples over ≥ 3 distinct
|
||||
upload batches per side, and ≥ 80 % of the individual IME deltas (relative to the other
|
||||
side's median) themselves sit within 2 min of some grid multiple — added after the 2026-08-28
|
||||
soak, when relaunched agents re-tailing the IME log produced one burst holding most of a
|
||||
session's IME samples with a continuum of ages whose median hit the grid by chance (7 false
|
||||
fires in a week, 0 true) — a `CmTraceTimeSkewRegression` ops event fires (category Agent,
|
||||
warning), carrying all numbers plus the session's `sourceOffsetOrigin` histogram.
|
||||
Bias-dominated sessions are suppressed (writer-declared offsets cannot be an anchoring
|
||||
regression); `measuredWriterOffsetMinutes` is never consulted (sticky after era
|
||||
flip-backs). Kill switch: app setting `CmTraceSkewTripwireDisabled`. Goal state: the event
|
||||
never fires — any hit is an anchoring case the per-line design misses, or a detector bug.
|
||||
|
||||
Samples are windowed to the session's most recent INGEST ERA before any of that is
|
||||
computed: the batch stamps of both sides are walked backwards from the newest one and the
|
||||
era ends at the first `ReceivedAt` gap wider than 2 h. A session partition is not one agent
|
||||
run — a pre-provisioning session's Part 1 is written by whatever agent build was current
|
||||
weeks earlier, and its events stay in the partition forever. Field 2026-09-01 (sessions
|
||||
`e797117b` / `c06d639d` / `d7c8032b`, one tenant): 26 IME samples at exactly −60 min from a
|
||||
2026-08-20 technician leg under agent 2.0.1409 — before per-line anchoring — outnumbered the
|
||||
3…9 clean samples of the user leg that completed that morning and fired the tripwire against
|
||||
2.0.1445, a build the device had self-updated to minutes earlier. The skew was real, 12 days
|
||||
old, and already fixed; no scan-wide statistic can tell those apart, because event rows carry
|
||||
no agent version. The era boundary can. `eraStartUtc` and the excluded per-side sample counts
|
||||
travel in the ops event, so an inherited leg stays visible without being alarmed on. The
|
||||
2 h threshold sits above any in-leg upload gap (a live agent uploads self-metrics every few
|
||||
minutes, reports `session_stalled` at 60 min idle, and the maintenance sweep classifies
|
||||
silence at 2 h) and below every pre-provisioning handover.
|
||||
|
||||
Device-clock problems (the customer-actionable cousin) are covered separately by the
|
||||
`clock_skew` analyze condition behind `ANALYZE-DEV-008`, which excludes IME-derived events
|
||||
precisely so an anchoring regression can never surface as a customer finding.
|
||||
|
||||
# Citations
|
||||
|
||||
* `CmTraceOffsetCalibrator.TryMeasureOffset` — the pure grid measurement, shared by
|
||||
anchoring and the observational per-file path.
|
||||
* `CmTraceSkewTripwire` + `CmTraceSkewTripwireTests` — the backend tripwire above.
|
||||
* `RuleEngine.ConditionEvaluators.EvaluateClockSkewCondition` + `RuleEngineClockSkewTests`
|
||||
— the customer-facing device-clock metrics (batch-median frames, spool-spread filter,
|
||||
plateau step detection, end-state persistence).
|
||||
* `ImeLogTracker.LogProcessing.cs` `ResolveEntryUtc` — the resolution order above.
|
||||
* `ImeLogTrackerLineAnchoringTests` — resolves-to-T across writer beliefs, interleaved
|
||||
eras in one chunk, freshness boundaries (first sight, empty first sight, poll gap,
|
||||
restart restore), the grid-replay edge, and the field fixture replayed at production
|
||||
poll cadence.
|
||||
* `CmTraceCalibratorFileIsolationTests` — pins per-file isolation of the observational
|
||||
measurement (session e9753578 logged a cross-file pairing the committed code could not
|
||||
be shown to produce; the mechanism remains open, the resolution path is immune by
|
||||
design, and `CalibrateFrom` logs the anchor timestamp as a tripwire).
|
||||
* `tasks/todo.md` (2026-08-20) — full evidence trail: field measurements, the revert,
|
||||
the two-writer-era fixture, and the disproven foreign-binary hypothesis (the SHA
|
||||
mismatch was the known release race against the 5-minute AdminConfiguration cache).
|
||||
* `CmTraceLogParser.TryParseLine` + `CmTraceLogParserTests` — the linear parser and its
|
||||
legacy-regex equivalence oracle; `ImeLogTracker.LogProcessing.cs` multiline buffering
|
||||
(`MaxMultiLineBufferLines` / `MaxMultiLineBufferChars`, `skippingDroppedEntry`).
|
||||
@@ -1,114 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Continue-Anyway Observation Mode — Device-Phase ESP Failure Defang
|
||||
description: Operator-set tenant opt-in that turns an immediate Device-phase ESP terminal failure on a Continue-Anyway-enabled profile into a bounded 60-minute observation; the DAD-validated real-user desktop plus the Hello gate detect the true end, and the session completes as Succeeded with an esp-soft-failure marker.
|
||||
resource: /src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs
|
||||
tags:
|
||||
- agent
|
||||
- decision-engine
|
||||
- esp
|
||||
- continue-anyway
|
||||
- soft-failure
|
||||
- tenant-config
|
||||
timestamp: 2026-08-08T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Continue-Anyway Observation Mode
|
||||
|
||||
## Problem
|
||||
|
||||
A Classic user-driven enrollment whose Device ESP hits the configured terminal timeout
|
||||
(`syncFailureTimeoutMinutes`, typically 30) fails **before AccountSetup is ever entered**.
|
||||
The classic advisory defang (`HandleEspTerminalFailureV1`) requires
|
||||
`AccountSetupEnteredUtc != null`, so these sessions hard-fail immediately — even when the
|
||||
ESP profile allows "Continue anyway" and users demonstrably dismiss the failure screen and
|
||||
reach a working desktop. A fleet held at the timeout wall by one slow blocking app then
|
||||
shows ~100% Failed (tenant c9787ba2: 948/1000 `esp_terminal_failure`, culprit "Encompass
|
||||
Hybrid Installer"), which tells the tenant nothing.
|
||||
|
||||
## Mechanism
|
||||
|
||||
Opt-in flow (default off — absence keeps the immediate hard fail):
|
||||
|
||||
1. **Tenant setting** `EnableEspContinueAnywayObservation` (`TenantConfiguration`,
|
||||
operator-set only, NOT exposed in the tenant admin settings UI; the toggle lives in
|
||||
the Global-Admin tenant management modal next to Bootstrap Token / Unrestricted Mode,
|
||||
and the field is GA-gated on both write paths — the generic PUT silently reverts it
|
||||
for tenant admins, the field patch rejects it with 400) → served via `agent/config`
|
||||
(`AgentConfigResponse`, ConfigVersion 37).
|
||||
2. **Agent** stamps `espContinueAnywayObservationEnabled=true` into the
|
||||
`EspConfigDetected` signal payload (both emitters: the orchestrator FirstSync
|
||||
bootstrap and `DeviceInfoCollector.PostEspConfigDetectedSignal`) — only when enabled,
|
||||
so the kernel stays config-free and replay/recovery reconstruct the fact from the
|
||||
signal log.
|
||||
3. **Engine** (`HandleEspTerminalFailureV1`): a terminal ESP failure is defanged into an
|
||||
`esp_failure_advisory` (`advisoryReason=esp_failure_defanged_continueanyway_observation`)
|
||||
when ALL hold: profile allows Continue-Anyway, observation opt-in observed,
|
||||
`AccountSetupEnteredUtc == null`, and the scenario is Classic/Unknown with
|
||||
`PreProvisioningSide=None` and no registry self-deploying marker (WhiteGlove /
|
||||
SelfDeploying / DevicePreparation have no interactive user to press the button).
|
||||
4. **Observation window**: the shared `AdvisoryCompletion` deadline is armed with a
|
||||
**60-minute** window (classic advisory: 30) — long enough for failure screen →
|
||||
Continue anyway → sign-in → desktop, or a full device-phase re-run via "Try again".
|
||||
|
||||
## Detecting the real end
|
||||
|
||||
The active-advisory-without-AccountSetup shape is unique to this arming site, so
|
||||
`IsContinueAnywayObservationActive(state)` (advisory recorded ∧ not resolved ∧
|
||||
AccountSetup never entered) needs no separate opt-in re-check. Three outcomes:
|
||||
|
||||
* **Continue anyway** — the DAD-validated real-user desktop arrives (defaultuser0/SYSTEM
|
||||
excluded at the detector) and the Hello gate is satisfied (resolved, or policy
|
||||
explicitly disabled without an observed wizard): `HandleDesktopArrivedV1` completes
|
||||
eagerly through Finalizing (trigger `DesktopArrived:ContinueAnywayObservation`) — the
|
||||
session end reflects the real end, not the window edge. The IME user-session gate is
|
||||
deliberately waived: it anchors on `AccountSetupEnteredUtc`, which is structurally
|
||||
absent here. The terminal `enrollment_complete` carries `espSoftFailure=true`,
|
||||
`completionSource=continue_anyway_observation` and the failed category; the backend
|
||||
stamps `EspSoftFailure`/`CompletionSource` on the Sessions row (+ SessionsIndex
|
||||
mirror) and the web renders the amber "with issues" badge plus a detail banner.
|
||||
Sessions completing under the CLASSIC advisory (AccountSetup entered) carry the same
|
||||
marker with `completionSource=continue_anyway_post_accountsetup`.
|
||||
* **Try again** — the ESP re-runs the failed category; `DeviceSetupProvisioningComplete`
|
||||
resolves the advisory via the 4910a5a5 recovery hook (`esp_failure_advisory_resolved`)
|
||||
and the session continues on the normal rails, completing clean (no marker).
|
||||
* **Nothing** — the window expires: un-defang to `enrollment_failed` with the original
|
||||
`esp_terminal_failure` reason and
|
||||
`advisoryReason=continue_anyway_observation_window_expired_without_completion_evidence`;
|
||||
`LastFailureTrigger=EspTerminalFailure` keeps the likely-stuck app promotion on. Every
|
||||
observed session therefore still gets a terminal verdict ≤ ~60 min after the failure.
|
||||
|
||||
Deadline-fire edge cases mirror the classic paths: desktop + unknown Hello promotes to
|
||||
`AwaitingHello` with the HelloSafety window (never-observed promote), and enforcement
|
||||
progress since arming re-arms the window (30-min re-arm, convergent) instead of failing a
|
||||
device mid-"Try again".
|
||||
|
||||
## Status semantics
|
||||
|
||||
No new terminal status: the session is `Succeeded` + `EspSoftFailure=true` (user
|
||||
decision 2026-08-08). Success-rate conventions stay untouched; honesty lives in the
|
||||
badge, the detail banner, and ANALYZE-ESP-004 (which continues to fire on the un-defang
|
||||
expiry path via `mayHaveContinuedAnyway`).
|
||||
|
||||
# Examples
|
||||
|
||||
Session `53d1e9f6-4eef-4f3b-a999-c63e96c5148d` (tenant c9787ba2, 2026-08-07) is the
|
||||
motivating shape: DeviceSetup/Certificates failed at the 30-min wall while "Encompass
|
||||
Hybrid Installer" was still installing, AccountSetup 0/5, Continue-Anyway allowed —
|
||||
hard-failed under the default semantics.
|
||||
|
||||
Tests: `ContinueAnywayObservationTests` (DecisionCore.Tests) — defang, 60-min arming,
|
||||
opt-out/scenario regressions, eager desktop completion with marker, clean-completion
|
||||
marker absence, expiry un-defang, hello promotes, category recovery.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs` — arming site,
|
||||
observation branches in `HandleAdvisoryCompletionDeadlineFired`,
|
||||
`IsContinueAnywayObservationActive`.
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Classic.cs` —
|
||||
`HandleDesktopArrivedV1` observation completion, `BuildEnrollmentCompleteEffect`
|
||||
soft-failure marker.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs`
|
||||
/ `TableStorageService.Sessions.cs` — session stamping + index mirror.
|
||||
* `src/Web/autopilot-monitor-web/components/SessionStatusBadge.tsx` — amber badge.
|
||||
@@ -1,271 +0,0 @@
|
||||
---
|
||||
type: Reference
|
||||
title: V2 Agent — Decision Engine (DecisionCore)
|
||||
description: The pure-reducer state machine that decides enrollment completion/failure — signals, state, effects, the A/B/C completion arms, and its invariants.
|
||||
resource: /src/Shared/AutopilotMonitor.DecisionCore
|
||||
tags:
|
||||
- agent
|
||||
- decision-engine
|
||||
- reducer
|
||||
- completion
|
||||
timestamp: 2026-08-30T00:00:00+02:00
|
||||
---
|
||||
|
||||
# V2 Agent — Decision Engine (DecisionCore)
|
||||
|
||||
DecisionCore (`src/Shared/AutopilotMonitor.DecisionCore`, netstandard2.0) is a **pure
|
||||
reducer**: a deterministic state machine with no I/O, no clock reads, no threads. The
|
||||
agent feeds it an ordered stream of *signals*; it returns a new immutable state plus a
|
||||
list of *effects* the host must execute. Because it is pure, the same persisted signal
|
||||
stream always reproduces the same journal and terminal outcome — that is what makes
|
||||
crash recovery a replay instead of a guess (determinism guarantee "L.5",
|
||||
`Engine/IDecisionEngine.cs`).
|
||||
|
||||
# The reducer contract
|
||||
|
||||
```
|
||||
DecisionStep step = engine.Reduce(DecisionState oldState, DecisionSignal signal)
|
||||
// DecisionStep = { NewState, Transition, Effects[] }
|
||||
```
|
||||
|
||||
- `Reduce` never throws: a handler exception advances only bookkeeping and journals a
|
||||
`reducer_exception` dead-end (fail-safe L.16).
|
||||
- Dispatch is a switch on `(signal.Kind, signal.KindSchemaVersion)`; unknown pairs
|
||||
become journaled dead-ends, never crashes.
|
||||
- Every step yields a `DecisionTransition` journal record: `Taken` (state change vs
|
||||
dead-end + reason), `StepIndex`, `FromStage/ToStage`, `Trigger`, `Guards`,
|
||||
`ReducerVersion`.
|
||||
|
||||
# State
|
||||
|
||||
`DecisionState` (schema **v4**) is an immutable value object; "change" = builder copy.
|
||||
Key content:
|
||||
|
||||
- Identity + progress: `SessionId`, `TenantId`, `Stage`, `Outcome`, `StepIndex`,
|
||||
`LastAppliedSignalOrdinal`, `AgentBootUtc` (replay-safety anchor).
|
||||
- **Facts** as `SignalFact<T>` — value **plus** `SourceSignalOrdinal` provenance, e.g.
|
||||
`AccountSetupEnteredUtc`, `EspFinalExitUtc`, `DesktopArrivedUtc`, `HelloResolvedUtc`,
|
||||
`ImeUserSessionCompletedUtc`, `HelloPolicyEnabled`, `HelloOutcome`.
|
||||
- Aggregates: `AppInstallFacts`, `ScenarioProfile` + `ScenarioObservations`
|
||||
(enrollment-scenario classification, monotonic), `ClassifierOutcomes`, `RealmJoinFacts`.
|
||||
- `Deadlines` — active timers the host has been asked to arm.
|
||||
|
||||
Stages: `SessionStarted → EspDeviceSetup → EspAccountSetup → AwaitingHello →
|
||||
AwaitingDesktop → Finalizing → Completed` (plus `Failed`, `WhiteGloveCandidate/Sealed`,
|
||||
`AwaitingDeviceOnlyEsp` for self-deploying, and edge stages). Terminal =
|
||||
`Completed | Failed | WhiteGloveSealed`. `Finalizing` is deliberately non-terminal
|
||||
(grace window). Outcomes: `EnrollmentComplete`, `EnrollmentFailed`,
|
||||
`WhiteGlovePart1Sealed`, `Aborted`, `AdminPreempted`.
|
||||
|
||||
# Signals
|
||||
|
||||
`DecisionSignal` is the only input. Fields: `Kind` + `KindSchemaVersion` (dispatch key),
|
||||
`SessionSignalOrdinal` (monotonic, assigned **only** by the ingress worker — never by
|
||||
collectors), `OccurredAtUtc`, mandatory `Evidence` (classified `Raw | Derived |
|
||||
Synthetic` — a signal without evidence is a bug), `Payload` (string dict) and
|
||||
`TypedPayload` (structured sidecar).
|
||||
|
||||
Signal kinds by family (`Signals/DecisionSignalKind.cs`):
|
||||
|
||||
- **Raw observations**: `EspPhaseChanged`, `EspExiting`, `EspResumed`,
|
||||
`EspTerminalFailure`, `DesktopArrived`, `HelloResolved`, `ImeUserSessionCompleted`,
|
||||
`DeviceSetupProvisioningComplete`, `AccountSetupProvisioningComplete`,
|
||||
`AppInstallCompleted/Failed`, `AadUserJoinedLate`, `SystemRebootObserved`,
|
||||
`EspConfigDetected`, `HelloPolicyDetected`, WhiteGlove + RealmJoin kinds, …
|
||||
- **Synthetic**: `DeadlineFired` (scheduler timer), `ClassifierVerdictIssued`,
|
||||
`EffectInfrastructureFailure`.
|
||||
- **Lifecycle**: `SessionStarted`, `SessionAborted`, `AdminPreemptionDetected`.
|
||||
- **`InformationalEvent`** — the single-rail pass-through: telemetry that must appear
|
||||
on the Events timeline but never influences decisions.
|
||||
|
||||
**Ordering is the ingest ordinal, never wall-clock.** Replayed IME/CMTrace lines carry
|
||||
backdated source timestamps; all before/after comparisons in the reducer therefore
|
||||
compare `SourceSignalOrdinal`s (e.g. "final exit after AccountSetup entry").
|
||||
|
||||
# Effects — the only side-channel
|
||||
|
||||
`DecisionEffect` kinds and how the host's `EffectRunner` treats failures:
|
||||
|
||||
| Kind | Purpose | Failure class |
|
||||
| --- | --- | --- |
|
||||
| `ScheduleDeadline` / `CancelDeadline` | arm/cancel OS timers that come back as `DeadlineFired` | **critical** — failure aborts the session (a lost timer means silent parking) |
|
||||
| `EmitEventTimelineEntry` | timeline events incl. terminal `enrollment_complete` / `enrollment_failed` | transient — retried 100/400/1600 ms |
|
||||
| `PersistSnapshot` | hint to write the recovery snapshot | transient |
|
||||
| `RunClassifier` | scenario classification; verdict returns as a `ClassifierVerdictIssued` signal | optional — exception ⇒ Inconclusive, never aborts |
|
||||
|
||||
Terminal and state-changing timeline effects carry a structured **audit trail** in
|
||||
`TypedPayload` (built by `DecisionAuditTrailBuilder`): `decisionSource`, `trigger`,
|
||||
`sessionStage`, `stepIndex`, `signalsSeen`, `signalEvidence`, `signalTimestamps`,
|
||||
`scenario` — so every completion/failure in the portal is explainable from its own
|
||||
event payload.
|
||||
|
||||
# Completion logic
|
||||
|
||||
## The A/B/C arms
|
||||
|
||||
The arms are **not** three pipelines — they are three alternative sufficient conditions
|
||||
inside one guard, `ShouldTransitionToAwaitingHello`
|
||||
(`Engine/DecisionEngine.Shared.cs`), which gates promotion out of the ESP stages toward
|
||||
Hello/desktop completion:
|
||||
|
||||
- **Arm A — registry-strong AccountSetup success**: `AccountSetupProvisioningSucceededUtc`
|
||||
is set (the ESP registry reported `categorySucceeded`). Merely *entering* AccountSetup
|
||||
is insufficient — Shell-Core 62407 fires at every ESP page transition.
|
||||
- **Arm B — SkipUser flow**: `ScenarioObservations.SkipUserEsp == true` — with no
|
||||
User-ESP page, the first `esp_exiting` *is* the genuine final exit.
|
||||
- **Arm C — 4-fact eager completion** (all four mandatory):
|
||||
1. AccountSetup was entered, and
|
||||
2. a post-AccountSetup ESP final exit occurred (ordinal-ordered), and
|
||||
3. a *genuine* IME user-session completion (at-or-after the AccountSetup anchor —
|
||||
guards against the `defaultuser0` ghost), and
|
||||
4. a DAD-validated real-user desktop arrival.
|
||||
Arm C applies eagerly the same conjunction the 30-minute `AdvisoryCompletion`
|
||||
backstop would eventually trust.
|
||||
|
||||
**Layering note:** Arm C's app-install hardening lives in the **adapter**, not the
|
||||
reducer. `ImeLogTrackerAdapter` defers posting `ImeUserSessionCompleted` while required
|
||||
user-ESP Install-intent apps are still pending (IME logs "Completed user session" per
|
||||
*pass*, not once per session — sessions 14690fc2/6cb01530). The reducer stays free of
|
||||
app-pending guards; adapters sanitize evidence before it becomes a signal.
|
||||
|
||||
**Zero user-targeted apps** (sessions 81daa77f/75d6ae8e): IME never writes its
|
||||
"Completed user session" line at all when the user has no assigned apps — its zero-app
|
||||
branch returns first — so fact 3 was unreachable by construction and every such device
|
||||
fell through to the backstop. `IME-USER-SESSION-ZERO-APPS` reads the line IME *does*
|
||||
write there and posts the same signal, guarded to the AccountSetup phase. The event
|
||||
carries an `evidence` field (`zero_user_apps_enumerated` vs `user_session_completed`)
|
||||
naming which observation carried the completion. See
|
||||
[zero-user-apps-completion-gate.md](zero-user-apps-completion-gate.md).
|
||||
|
||||
## Completion attempt sites and sequence
|
||||
|
||||
Completion can be attempted from whichever fact arrives last: `HelloResolved`,
|
||||
`DesktopArrived` (including the Hello-disabled fast-path when `HelloPolicyEnabled ==
|
||||
false` and an arm holds), `EspExiting`, `ImeUserSessionCompleted`,
|
||||
`AccountSetupProvisioningComplete`, the Hello safety deadline, or the 30-minute
|
||||
advisory backstop. All routes converge on `CompleteThroughFinalizingOrDefer`:
|
||||
|
||||
1. **Completion gates** (currently only the RealmJoin AND-gate): if RealmJoin was
|
||||
detected and has no terminal outcome yet, completion is *deferred* — the session
|
||||
stays in stage and emits `completion_waiting`. Three terminals open the gate:
|
||||
`Resolved` (DeploymentPhase 110), `FirstDeploymentIncomplete` (phase left the
|
||||
first-deployment window 100/101 for 200/210 without 110 — the RJ ESP was aborted,
|
||||
typically by an interactive logon reclassifying the run as secondary-user; emits
|
||||
the `realmjoin_first_deployment_incomplete` Warning, session 224b2087), and
|
||||
`Timeout` (60-min deadline from detection). The abort rule reads the *persisted*
|
||||
`RealmJoinFacts.LastDeploymentPhase` (updated by the typed `RealmJoinPhaseChanged`
|
||||
signal), so it survives agent restarts — the `RealmJoinDetected` replay path
|
||||
evaluates it too. A first observation of 210 does **not** release (RJ may have
|
||||
completed before agent boot and deploy afterwards).
|
||||
|
||||
**Activity-based timeout extension** (report 55e6afd61c9d — a large RJ package
|
||||
catalog outlived the 60-min window mid-install): when the timeout deadline fires
|
||||
while the first deployment is demonstrably active — `LastDeploymentPhase` 100/101
|
||||
*and* deployment activity (`RealmJoinPhaseChanged` or a package observation, tracked
|
||||
in `RealmJoinFacts.LastActivityUtc`; the phase captured at detection deliberately
|
||||
does not count) within the last 60 min — the handler re-arms the deadline to
|
||||
`lastActivity + 60 min` instead of timing out, emitting an Info
|
||||
`realmjoin_timeout_extended` event. Re-arms clamp to an absolute cap of
|
||||
**detection + 4 h**. Package activity is only observable at package *completion*
|
||||
(RJ writes the registry subkey when the install finishes), so a single install
|
||||
that is registry-silent for over 60 min still times out — the eventual
|
||||
`realmjoin_timeout` carries a `reason` payload (`hard_timeout` / `inactivity` /
|
||||
`absolute_cap`) telling the three shapes apart. A stale fire of an older deadline
|
||||
incarnation (armed due later than the fire time; `OccurredAtUtc = DueAtUtc` per
|
||||
scheduler contract) dead-ends as `realmjoin_timeout_stale_superseded_by_rearm`.
|
||||
The idle case (detected, no deployment activity ever) times out at 60 min
|
||||
unchanged, and phase 200/210 activity never extends.
|
||||
**Self-update observation** (gktatooine session 946ccbd6): the version on
|
||||
`RealmJoinDetected` is read once from `RealmJoin.exe`, but the tray's `AppUpdater`
|
||||
often replaces the MSI-shipped binary on its first start — minutes after detection,
|
||||
before the deployment begins. `RealmJoinWatcher` therefore arms a
|
||||
`RegNotifyChangeKeyValue` watcher on `HKLM\...\Uninstall\RealmJoin` (the only
|
||||
registry write of the updater; appearance-watch on the `Uninstall` parent when the
|
||||
entry does not exist yet) once Detected fired, re-reads the binary version after a
|
||||
2 s debounce (registry `DisplayVersion` as fallback while the file is locked) and
|
||||
posts the typed `RealmJoinAutoUpdateDetected` signal plus the
|
||||
`realmjoin_autoupdate_detected` timeline event when the version differs from the
|
||||
baseline. The reducer overrides `RealmJoinFacts.ProductVersion` / `ReleaseChannel`
|
||||
(`WithVersionOverride` — the one legitimate later write past the set-once helpers);
|
||||
it never touches the gate, the deadline, or `LastActivityUtc`. An unreadable
|
||||
baseline at detection is seeded silently by the first successful re-read.
|
||||
A closed gate also outranks the 30-minute `AdvisoryCompletion` backstop: that
|
||||
handler re-arms (trigger suffix `:CompletionGateHolding`) instead of failing while
|
||||
the RealmJoin gate is closed *and* its own deadline is still armed, because the gate
|
||||
carries a bounded resolution of its own. Before this, the shorter timer always won —
|
||||
session 75d6ae8e was failed 2 min after its 25th successful RealmJoin package.
|
||||
2. `Finalizing` stage + ~5 s `FinalizingGrace` deadline.
|
||||
3. Grace fires → `Completed`, `EnrollmentComplete` outcome, `enrollment_complete`
|
||||
timeline event with full audit payload.
|
||||
|
||||
**Self-deploying/kiosk** is a separate rail: no Hello, no user desktop —
|
||||
`AwaitingDeviceOnlyEsp` with a detection deadline as the sole terminal entry, protected
|
||||
by stale-fire and race guards. A registry-confirmed self-deploying profile also waives
|
||||
IME false-positive AccountSetup entries.
|
||||
|
||||
# Invariants (violating any of these is a bug)
|
||||
|
||||
- **Post-terminal dispatch guard**: after `Completed`/`Failed`, every signal is a
|
||||
bookkept dead-end — **except `InformationalEvent`** (single-rail pass-through must
|
||||
keep flowing). `WhiteGloveSealed` is unguarded on purpose (Part 2 resumes fresh).
|
||||
- **`InformationalEvent` never mutates decision state** — exactly one pass-through
|
||||
timeline effect, payload copied 1:1.
|
||||
- **`AadUserJoinedLate` is observation-only** — never completion evidence.
|
||||
- **Hello policy is orthogonal to completion** — `HelloPolicyEnabled` tunes wait
|
||||
cadence, never gates the outcome.
|
||||
- **Raw IME "Completed user session" is weak evidence** — can run under `defaultuser0`;
|
||||
only meaningful with a real-user desktop and the AccountSetup anchor (Arm C).
|
||||
- **Deadline bases are floored at `AgentBootUtc`** — replayed backdated signals must
|
||||
not arm past-due timers at boot.
|
||||
- **Reducer-side deadline cancels need an explicit `CancelDeadline` effect** — a
|
||||
builder-only removal leaves the OS timer live.
|
||||
- **No silent parking**: a non-terminal post-AccountSetup session must always hold a
|
||||
resolution-capable deadline; a 10-min dwell tripwire emits
|
||||
`session_parked_without_deadline`.
|
||||
- **Every payload-derived string stored into state is bounded** (`FactStringBounds`,
|
||||
256 chars): RealmJoin package id/version/scope (scope normalised to `machine`/`user`),
|
||||
failed app ids, `HelloOutcome`, `ImeMatchedPatternId`, `EspAdvisoryFailureCategory`,
|
||||
RealmJoin product version/channel. The bound is applied *before* any dedupe comparison
|
||||
so a forged multi-hundred-KB key neither costs a full-length compare per stored row nor
|
||||
lands in snapshots/audit trails. It never truncates real telemetry — the sources are
|
||||
registry key names (Windows caps them at 255), GUIDs and enum-like strings. The same
|
||||
engine replays device-uploaded signals server-side (`ReducerVerifier`), which is why
|
||||
the bound lives in the reducer, not in the agent adapter alone.
|
||||
|
||||
# Hosting & threading (inside the agent)
|
||||
|
||||
Wired by `EnrollmentOrchestrator.Start` (`src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/`):
|
||||
|
||||
- `SignalIngress` — bounded queue (256, back-pressure) + **one** worker thread. Per
|
||||
signal, in order: assign ordinals → `SignalLog.Append` (durable, *before* reducing;
|
||||
append failure ⇒ reducer does not run, no ordinal gap) → `engine.Reduce` →
|
||||
`DecisionStepProcessor.ApplyStep`. A `Post` issued *by the worker thread itself*
|
||||
(the `EffectRunner` posting `ClassifierVerdictIssued` from inside `ApplyStep`) never
|
||||
enters the bounded queue: it goes to a worker-local follow-up queue that is drained,
|
||||
ahead of the channel, right after the current item — the worker is the queue's only
|
||||
consumer, so blocking on a full channel there would freeze the pipeline for the rest
|
||||
of the run.
|
||||
- `DecisionStepProcessor.ApplyStep` (same thread, no locks): journal append (the only
|
||||
hard-throwing path; 3 consecutive failures escalate to quarantine) → effects run
|
||||
synchronously via `EffectRunner` → best-effort snapshot → advance `CurrentState` →
|
||||
one-shot terminal-stage hook (raises the agent's `Terminated` event off-thread).
|
||||
- The `DeadlineScheduler`'s `Fired` event loops back in as synthetic `DeadlineFired`
|
||||
signals.
|
||||
- **Pipeline health in `agent_metrics_snapshot`** (2026-09-01, motivated by the WG-seal
|
||||
soak in which a frozen engine and a deliberate Weak sealing verdict were telemetrically
|
||||
indistinguishable): every snapshot carries monotonic totals from `SignalIngress`
|
||||
(`signal_last_ordinal`, `signal_processed_total`, `signal_pending_count`,
|
||||
`signal_reentrant_total`, `signal_queue_peak` — a monotonic peak, deliberately no
|
||||
momentary queue length, so snapshot sampling cannot alias it) plus `decision_stage`,
|
||||
`decision_last_applied_ordinal` and — once the sealing classifier has produced a
|
||||
verdict — `wg_sealing_level`/`wg_sealing_score` from the immutable `DecisionState`.
|
||||
Two snapshots therefore prove whether the worker ran in between: a frozen worker keeps
|
||||
`signal_pending_count` growing while the processed/applied counters stand still. The
|
||||
probe chain is read-only (`EnrollmentOrchestrator` → `IComponentFactory.
|
||||
CreateCollectorHosts(decisionStateProbe)` → `PeriodicCollectorLifecycleHost` →
|
||||
`AgentSelfMetricsCollector`); the snapshot event itself travels the collector/event
|
||||
path, which is independent of the signal pipeline it measures — field-proven by the
|
||||
freeze cases, whose perf/metrics events kept flowing while the engine stood still.
|
||||
- On restart, `RecoveryCoordinator` folds the persisted signal log through a transient
|
||||
`new DecisionEngine()` before the live pipeline starts — see
|
||||
[logs & persistence](logs-and-persistence.md).
|
||||
@@ -1,191 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Diagnostics Package
|
||||
description: What the agent's diagnostics ZIP contains and why — the built-in section catalog shared with the portal, the RealmJoin and Device Preparation collection gates, the configured-path guards, the handle-validated reads that close the enumerate-then-open junction race, caps, and the package manifest.
|
||||
resource: agent
|
||||
tags: [diagnostics, realmjoin, device-preparation, guardrails, portal, security]
|
||||
timestamp: 2026-08-30
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
The diagnostics package is the agent's last-resort support artifact: a ZIP built in memory
|
||||
by `DiagnosticsPackageService.BuildArchiveBytes` and uploaded through a short-lived SAS
|
||||
URL at the end of a session, at the WhiteGlove seal (`Always` mode) or on a server-requested
|
||||
on-demand collection ([server actions](../backend/server-actions-on-demand-diagnostics.md)).
|
||||
|
||||
## Archive layout
|
||||
|
||||
```
|
||||
sessioninfo.txt session/tenant/device/outcome, hardware via WMI
|
||||
<built-in sections> catalog order, see below
|
||||
AdditionalLogs/<last folder>/ admin-configured paths (global ∪ tenant)
|
||||
package-manifest.txt every packaging decision (always written)
|
||||
_TRUNCATED.txt only when a cap skipped something
|
||||
```
|
||||
|
||||
## The built-in catalog is the single source of truth
|
||||
|
||||
Everything the package collects **before** any configured path is data, not code paths:
|
||||
`DiagnosticsBuiltInSections.All` in Shared (`Models/Diagnostics/DiagnosticsBuiltInSections.cs`).
|
||||
The agent iterates it in order; the backend serves the same list to the portal
|
||||
(`GET /api/diagnostics/paths`, MemberRead) so what administrators see is exactly what the
|
||||
agent does. There is no second copy to drift.
|
||||
|
||||
| Id | ZIP folder | Source (unexpanded) | Patterns | Recursive | Condition |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| AgentLogs | `AgentLogs` | `%ProgramData%\AutopilotMonitor\Logs` | log patterns | no | Always |
|
||||
| ImeLogs | `ImeLogs` | `%ProgramData%\Microsoft\IntuneManagementExtension\Logs` | log patterns | no | Always |
|
||||
| ImeBootstrapperEventLog | `ImeLogs` | `C:\Windows\System32\winevt\Logs` | `BootstrapperAgentServiceLogProvider.evtx` | no | DevicePreparation |
|
||||
| AgentState | `AgentState` | `%ProgramData%\AutopilotMonitor\State` | log patterns + `*.complete`, `*.marker` | yes | Always |
|
||||
| AgentSpool | `AgentSpool` | `%ProgramData%\AutopilotMonitor\Spool` | `*.jsonl`, `*.json` | no | Always |
|
||||
| AgentMarkers | `AgentMarkers` | `%ProgramData%\AutopilotMonitor` | `*.complete`, `*.marker` | no | Always |
|
||||
| RealmJoinWindows | `RealmJoinLogs/Windows` | `C:\Windows\Logs` | `realmjoin*.log` | no | RealmJoinWatcher |
|
||||
| RealmJoinPackages | `RealmJoinLogs/Windows/RealmJoin` | `C:\Windows\Logs\RealmJoin` | `*.log` | yes | RealmJoinWatcher |
|
||||
| RealmJoinChoco | `RealmJoinLogs/Choco` | `%ProgramData%\RealmJoin\choco\logs` | `*.log` | yes | RealmJoinWatcher |
|
||||
| RealmJoinUserTray | `RealmJoinLogs/User` | `%LOGGED_ON_USER_PROFILE%\AppData\Local\RealmJoin` | `tray*.log` | no | RealmJoinWatcher |
|
||||
| RealmJoinUserLogs | `RealmJoinLogs/User/Logs` | `%LOGGED_ON_USER_PROFILE%\AppData\Local\RealmJoin\Logs` | `*.log` | yes | RealmJoinWatcher |
|
||||
|
||||
"Log patterns" = `*.log *.txt *.json *.jsonl *.etl *.evtx *.xml *.csv *.cab`. The five
|
||||
`Always` sections are the historical layout (PR1-B forensics) and are lock-tested in order;
|
||||
the RealmJoin ZIP layout mirrors the disk (`C:\Windows\Logs\{realmjoin*.log, RealmJoin\…}` →
|
||||
`RealmJoinLogs/Windows/…`, `%LOCALAPPDATA%\RealmJoin\{tray*.log, Logs\…}` →
|
||||
`RealmJoinLogs/User/…`), so a flat section and a tree section never produce the same entry
|
||||
name. Two sections may share a ZIP folder only when they read different source folders
|
||||
(`ImeLogs` + the bootstrapper evtx) — `(ZipFolder, SourceFolder)` pairs are unique.
|
||||
|
||||
### Conditions
|
||||
|
||||
`DiagnosticsSectionCondition` is evaluated by the agent (`IsSectionActive`, pure) against its
|
||||
effective configuration and the enrollment scenario; unknown values fail closed.
|
||||
|
||||
* **RealmJoinWatcher** — the tenant's RealmJoin Watcher toggle
|
||||
(`AnalyzerConfiguration.EnableRealmJoinWatcher`). The package service only sees
|
||||
`AgentConfiguration`, so `RemoteConfigMerger` mirrors that one Analyzers knob onto
|
||||
`AgentConfiguration.EnableRealmJoinWatcher` (the RealmJoin host keeps reading the remote
|
||||
block). Deliberately NOT additionally gated on the compile-time `RealmJoinTrackingEnabled`
|
||||
kill switch: if the live watcher is ever disabled, the logs become more useful, not less.
|
||||
No ConfigVersion bump — no new wire field.
|
||||
* **DevicePreparation** — `EnrollmentRegistryDetector.IsDeterministicDevicePreparation()`,
|
||||
the same deterministic `DevicePreparation\BootstrapperAgent\ExecutionContext` marker every
|
||||
other WDP gate keys on ([WDP scenario gates](wdp-scenario-gates.md)); CloudAssigned*
|
||||
fallback rules count as Classic. The probe is a registry read, evaluated once per build
|
||||
(`SCENARIO: devicePreparation=… realmJoinWatcher=…` in the manifest); a probe failure is
|
||||
treated as Classic and logged — the evtx export is skipped, never the package. The channel
|
||||
is exclusively locked while active, so the existing `wevtutil epl` export path
|
||||
(`ResolveChannelForEvtxFile`, field case 2026-08-17) handles it.
|
||||
|
||||
Inactive sections are never silent: `BUILT-IN SKIPPED (RealmJoin Watcher disabled): <Id>`,
|
||||
`BUILT-IN SKIPPED (not a Device Preparation enrollment): <Id>`,
|
||||
`BUILT-IN SKIPPED (no user session for token): <Id> '<source>'` (the
|
||||
`%LOGGED_ON_USER_PROFILE%` token resolves via `UserProfileResolver` only once an
|
||||
interactive user exists — identical to configured paths). Active ones log
|
||||
`BUILT-IN: <Id> -> folder='…' zip='…' recursive=…` followed by the per-pattern
|
||||
`ADDED / NO MATCH / FOLDER MISSING` lines.
|
||||
|
||||
## Built-ins bypass the configured-path guards — on purpose
|
||||
|
||||
`DiagnosticsPathGuards` ([gather-rule guardrails](../rules/gather-rule-guardrails.md))
|
||||
validates **admin-typed** paths: allowlisted prefixes, wildcard only in the last segment,
|
||||
`C:\Users` blocked except the token's `AppData\Local|Roaming`. The catalog is reviewed code,
|
||||
exactly like the five sections that never went through the guard before — and
|
||||
`%ProgramData%\RealmJoin` is not on the allowlist at all. What does apply to every section:
|
||||
the handle-validated read (next section), the `.evtx` channel export, and the caps
|
||||
(100 MB per file, 500 MB total, 5000 files) tracked globally across sections by
|
||||
`BudgetTracker`, with every skip recorded in the manifest and `_TRUNCATED.txt`.
|
||||
|
||||
## Reads are validated on the handle, not the path
|
||||
|
||||
Every guard above judges a path string. Reading the bytes from that string later —
|
||||
`File.GetAttributes` on the candidate, then a `FileStream` open after earlier files have been
|
||||
compressed — leaves a window: under any folder a local user can write to (the user-profile
|
||||
token paths, `ProgramData` subtrees, `C:\Install\Log`) that user can turn a subdirectory into
|
||||
a junction in between. An attribute check on the file resolves through a junction in a
|
||||
*parent* component, and SYSTEM would copy the junction target into the package.
|
||||
`PinnedSourceFolder` (`Monitoring/Runtime/PinnedSourceFolder.cs`, Win32 declarations in
|
||||
`Monitoring/Interop/FileNativeMethods.cs`) closes the window by deciding on the handle that
|
||||
is read:
|
||||
|
||||
* `AddLogFiles` pins the section folder once. It is opened with
|
||||
`FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS`, must not be a reparse point
|
||||
itself, and its `GetFinalPathNameByHandle` result — every reparse point resolved — must
|
||||
equal the validated lexical path (8.3 names expanded through `GetLongPathName`, so a
|
||||
`%TEMP%`-style short name is not mistaken for a junction). Anything else skips the section
|
||||
with `FOLDER REJECTED (<reason>)`; a missing folder stays `FOLDER MISSING`. The handle is
|
||||
held, with `FILE_LIST_DIRECTORY` access and no delete share, until the section is done:
|
||||
NTFS refuses to rename or delete a directory while it or anything beneath it is open, so
|
||||
the validated chain cannot be swapped underneath the build either.
|
||||
* Every candidate file is opened by `PinnedSourceFolder.TryOpenFile` with
|
||||
`FILE_FLAG_OPEN_REPARSE_POINT` — a symlink at the final component is opened as itself and
|
||||
refused (`SKIPPED (reparse point)`) — and the handle's final path must be exactly
|
||||
`<canonical folder>\<relative path>`. A subdirectory that became a junction after
|
||||
enumeration resolves elsewhere: `SKIPPED (resolved outside validated folder)`, naming only
|
||||
the requested path in the manifest, never the resolved one. The length charged against the
|
||||
caps and the stream copied into the archive come from that same handle; any other open
|
||||
failure is `FAILED (open): <path>: <reason>`.
|
||||
* `CollectFilesNoReparseDirs` still skips reparse-point directories while enumerating — as a
|
||||
candidate filter and to keep the walk out of junction loops — but nothing it decides is
|
||||
trusted at copy time.
|
||||
* The `.evtx` channel export is the agent's own file under an unguessable name in SYSTEM's
|
||||
temp folder and is streamed as-is. Hard links are out of scope: Windows refuses to create
|
||||
one to a file the caller cannot write.
|
||||
|
||||
The race itself is pinned by `DiagnosticsPackageServiceTests`: the `BeforeSourceFileOpen`
|
||||
seam swaps a subdirectory for a junction after enumeration and before the open, and the
|
||||
archive must not carry the file.
|
||||
|
||||
## Configured paths: global ∪ tenant
|
||||
|
||||
`GetAgentConfigFunction.MergeDiagnosticsLogPaths` sends global paths (Global Admin,
|
||||
`AdminConfiguration.DiagnosticsGlobalLogPathsJson`) first, then tenant paths
|
||||
(`TenantConfiguration.DiagnosticsLogPathsJson`), dropping blank entries and collapsing
|
||||
duplicates (trimmed, case-insensitive) onto the first occurrence — a path present in both
|
||||
lists used to produce duplicate ZIP entry names. The agent keys each entry's ZIP folder on
|
||||
the last directory name (`AdditionalLogs/<folder>`), so two configured paths ending in a
|
||||
same-named folder share one ZIP folder.
|
||||
|
||||
## Portal surfaces
|
||||
|
||||
* Global Admin → Settings → Diagnostics Log Paths: the built-in catalog (collapsed, read-only,
|
||||
neutral condition pills) above the editable global list; add-row first.
|
||||
* Tenant → Settings → Agent → Diagnostics Package: built-in catalog (header carries the
|
||||
tenant's persisted RealmJoin Watcher state, RealmJoin rows show on/off), the global paths
|
||||
read-only, then the tenant's own entries; every role with a tenant membership can read the
|
||||
catalog (the route is MemberRead, JWT-scoped). One line per entry (`DiagnosticsPathRow`),
|
||||
full values in tooltips.
|
||||
|
||||
## Test seams
|
||||
|
||||
* `DiagnosticsPackageService` internal ctor: the five historical folder overrides map onto
|
||||
catalog ids; `sectionFolderOverrides` covers any other section; `devicePreparationProbe`
|
||||
replaces the registry read (seam callers default to "not WDP", so existing rigs never touch
|
||||
the registry). `UserProfileResolver.SetForTesting(path|null)` / `Reset()` drive the token.
|
||||
`BeforeSourceFileOpen` fires with each candidate path after enumeration and before its open
|
||||
— the point at which the junction-race test swaps the directory.
|
||||
* Catalog invariants live in the backend test project (`DiagnosticsBuiltInSectionsTests`),
|
||||
because Shared has no test project.
|
||||
|
||||
# Examples
|
||||
|
||||
A RealmJoin tenant (watcher on) with a signed-in user: `RealmJoinLogs/Windows/realmjoin.log`,
|
||||
`RealmJoinLogs/Windows/RealmJoin/Packages/<pkg>/<timestamp>_install.log`,
|
||||
`RealmJoinLogs/Choco/<pkg>/<timestamp>_install.log`, `RealmJoinLogs/User/tray.log`,
|
||||
`RealmJoinLogs/User/Logs/RjImeHost.log`. The same tenant's package collected before the
|
||||
desktop: the two `User` sections appear as `BUILT-IN SKIPPED (no user session for token)`.
|
||||
A Classic enrollment: `BUILT-IN SKIPPED (not a Device Preparation enrollment):
|
||||
ImeBootstrapperEventLog`. A tenant that had added
|
||||
`%LOGGED_ON_USER_PROFILE%\AppData\Local\RealmJoin\Logs\*.log` manually keeps working — the
|
||||
files simply appear twice (`AdditionalLogs/Logs/…` and `RealmJoinLogs/User/Logs/…`) until the
|
||||
entry is removed.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/DiagnosticsBuiltInSections.cs` — the catalog
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs` — `BuildArchiveBytes`, `AddBuiltInSection`, `IsSectionActive`, `AddLogFiles` / `AddLogFile` / `OpenSource`, caps, evtx export
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/PinnedSourceFolder.cs` — pinned folder handle, `TryOpenFile`, `SourceFile`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/FileNativeMethods.cs` — `CreateFileW`, `GetFileInformationByHandle`, `GetFinalPathNameByHandleW`, `GetLongPathNameW`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs` — `EnableRealmJoinWatcher` mirror
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Diagnostics/GetDiagnosticsPathsFunction.cs` — `GET diagnostics/paths`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs` — `MergeDiagnosticsLogPaths`
|
||||
* `src/Web/autopilot-monitor-web/components/diagnostics/` — `DiagnosticsPathRow`, `BuiltInSectionsList`, `builtInSectionDisplay`
|
||||
* Tests: `DiagnosticsPackageServiceTests` (RealmJoin on/off, no user session, bootstrapper evtx, probe failure, junction swapped in after enumeration, junction as section folder), `PinnedSourceFolderTests` (final-path equality, short names, junction in the chain, symlink at the final component, rename refused while pinned), `DiagnosticsBuiltInSectionsTests`, `GetDiagnosticsPathsPayloadTests`, `GetAgentConfigFunctionTests.MergeDiagnosticsLogPaths_*`
|
||||
@@ -1,113 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: DO GroupId Auto-Set — Delivery Optimization peering group from a network fingerprint
|
||||
description: EnableDoGroupIdAutoSet (ConfigVersion 40) makes the agent set the DOGroupId policy value to a deterministic GUID derived from the default gateway's IPv4 address and MAC (RealmJoin-compatible byte layout), so devices on the same local network form one DO peering group during enrollment; foreign DOGroupId/DOGroupIdSource policies always win.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/DoGroupIdService.cs
|
||||
tags:
|
||||
- agent
|
||||
- delivery-optimization
|
||||
- network
|
||||
- tenant-configuration
|
||||
timestamp: 2026-08-21T00:00:00+02:00
|
||||
---
|
||||
|
||||
# DO GroupId Auto-Set
|
||||
|
||||
## Schema
|
||||
|
||||
### The problem
|
||||
|
||||
With Delivery Optimization download mode Group (2), peering is scoped by GroupId. When no
|
||||
GroupId is configured, Windows derives it from AD site → authenticated domain SID → Entra
|
||||
tenant ID — for a cloud-native tenant that means **one tenant-wide group**, so "peers on
|
||||
the same network" degrades to "peers anywhere in the tenant" and DO's cloud service has to
|
||||
match them by public IP. A GroupId derived from the local network makes the group exactly
|
||||
"devices behind the same gateway" — including branch offices behind NAT. RealmJoin ships
|
||||
this as its network-fingerprint GroupId; our agent is on the device even earlier (during
|
||||
ESP, when the big app downloads actually happen), so it can seed the same value for
|
||||
Autopilot enrollments.
|
||||
|
||||
### The setting
|
||||
|
||||
`TenantConfiguration.EnableDoGroupIdAutoSet` (`bool?`, default null→false), served as
|
||||
`AgentConfigResponse.EnableDoGroupIdAutoSet` — **ConfigVersion 40**. Portal: Settings →
|
||||
Agent → Settings, toggle "Set Delivery Optimization Group ID", stored via the usual
|
||||
tenant-config roundtrip (`TableConfigRepository` Store+Map). Also shown in the admin
|
||||
tenant-config report. `RemoteConfigMerger` maps the bool 1:1; it survives the offline
|
||||
config cache like every plain feature flag.
|
||||
|
||||
The agent does NOT configure `DODownloadMode` — the fingerprint GroupId only has an
|
||||
effect when the tenant already deploys download mode Group (2) via Intune/GPO.
|
||||
|
||||
### The fingerprint
|
||||
|
||||
`DoGroupIdService.ComputeGroupId(ipv4Bytes, macBytes)` builds a 16-byte GUID:
|
||||
|
||||
```
|
||||
{101,48,6} + {0,0,0} + {ip[1], 0, ip[2], ip[3]} + mac[0..5] → new Guid(byte[16])
|
||||
```
|
||||
|
||||
Gateway discovery reuses the shared `NetworkInterfaceLocator` heuristic (Up, not
|
||||
Loopback/Tunnel, non-0.0.0.0 gateway; IPv4 gateway required), the gateway MAC comes from
|
||||
`ArpNativeMethods` (`iphlpapi!SendARP`). The byte layout — including the IP shuffle and
|
||||
the zero padding — is **RealmJoin-compatible and must never change**: devices managed by
|
||||
RealmJoin on the same network compute the identical GUID and peer with ours. The registry
|
||||
value is written as `Guid.ToString()` (braceless lowercase), identical to RealmJoin's
|
||||
`SetValue(name, Guid)` string form. Deliberate divergence from RealmJoin: **no
|
||||
random-GUID fallback** — a random per-device ID would isolate the device in a one-member
|
||||
group, which is worse than the OS default grouping, so no gateway/no MAC means skip.
|
||||
|
||||
### Ownership + conflict rules (never fight a foreign policy)
|
||||
|
||||
Target: `HKLM\SOFTWARE\Policies\Microsoft\Windows\DeliveryOptimization\DOGroupId`
|
||||
(64-bit registry view — the Policies store is WOW64-redirected; key paths reused from
|
||||
`DoThrottlePolicyReader`). Only this GPO-store value is written; the legacy pre-1607
|
||||
`GroupId` value name is intentionally not.
|
||||
|
||||
Skip conditions (Info event with `skipReason`, nothing written):
|
||||
|
||||
| skipReason | Meaning |
|
||||
|---|---|
|
||||
| `no_ipv4_gateway` | No active interface with an IPv4 default gateway |
|
||||
| `arp_failed` | Gateway MAC could not be resolved via SendARP |
|
||||
| `group_id_source_policy` | `DOGroupIdSource` configured in GPO or MDM store (dynamic group source wins) |
|
||||
| `mdm_policy_present` | `DOGroupId` managed via the Policy CSP store (Intune owns it) |
|
||||
| `foreign_value_present` | Existing GPO value neither matches the computed GUID nor the one we last wrote |
|
||||
|
||||
Ownership is tracked through the `StartupEventGate` fingerprint for the
|
||||
`do_group_id_auto_set` key: `written:<guid>` records the GUID we last wrote, so after a
|
||||
gateway change the agent recognizes its own stale value and updates it — while a value
|
||||
written by anyone else is never overwritten (`DecideAction`: empty→write, equal→no-op,
|
||||
ours→update, else→skip; unparseable existing values count as foreign). Every write is
|
||||
verified by readback.
|
||||
|
||||
### The event
|
||||
|
||||
One `do_group_id_auto_set` event per outcome change (Source `StartupEnvironmentProbes`,
|
||||
gated on the fingerprint — unchanged reruns across reboots stay silent; errors never
|
||||
latch, so each failing start keeps its retry trail). Severity: Info for
|
||||
success/skip, Warning for errors. Data: `groupId`, `previousValue`, `gatewayIp`,
|
||||
`gatewayMac`, `written`, `skipped`, `skipReason`, `success`, `error`.
|
||||
|
||||
## Examples
|
||||
|
||||
First boot on a network with gateway `192.168.178.1` / MAC `00-11-22-33-44-55`:
|
||||
|
||||
```
|
||||
do_group_id_auto_set (Info): Delivery Optimization GroupId set to
|
||||
00063065-0000-00a8-b201-001122334455 (gateway 192.168.178.1)
|
||||
```
|
||||
|
||||
Tenant already deploys a DO GroupId via Intune:
|
||||
|
||||
```
|
||||
do_group_id_auto_set (Info): Delivery Optimization GroupId not set: mdm_policy_present
|
||||
```
|
||||
|
||||
## Citations
|
||||
|
||||
* Service (fingerprint, ownership, write+readback): `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/DoGroupIdService.cs`
|
||||
* ARP interop: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/ArpNativeMethods.cs`
|
||||
* Probe wiring + event + fingerprint dedup: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Runtime/StartupEnvironmentProbes.cs`
|
||||
* Backend serve + ConfigVersion: `../../src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs`
|
||||
* DO policy stores (key paths, Registry64 rationale): `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/DoThrottlePolicyReader.cs`
|
||||
@@ -1,88 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Agent Endpoint Migration — Config-Channel Re-Home
|
||||
description: How the backend re-homes agents to a new API base URL (backend move or per-tenant region move) via MigrateToApiBaseUrl on the agent config response, and why HTTP redirects were rejected.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/EndpointMigration.cs
|
||||
tags:
|
||||
- agent
|
||||
- backend
|
||||
- config
|
||||
- migration
|
||||
- security
|
||||
timestamp: 2026-07-18T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
The agent's API base URL is compiled into the binary (`Constants.ApiBaseUrl`). No DNS/domain
|
||||
indirection is possible in front of the API: mTLS client-cert auth does not survive a reverse
|
||||
proxy (Front Door), and Flex Consumption custom-domain TLS is broken. A backend move (like the
|
||||
2026-07-18 WEU→GWC subscription migration) therefore hard-cuts every deployed agent still
|
||||
pointing at the old hostname. In-flight agents self-terminate safely (local MaxLifetime watchdog
|
||||
+ CleanupService), but their sessions are lost and a per-tenant region move (EU→US) has no
|
||||
mechanism at all.
|
||||
|
||||
# Mechanism
|
||||
|
||||
`AgentConfigResponse.MigrateToApiBaseUrl` — served on `GET /api/agent/config` (incl. the
|
||||
bootstrap wrapper), the same control channel as the kill switch, fetched at every agent
|
||||
process start.
|
||||
|
||||
Agent flow (`Program.RunAgent`, Phase 4):
|
||||
|
||||
1. Config fetched from the compiled-in (old) base URL.
|
||||
2. Kill-switch check runs FIRST — **kill wins over migrate**.
|
||||
3. `AgentRuntimeConfig.Resolve` detects the migration signal directly after the fetch and
|
||||
returns early, BEFORE merge / binary-integrity / bootstrap-cleanup — none of those side
|
||||
effects may run against a config from the backend being abandoned (a migration stub may
|
||||
advertise stale agent hashes; a spurious `IntegrityCheckFailed` + force-update would be
|
||||
harmful).
|
||||
4. `Program.RunAgent` swaps `AgentConfiguration.ApiBaseUrl`, disposes the old client stack,
|
||||
rebuilds Phase 3 (auth clients) and re-runs Phase 4 against the new URL —
|
||||
`allowEndpointMigration: false`, so the new backend's config **cannot chain a second hop**.
|
||||
5. Kill-switch check runs again on the NEW backend's config; startup continues normally
|
||||
(Phase 5+ builds mTLS/telemetry clients from the updated base URL).
|
||||
|
||||
# Hardening (mirrors the kill switch)
|
||||
|
||||
* **Live fetch only** — the on-disk config cache strips `MigrateToApiBaseUrl` on write AND
|
||||
read (`RemoteConfigService.CacheConfig` / `LoadCachedConfig`); `EndpointMigration.ResolveTarget`
|
||||
re-checks `LastFetchOutcome == Succeeded` as defence-in-depth. A planted cache file can
|
||||
never re-home an agent.
|
||||
* **Allowlist validated on BOTH sides** — `AgentEndpointMigrationRules` (Shared, single
|
||||
source): https only, host must end in `.azurewebsites.net` on a label boundary, no
|
||||
path/query/userinfo/port. The backend refuses to serve an invalid admin value
|
||||
(`AgentMigrateRejected` warning); the agent refuses to honour one even if the backend was
|
||||
compromised.
|
||||
* **One hop, no loops** — second Resolve pass never migrates; a target equal to the current
|
||||
base URL is a no-op (steady state during the migration window).
|
||||
|
||||
# Operation
|
||||
|
||||
Global admin settings (`AdminConfiguration`, admin portal → Global Settings):
|
||||
|
||||
* `AgentMigrateApiBaseUrl` — global target, set on the backend being **abandoned**; clear it
|
||||
after the migration window.
|
||||
* `AgentMigrateTenantOverridesJson` — JSON object `tenantId → URL` for per-tenant moves.
|
||||
An entry with an **empty string** pins that tenant against the global target (staged rollout).
|
||||
Overrides win over the global value.
|
||||
|
||||
Delivery evidence: every served target logs `AgentMigrateServed` (Warning — reaches App
|
||||
Insights) with tenant, serial, agent version and target. Agent-side, the re-home is logged as
|
||||
`Endpoint migration: backend re-homed this agent from … to …`.
|
||||
|
||||
Reach limitation: agents that cannot complete an authenticated config fetch (e.g. cert-broken
|
||||
devices) never see the signal — they fall back to the local MaxLifetime/emergency-break
|
||||
lifecycle. During a migration window the old backend must keep running as a serving stub;
|
||||
keeping the old `*.azurewebsites.net` name parked afterwards also prevents hostname
|
||||
re-registration by third parties while stragglers drain.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Services/AgentEndpointMigrationRules.cs` — shared validation rules
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs` — wire field
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs` — `ResolveMigrateTarget` + serving (ConfigVersion 33)
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/EndpointMigration.cs` — agent-side evaluation
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs` — early return before merge/integrity
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2/Program.cs` — one-hop rebuild in `RunAgent`
|
||||
* Tests: `AgentEndpointMigrationTests` (backend), `EndpointMigrationTests` + `RemoteConfigServiceCacheHardeningTests` (agent)
|
||||
@@ -1,151 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ESP Policy-Provider Stall Detection — the no-timeout Setup/Apps wait and the co-management Sidecar gap
|
||||
description: Why the ESP's Apps wait is keyed to the Intune-registered "Sidecar" provider by name — a foreign provider (ConfigMgr) with TrackingPoliciesCreated=1 does not satisfy it — plus the classic never-completes stall, and how the always-on two-arm esp_policy_provider_stalled tripwire (reason=provider_incomplete | sidecar_provider_missing, 15-min wall-clock dwell, StartupEventGate cross-restart dedup) surfaces both.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs
|
||||
tags:
|
||||
- agent
|
||||
- esp
|
||||
- enrollment-status-tracking
|
||||
- co-management
|
||||
- stall-detection
|
||||
timestamp: 2026-07-24T00:00:00+02:00
|
||||
---
|
||||
|
||||
# ESP Policy-Provider Stall Detection
|
||||
|
||||
## Schema
|
||||
|
||||
The Enrollment Status Page decides when its "Apps" category may progress from
|
||||
"Identifying" using the **EnrollmentStatusTracking CSP**
|
||||
(<https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp>),
|
||||
mirrored in the registry under
|
||||
`HKLM\SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking\{Device|<UserSID>}\…`.
|
||||
Two contracts matter:
|
||||
|
||||
1. **`Setup/Apps/PolicyProviders/<Name>`** — per the CSP doc: *"Existence of this node
|
||||
indicates to the ESP that it shouldn't show the tracking status message until the
|
||||
TrackingPoliciesCreated node has been set to true."* This wait has **no timeout**. A
|
||||
registered provider that never sets `TrackingPoliciesCreated=1` parks the ESP at
|
||||
"Apps (Identifying)" indefinitely — observed for days in the field. Exists in device
|
||||
scope (`Device`) and user scope (`S-<SID>`, appears at sign-in).
|
||||
2. **`DevicePreparation/PolicyProviders/<Name>/InstallationState`** — must reach 2
|
||||
(NotRequired) or 3 (Completed); 1=NotInstalled, 4=Error. Device scope only. The ESP
|
||||
applies a default 15-minute timeout here, so these stalls self-surface as ESP errors —
|
||||
the detector still records them for the diagnostic trail.
|
||||
|
||||
Known provider names: **`Sidecar` is the Intune Management Extension** (the CSP doc names
|
||||
"Intune's agents, such as SideCar"); **`ConfigMgr`** is the Configuration Manager client,
|
||||
registered during Autopilot-into-co-management — the troubleshooting section of
|
||||
<https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment> queries
|
||||
exactly these registry paths. The same document states that ESP policy providers **are not
|
||||
aware of each other** and that **pre-provisioning is unsupported** with Autopilot into
|
||||
co-management.
|
||||
|
||||
## The field case (issue #106)
|
||||
|
||||
A tenant pre-provisions devices and installs the ConfigMgr client as a blocking app.
|
||||
Occasionally the user-phase ESP reaches "Apps (Identifying)" and never leaves it — for
|
||||
hours to days. On those devices `Device\Setup\Apps\PolicyProviders` contains **only a
|
||||
`ConfigMgr` key (no `Sidecar`)** — and, per the customer's follow-up, **`TrackingPoliciesCreated=1`
|
||||
IS set under it**. The provider is "complete" by the CSP value contract, yet the ESP
|
||||
still waits. Manually renaming the key to `Sidecar` — which creates no tracking entries,
|
||||
only satisfies the name — unblocks the ESP instantly.
|
||||
|
||||
The naive CSP-doc reading ("any provider with TrackingPoliciesCreated=1 satisfies the
|
||||
wait") is therefore wrong for this case. What the evidence supports instead: provider
|
||||
nodes are registered **by the Intune service at enrollment**, and its ESP-policy WAP
|
||||
payload always registers `Sidecar` (ConfigMgr joins only via a co-management settings
|
||||
policy — see the payload documented by Niehaus). The ESP waits on the provider it
|
||||
expects **by name**; a foreign provider occupying the slot does not satisfy it,
|
||||
regardless of its completion values. On the affected devices the CM client (installed
|
||||
as a plain Win32 app, no co-management settings policy) evidently displaced the
|
||||
Sidecar node — consistent with Microsoft's warning that providers "aren't currently
|
||||
aware of others" and that pre-provisioning is unsupported with co-management. The exact
|
||||
CloudExperienceHost mechanism is not publicly documented; the registry signature —
|
||||
setupApps providers present, `Sidecar` absent — is deterministic either way.
|
||||
|
||||
## Detection
|
||||
|
||||
`EspPolicyProviderProbe` (fail-soft, static, `ScopedOverride` test seam — structural clone
|
||||
of `EspTrackingInfoProbe`) enumerates all providers in both kinds and scopes.
|
||||
`EspPolicyProviderStallDetector` evaluates a pure transition function on a 60-second tick
|
||||
from the always-on `EspPolicyProviderStallHost` — **two arms**, same 15-minute wall-clock
|
||||
dwell, one shared event type discriminated by `reason`:
|
||||
|
||||
* **Arm 1, `reason=provider_incomplete`** — a provider **continuously incomplete for
|
||||
≥ 15 minutes**. The dwell clock starts at first incomplete observation and resets on
|
||||
completion or disappearance. Missing values parse as incomplete
|
||||
(fail-toward-observation). Catches e.g. Sidecar itself never reporting.
|
||||
* **Arm 2, `reason=sidecar_provider_missing`** — **≥ 1 `Setup\Apps` provider registered
|
||||
but none named `Sidecar`**, continuously for ≥ 15 minutes. Deliberately ignores
|
||||
provider completeness (the field case had `TrackingPoliciesCreated=1`). Sidecar
|
||||
registering later — the legitimate startup ordering — clears the condition and resets
|
||||
the clock. DevicePreparation-only snapshots don't activate it.
|
||||
* When both arms cross in the same round, ONE event is emitted and
|
||||
`sidecar_provider_missing` wins as `reason` (it explains the hang regardless of the
|
||||
foreign provider's state).
|
||||
* Missing root key / empty provider list = normal early enrollment: clears all dwell
|
||||
state (both arms), never fires.
|
||||
* One `esp_policy_provider_stalled` **Warning** per newly stalled condition set
|
||||
(`ImmediateUpload`), one-shot per condition in-process, cross-restart deduped via a
|
||||
`StartupEventGate` fingerprint over the sorted currently-active fired conditions
|
||||
(provider keys + a `sidecarMissing` sentinel) — a restart into the same stall stays
|
||||
silent, a new condition joining re-reports.
|
||||
* Payload: `reason`, `providers[]` (full table: name, scope, kind, raw values, complete
|
||||
flag), `stalledProviders[]` with `stalledForMinutes`, `sidecarMissingForMinutes`
|
||||
(arm 2 only), `sidecarRegistered`, `dwellMinutes`.
|
||||
* **Purely observational**: emitted via `InformationalEventPost` (dispatch-guard exempt
|
||||
pass-through) — never touches decision state. No config toggle, no ConfigVersion bump
|
||||
(precedent: `disk_space_low`).
|
||||
|
||||
Deliberately **not** built on `StallProbeCollector` (its idle clock resets on any session
|
||||
activity — a provider stall coexists with a chatty session) and not on the periodic
|
||||
collectors (idle-stopped after 15 minutes — exactly the dormant window the dwell must
|
||||
observe).
|
||||
|
||||
## Guardrail companion
|
||||
|
||||
`rules/guardrails.json` allow-lists `SOFTWARE\Microsoft\Windows\Autopilot` (category
|
||||
"Autopilot / OOBE / Setup") so tenants can build their own gather rules on the subtree —
|
||||
e.g. a `not_exists` registry rule on the `Sidecar` provider key. The subtree carries
|
||||
deployment state only (provider registrations, tracking lists, app IDs, per-user SIDs);
|
||||
sensitivity is at or below existing entries. Enforcement is agent-side (embedded
|
||||
resource), so both the detector and the guardrail change ship with the agent release.
|
||||
|
||||
## Examples
|
||||
|
||||
Customer-case event (abbreviated) — note the provider is "complete" by CSP values, the
|
||||
hang is carried entirely by `reason=sidecar_provider_missing`:
|
||||
|
||||
```json
|
||||
{
|
||||
"eventType": "esp_policy_provider_stalled",
|
||||
"severity": "Warning",
|
||||
"data": {
|
||||
"reason": "sidecar_provider_missing",
|
||||
"dwellMinutes": 15,
|
||||
"sidecarRegistered": false,
|
||||
"sidecarMissingForMinutes": 15.0,
|
||||
"providers": [
|
||||
{ "name": "ConfigMgr", "scope": "device", "kind": "setupApps",
|
||||
"trackingPoliciesCreated": 1, "installationState": null, "complete": true }
|
||||
],
|
||||
"stalledProviders": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`reason=sidecar_provider_missing` is the co-management signature;
|
||||
`reason=provider_incomplete` with `sidecarRegistered=true` covers a stalled Sidecar or
|
||||
third-party provider — the generalized contract violation.
|
||||
|
||||
# Citations
|
||||
|
||||
* EnrollmentStatusTracking CSP — <https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp>
|
||||
* Windows Autopilot into co-management (requirements, provider-conflict warning, troubleshooting registry paths) — <https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment>
|
||||
* Troubleshoot the ESP (models app tracking exclusively via the `Sidecar` subkey) — <https://learn.microsoft.com/en-us/troubleshoot/mem/intune/device-enrollment/understand-troubleshoot-esp>
|
||||
* Niehaus: the Intune ESP-policy WAP payload registering the provider nodes (`Sidecar` always, `ConfigMgr` only with a co-management settings policy) — <https://oofhours.com/2023/08/25/run-an-sccm-task-sequence-during-autopilot/>
|
||||
* Niehaus: ESP tracks arbitrarily named providers when they are properly registered — <https://oofhours.com/2023/09/18/track-anything-using-esp/>
|
||||
* Agent code: `EspPolicyProviderProbe.cs`, `EspPolicyProviderStallDetector.cs`, `EspPolicyProviderStallHost.cs` (SystemSignals / Orchestration.Hosts)
|
||||
* Related: [MDM Reboot Coalescing](mdm-reboot-coalescing.md), [Gather-Rule Guardrails](../rules/gather-rule-guardrails.md)
|
||||
@@ -1,126 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Gather Rule Debug Log — the per-tenant evaluation trace for rules that produce nothing
|
||||
description: EnableGatherRuleDebugLog (ConfigVersion 36) makes the agent write a local trace file explaining every gather-rule evaluation decision — scope skips, missing intervalSeconds, on_change suppression, empty collector results, logparser position/parse-mode/no-match outcomes — so customers can self-diagnose rules that never show up in the timeline.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/GatherRuleDebugLog.cs
|
||||
tags:
|
||||
- agent
|
||||
- gather-rules
|
||||
- diagnostics
|
||||
- tenant-configuration
|
||||
timestamp: 2026-07-28T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Gather Rule Debug Log
|
||||
|
||||
## Schema
|
||||
|
||||
### The problem
|
||||
|
||||
A gather rule that produces no timeline events is indistinguishable from a rule that
|
||||
never ran. The pipeline has many *silent-nothing* exits — before this feature, none of
|
||||
them were visible to the customer:
|
||||
|
||||
| # | Decision point | Where |
|
||||
|---|---|---|
|
||||
| 1 | Rule delivered but `Enabled=false` | `GatherRuleExecutor.UpdateRules` |
|
||||
| 2 | `trigger=interval` without `intervalSeconds` — never runs | `UpdateRules` timer filter |
|
||||
| 3 | Interval timer's first tick comes only after one **full** interval | `UpdateRules` timer creation |
|
||||
| 4 | Out of phase scope (`activePhases` / `activeFromPhase` not latched); incl. `currentPhase=Unknown` before the first phase signal | `IsRuleInScopeLocked` |
|
||||
| 5 | phase_change / phase_exit dedup — once per (rule, phase) | `OnPhaseChanged` |
|
||||
| 6 | Collector returned an **empty** result (e.g. `emitOnlyIfExists=true` and target absent) | `ExecuteRule` |
|
||||
| 7 | `emit_mode=on_change` — result hash unchanged, emit suppressed | `ShouldEmitOnChange` |
|
||||
| 8 | Guard blocked the target (emits `security_warning`, but the rule yields no data) | `GatherRuleContext.EmitSecurityWarning` |
|
||||
| 9 | logparser: no files matched, position tracker says nothing new, regex no-match, plain-text file parsed in default `cmtrace` mode, `maxLines` cap, wildcard capped at 20 files | `LogParserCollector` |
|
||||
| 10 | Rule execution threw (agent log had only the message, no stack) | `ExecuteRule` catch |
|
||||
|
||||
### The setting
|
||||
|
||||
`TenantConfiguration.EnableGatherRuleDebugLog` (`bool?`, default null→false), served to the
|
||||
agent as `AgentConfigResponse.EnableGatherRuleDebugLog` — **ConfigVersion 36**. Portal:
|
||||
Settings → Agent → Settings, toggle "Gather Rule Debug Log", stored via the usual
|
||||
tenant-config roundtrip (`TableConfigRepository` Store+Map). Also shown in the admin
|
||||
tenant-config report.
|
||||
|
||||
`RemoteConfigMerger` projects the bool onto a runtime path (same pattern as
|
||||
`EnableImeMatchLog`): `true` → `Constants.GatherRuleDebugLogPath` =
|
||||
`%ProgramData%\AutopilotMonitor\Logs\gather-rules-debug.log`; `false` → null, unless a
|
||||
`--gather-debug-log <path>` CLI override supplied a non-default path. The field is not
|
||||
security-sensitive and deliberately **survives the offline config cache**.
|
||||
|
||||
### The trace file
|
||||
|
||||
`GatherRuleDebugLog` is an append-only writer held by `GatherRuleExecutor` and exposed
|
||||
to collectors via `GatherRuleContext.DebugLog(ruleId, stage, message)` (null-safe no-op
|
||||
when disabled — zero cost on the normal path). Line format:
|
||||
|
||||
```
|
||||
2026-07-28T09:15:02.123Z | RULE-ID | stage | message
|
||||
```
|
||||
|
||||
Stages: `config` (registration summary per rule, header, zero-rules note), `phase`
|
||||
(enrollment-phase transition marker `phase change: Old -> New`, ruleId `-` — every
|
||||
rule-level line below it reads against the new phase), `trigger` (timer scheduled,
|
||||
phase/event fired, dedup), `scope` (skips with reason incl. current phase), `exec`
|
||||
(execution start), `collector` (null/empty results with likely cause), `guard`
|
||||
(allowlist block), `emit` (event emitted, hash change), `suppress` (on_change
|
||||
suppression with hash prefix + streak), `logparser` (per-file outcome: lines read,
|
||||
position range, match count, parse failures, regex timeouts, mode; plus the
|
||||
`format=text` hint when every line fails CMTrace parsing; plus one line **per regex
|
||||
match** — line number, matched text, capture groups — capped at 10 per file per run),
|
||||
`error` (full exception **with stack trace** — the agent log keeps only the message;
|
||||
an invalid logparser regex is reported here including the offending pattern string).
|
||||
|
||||
Volume control: 10 MB cap with a single `.old` rotation generation (~20 MB worst case).
|
||||
Every write failure is swallowed — tracing must never break gathering. The trace never
|
||||
leaves the device on its own — but because it lives in the agent log folder
|
||||
(`Constants.LogDirectory`), the diagnostics package (`DiagnosticsPackageService`,
|
||||
`*.log` pattern, `AgentLogs/` section) picks it up automatically, so a customer-initiated
|
||||
diagnostics upload hands the trace to support without manual copying.
|
||||
|
||||
### Semantics
|
||||
|
||||
* **No hot-reload** — the flag is consumed once when `DefaultComponentFactory` builds
|
||||
the `GatherRuleExecutorHost` at session start (per-enrollment lifecycle, same as
|
||||
`EnableImeMatchLog`). Enable it in the tenant, then the *next* enrollment writes the
|
||||
trace.
|
||||
* **Zero-rules case is traced too** — flag on + backend delivered no gather rules →
|
||||
a one-shot standalone `config` line, so the file is never just absent.
|
||||
* **`--run-gather-rules` always traces** (operator diagnostic mode; CLI override or
|
||||
default path) and mirrors every line to the console under `--console`.
|
||||
|
||||
## Examples
|
||||
|
||||
A logparser rule with `emit_mode=on_change` that "delivers nothing" becomes:
|
||||
|
||||
```
|
||||
... | LP-01 | config | registered: trigger=interval, collector=logparser, target=C:\...\app.log, unscoped, emitMode=on_change, interval=300s
|
||||
... | LP-01 | trigger | interval timer scheduled every 300s (first run after one full interval — no immediate execution)
|
||||
... | - | phase | phase change: DeviceSetup -> AccountSetup
|
||||
... | LP-01 | exec | executing: collector=logparser, target=C:\...\app.log
|
||||
... | LP-01 | logparser | app.log: read 1000 lines from position 0->81234, matched 0, parseFailures=1000, regexTimeouts=0, mode=cmtrace
|
||||
... | LP-01 | logparser | every line failed CMTrace parsing — if this is a plain-text log, set parameter format=text
|
||||
```
|
||||
|
||||
— the answer ("plain-text log parsed in CMTrace mode") is in the file.
|
||||
|
||||
A matching rule shows *how* it matched (capped at 10 per-match lines per file per run):
|
||||
|
||||
```
|
||||
... | LP-02 | logparser | setupact.log: line 42 matched "Error 0x80070005" — groups: code="0x80070005"
|
||||
... | LP-02 | logparser | setupact.log: read 500 lines from position 0->40123, matched 1, parseFailures=0, regexTimeouts=0, mode=text
|
||||
```
|
||||
|
||||
and a broken regex names itself:
|
||||
|
||||
```
|
||||
... | LP-03 | error | invalid regex pattern '([unclosed' — rule can never match: Invalid pattern '([unclosed' at offset 10. Not enough )'s.
|
||||
```
|
||||
|
||||
## Citations
|
||||
|
||||
* Executor + trace points: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/GatherRuleExecutor.cs`
|
||||
* Writer: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/GatherRuleDebugLog.cs`
|
||||
* Config projection: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs`
|
||||
* Backend serve + ConfigVersion: `../../src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs`
|
||||
* Phase scoping / emit mode background: `../rules/gather-rule-phase-scoping.md`
|
||||
@@ -1,108 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Hello Wizard Un-Skip — vetoing the policy-disabled Hello shortcut
|
||||
description: Why a single "WHfB disabled" policy read is not trusted for completion decisions — the HelloWizardStarted signal (prevention + cure) and the HelloTracker confirmation second read.
|
||||
resource: /src/Shared/AutopilotMonitor.DecisionCore
|
||||
tags:
|
||||
- agent
|
||||
- decision-engine
|
||||
- hello
|
||||
- completion
|
||||
- whfb
|
||||
timestamp: 2026-07-13T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Hello Wizard Un-Skip
|
||||
|
||||
Session `772fe502` (2026-07-13) completed as *Succeeded* while the user was inside the
|
||||
Windows Hello wizard. Cause: the user-scoped WHfB CSP value
|
||||
(`HKLM\SOFTWARE\Microsoft\Policies\PassportForWork\{tenant}\{scope}\Policies\UsePassportForWork`)
|
||||
**flip-flops** during User-ESP while Intune policy sync is still writing. A single read
|
||||
caught "disabled", the engine set `HelloPolicyEnabled=false` (set-once, poll stops), and
|
||||
the arm-C completion synthesized `HelloOutcome="Skipped"` → 5 s `finalizing_grace` →
|
||||
`enrollment_complete` — 230 ms *after* Shell-Core event 62404 (CXID `AADHello`) proved
|
||||
the wizard had launched. The wizard-start was previously an `InformationalEvent` only
|
||||
(timeline pass-through, no state mutation).
|
||||
|
||||
Two independent defense layers fix this:
|
||||
|
||||
# Layer 1 — `DecisionSignalKind.HelloWizardStarted` (engine)
|
||||
|
||||
Shell-Core 62404 with CXID `AADHello`/`NGC` is a **genuine wizard-lifecycle observation**
|
||||
(unlike the unreliable `hello_provisioning_willlaunch` registry snapshot). It is now a
|
||||
dedicated signal: `ShellCoreTracker.HelloWizardStarted` event → coordinator forward
|
||||
(`EspAndHelloTracker`, timestamp mirror) → `EspAndHelloTrackerAdapter` (fire-once) →
|
||||
reducer. The startup backfill also replays 62404 (5-min lookback), so an agent restart
|
||||
mid-wizard does not lose the observation. The reducer records the set-once fact
|
||||
`DecisionState.HelloWizardStartedUtc`.
|
||||
|
||||
**Prevention** — every "may policy-disabled stand in for a Hello resolution?" decision
|
||||
routes through one predicate (`DecisionEngine.Completion.cs`):
|
||||
|
||||
```
|
||||
HelloPolicyDisabledWithoutWizard = HelloPolicyEnabled == false && HelloWizardStartedUtc == null
|
||||
HelloSatisfiedForCompletion = HelloResolvedUtc != null || HelloPolicyDisabledWithoutWizard
|
||||
```
|
||||
|
||||
Once a wizard start is on record, the policy-disabled shortcut is vetoed at all five
|
||||
Skipped-synthesis sites (EspExiting DeferredCompletion, DesktopArrived fast-path,
|
||||
ImeUserSessionCompleted arm-C, AdvisoryCompletion deadline, AccountSetupProvisioningComplete
|
||||
DeferredCompletion) *and* in the `completion_waiting` missing-prerequisites computation.
|
||||
Sessions then take their pessimistic path: promote to `AwaitingHello` + arm HelloSafety
|
||||
(300 s). The AdvisoryCompletion handler needed a dedicated
|
||||
`HelloWizardObservedPromote` branch — its fallthrough is re-arm-or-**fail**, and
|
||||
prevention alone would have failed a live-wizard session there.
|
||||
|
||||
**Cure** — `HandleHelloWizardStartedV1` (`DecisionEngine.Classic.cs`): when the state
|
||||
already carries the engine-synthesized skip, the handler retracts it — cancel
|
||||
`finalizing_grace` (if armed), null `HelloResolvedUtc`/`HelloOutcome`, back to
|
||||
`AwaitingHello`, arm HelloSafety, emit `completion_waiting` (trigger
|
||||
`HelloWizardStarted:UnSkip`). A real `HelloResolved` — or the HelloSafety timeout —
|
||||
then decides the session.
|
||||
|
||||
Two deliberate design points:
|
||||
|
||||
1. **Synthetic-outcome discriminator.** The engine's synthesized outcomes are exact-case
|
||||
`"Skipped"` / `"Timeout"` (constants `SyntheticHelloOutcomeSkipped`/`...Timeout`);
|
||||
the tracker vocabulary is all-lowercase (`completed`, `skipped`, `not_configured`,
|
||||
`timeout`, `wizard_not_started`). The cure only ever retracts the exact-case
|
||||
`"Skipped"` — tracker-posted resolutions and the deliberate HelloSafety `"Timeout"`
|
||||
are never touched.
|
||||
2. **First fact-nulling reducer.** DecisionState facts are otherwise set-once/monotonic.
|
||||
The retraction is legal exactly here because the fact is provably engine-synthesized
|
||||
and provably wrong (the wizard it claimed would never appear is running). The cure is
|
||||
**stage-agnostic** (guarded by the synthetic outcome, not by `Stage==Finalizing`):
|
||||
a synthetic skip can also sit parked behind a closed RealmJoin completion gate, whose
|
||||
release path completes on `HelloResolvedUtc != null` — the same bug through another
|
||||
door.
|
||||
|
||||
# Layer 2 — confirmation second read (HelloTracker)
|
||||
|
||||
`HelloTracker.ApplyPolicyRead` (extracted state machine, `HelloTracker.cs`):
|
||||
|
||||
| Read | Effect |
|
||||
|------|--------|
|
||||
| `enabled` (first) | commit immediately (pessimistic direction; `confirmedReads=1`) |
|
||||
| `disabled` (first) | mark pending only; 10 s poll timer stays alive |
|
||||
| `disabled` while pending | commit disabled (`confirmedReads=2`) |
|
||||
| `enabled` while pending | commit enabled + `flipFlopDetected=true` (the 772fe502 catch) |
|
||||
| `null` while pending | clear pending, keep polling (a vanished value is flip-flop evidence) |
|
||||
|
||||
While pending, `_isPolicyConfigured` stays false, so the Hello wait timers use the
|
||||
longer policy-unknown cadence — the safe direction. The confirmation audit fields ride
|
||||
on the `hello_policy_detected` event `Data`.
|
||||
|
||||
# Residual risk
|
||||
|
||||
A wizard that starts only *after* `Completed` (agent already terminal + shut down) is
|
||||
still not catchable — the dispatch guard dead-ends post-terminal signals by design. The
|
||||
confirmation read shrinks the window for the stale-disabled commit; the wizard signal
|
||||
closes it for every ordering where 62404 fires before the 5 s grace elapses.
|
||||
|
||||
# Citations
|
||||
|
||||
- Reducer handler + prevention predicate: `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Classic.cs` (`HandleHelloWizardStartedV1`), `.../DecisionEngine.Completion.cs` (`HelloSatisfiedForCompletion`), `.../DecisionEngine.Edge.cs` (`HelloWizardObservedPromote`)
|
||||
- Agent signal path: `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ShellCoreTracker.cs` (62404 + backfill), `EspAndHelloTracker.cs`, `SignalAdapters/EspAndHelloTrackerAdapter.cs`
|
||||
- Confirmation read: `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/HelloTracker.cs` (`ApplyPolicyRead`)
|
||||
- Fixtures: `tests/fixtures/signal-kinds/hello-wizard-started-v1.json`, `tests/fixtures/enrollment-sessions/userdriven-hello-wizard-unskip-v1.jsonl`
|
||||
- Related: [Decision Engine](decision-engine.md)
|
||||
@@ -1,141 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: IME Log Tracker — Anchored Patterns and the Bounded Read Loop
|
||||
description: Why every shipped IME log pattern starts with '^', why the on-device matcher reads lines through a byte-bounded reader with a per-line matching budget, and what an operator sees when a hostile log file is tailed — the IME Logs folder is writable by standard users and the SYSTEM tracker must stay linear on whatever lands there.
|
||||
resource: src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime
|
||||
tags: [agent, ime, regex, redos, log-tracker, reliability, security]
|
||||
timestamp: 2026-08-30
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
`ImeLogTracker` tails every file in `%ProgramData%\Microsoft\IntuneManagementExtension\Logs`
|
||||
whose name matches the IME log families and matches each entry against every active pattern of
|
||||
the shipped pack (`rules/ime-log-patterns/*.json` → `rules/dist/ime-log-patterns.json`, embedded
|
||||
in agent and backend, delivered live through the config channel). That folder is writable by a
|
||||
standard user, so the input is untrusted: anyone signed in during enrollment can append lines.
|
||||
|
||||
Before 2026-08-30 three things multiplied:
|
||||
|
||||
* **Unanchored patterns with unbounded lazy gaps.** `\[Win32App\]\[EspManager\] … registered.*?App name: (?<name>.*?)$`
|
||||
re-scans its `.*?` tail at every offset where the literal prefix occurs; repeating the prefix
|
||||
*m* times in one line of *n* chars costs O(m·n). `IME-APP-VERSION` chained two lazy scans
|
||||
(`.*?"DetectedIdentityVersion":\{[^}]*?"NewValue"`) and was quadratic even when anchored.
|
||||
* **Per-pattern timeout, no per-line budget.** Each regex had a 1 s timeout, the timeout was
|
||||
swallowed at Debug, and the loop tried all ~80 active patterns for every line — one crafted
|
||||
line concatenating several trigger prefixes cost several seconds of SYSTEM CPU.
|
||||
* **Unbounded line length.** `StreamReader.ReadLineAsync` materializes the whole line before
|
||||
anyone can look at its size; a multi-hundred-MB line is an OutOfMemory in the poll loop, which
|
||||
retries from the same bookmark every second.
|
||||
|
||||
Genuine IME decision signals (app failures, enforcement states, ESP phases, user-session
|
||||
completion) queue behind that work — a monitoring blackout during the enrollment window.
|
||||
|
||||
# Design
|
||||
|
||||
## Anchored, linear pattern pack
|
||||
|
||||
* **Every shipped pattern starts with `^`.** The tracker compiles with
|
||||
`IgnoreCase | Compiled | Singleline` and no `Multiline`, so `^` is the start of the matched
|
||||
string — `CmTraceLogEntry.Message`, i.e. the text after `<![LOG[` (for multiline entries the
|
||||
assembled entry, still offset 0). .NET treats a leading `^` as a beginning anchor: a
|
||||
non-matching pattern costs O(prefix), never a scan. Evidence for the invariant: across eight
|
||||
real diagnostics sets (~9,000 entries, all six log families) every match of the old pack began
|
||||
at offset 0 — except one class of mid-line hits that were false positives (below).
|
||||
* **No chained unbounded scans.** `IME-APP-VERSION` now spells the report-delta JSON out
|
||||
(`\{(?:"OldValue":[^,}]{0,64},)?"NewValue":"`) instead of scanning `[^}]*?` at every
|
||||
occurrence. The test `ImeLogPatternPackLinearityTests` rejects `[^x]*` shapes outright.
|
||||
* **Raw lines.** A line that fails CMTrace parsing (orphaned continuation lines, lines after a
|
||||
dropped entry) is still matched raw; with `^` it matches only if it *begins* with the pattern.
|
||||
Field data: the unanchored `PS-SCRIPT-CONTEXT` fired 122 times on
|
||||
`[Win32App] SideCarScriptDetectionManager Launch powershell executor in machine session`
|
||||
(Win32 detection scripts in AppWorkload.log), each time overwriting the pending *platform*
|
||||
script's `RunContext`. Anchoring removed that contamination; it is asserted by test.
|
||||
* **Custom patterns are not rewritten.** Tenant-authored patterns (and cached pre-anchor
|
||||
configs) keep their semantics; `CompilePatterns` names the enabled patterns without `^` once
|
||||
at Info so a pass summary's budget breaks can be attributed.
|
||||
|
||||
## Bounded read loop (`CheckLogFilesAsync`)
|
||||
|
||||
* **`BoundedLineReader`** replaces `StreamReader`: byte-oriented, splits on `\n`, strips one
|
||||
`\r`, decodes UTF-8 per line, honours a BOM only at absolute file offset 0. A line beyond
|
||||
`MaxEntryBytes` (32 MB) is never materialized — the reader keeps the capped prefix, discards
|
||||
the rest while scanning for the terminator and flags `LastLineTruncated`. Its `Position` is
|
||||
the exact offset of the next unread byte, so bookmarks are exact (the StreamReader's decoder
|
||||
read-ahead used to lose up to a buffer of lines on a cancelled pass).
|
||||
* **Why 32 MB.** Real logs carry single lines of ~110 KB routinely
|
||||
(`IntuneManagementExtension.log` policy dumps) and one large tenant's `AppWorkload.log`
|
||||
writes a 5.6 MB `Get policies = […]` line that `IME-POLICIES` legitimately captures (app
|
||||
names for the dashboard). A "few KB" cap would drop genuine entries; with anchored patterns
|
||||
the CPU cost of a large line is O(n) for the handful of patterns whose prefix matches, so the
|
||||
cap only has to bound memory. The same constant caps an assembled multiline entry.
|
||||
* **Oversized lines are dropped, not matched.** Captures on a cut line would be wrong. If the
|
||||
capped prefix opens a CMTrace entry without closing it, the existing `skippingDroppedEntry`
|
||||
mechanism skips the entry's continuation lines instead of raw-matching them; a line that
|
||||
opens a *new* entry always ends the skip and is processed itself.
|
||||
* **Per-line matching budget.** `MatchLine` runs all active patterns under a 2 s budget that
|
||||
sums only the time spent inside `Regex.Match` on that line; a pattern that times out is
|
||||
skipped and counted, and once the budget is spent the remaining patterns are skipped for that
|
||||
line (matches already handled stand). No genuine line comes near it — the budget exists for
|
||||
unanchored custom patterns and hostile input. It is deliberately not the wall-clock across
|
||||
the pattern loop: that window also holds the match handlers (signal ingress, match-log
|
||||
append) and scheduler stalls, and session 946ccbd6 (2026-08-30, Hyper-V guest with 5–10 s
|
||||
process-wide stalls, 1 MB log, no line over 5 KB) broke the wall-clock budget twice on
|
||||
genuine lines and skipped their remaining patterns.
|
||||
* **Held-back tails.** An unterminated tail at EOF (a physical line without `\n`, or a
|
||||
multiline entry without `]LOG]!>`) is normally the writer mid-write. Instead of dropping it
|
||||
(and raw-matching its remainder next pass) the bookmark stays at the entry start. The hold
|
||||
lasts until the entry completes or the file has stood still for `HeldTailSettle` (1 s), and
|
||||
only for tails up to 1 MB — a never-closing hostile tail costs a handful of re-reads, then is
|
||||
processed as-is and the bookmark advances.
|
||||
* **One Warning per pass and file.** `oversizedLines`, `regexTimeouts`, `lineBudgetBreaks` and
|
||||
the first skipped pattern ID are logged in a single line after the pass when any is non-zero
|
||||
(fail-soft with a trace); the first timeout per pattern per process additionally names the
|
||||
pattern at Debug. No per-line logging, so a hostile file cannot flood the agent log.
|
||||
* **Deliberately not done: per-file preemption.** The first pass over a legitimate
|
||||
`AppWorkload.log` can be hundreds of MB and runs in one pass by design; after the changes
|
||||
above the work per byte is linear and small, so a writer can only delay processing in
|
||||
proportion to the bytes it writes.
|
||||
|
||||
## Pack hygiene against the decompiles (2026-08-30)
|
||||
|
||||
Every shipped pattern's distinctive phrase was checked in the IME decompiles 1.97.107.0,
|
||||
1.104.102.0 and 1.105.103.0 (`ime-decompiles` repo): no wording changed in any of them. The 11
|
||||
disabled legacy patterns whose literals last existed in IME 1.50 (fleet minimum 1.97) were
|
||||
deleted rather than left as re-enable traps; `IME-DO-TIMEOUT-1/2` now match the exact source
|
||||
literal (`[Win32App DO] DO downloading is not finished within timeout, jobId = …, fileId = …`)
|
||||
instead of two lookaheads; `IME-DO-TEL` has no emitter from 1.104 on and stays only while
|
||||
1.101/1.103 devices are in the fleet (`get_ime_version_history`).
|
||||
|
||||
## Observability (2026-08-30)
|
||||
|
||||
Everything the loop skips is counted (`ImeTrackerHealth`: lines read, entries matched, oversized
|
||||
lines, regex timeouts, budget breaks, held tails, unanchored patterns, files tailed, backlog
|
||||
bytes) and, together with a per-pattern match histogram, persisted in `ImeTrackerState`. The
|
||||
periodic `agent_metrics_snapshot` carries the counters as `ime_*` keys, the first pass that
|
||||
skipped work raises the one-shot `ime_tracker_degraded` Warning (immediate upload), and the
|
||||
termination handler emits `ime_pattern_hits` — the input of the backend's pattern-drift loop
|
||||
([IME Pattern Health](../backend/ime-pattern-health.md)).
|
||||
|
||||
# Verification
|
||||
|
||||
* `ImeLogPatternPackLinearityTests` loads the shipped pack, asserts the `^` invariant, runs four
|
||||
hostile constructions (repeated prefix, the reported `"DetectedIdentityVersion":{x`
|
||||
repetition, long word runs, prefix+keywords with newlines; 512 KB each) against every pattern
|
||||
with the agent's exact regex options and fails on a timeout or > 150 ms (best of three),
|
||||
and re-matches representative real lines with their captures.
|
||||
* `BoundedLineReaderTests`, `ImeLogTrackerReadLoopHardeningTests` (oversized single line,
|
||||
oversized multiline opener, held-back tail, settle, cancelled pass bookmark, single Warning).
|
||||
* Parity: the old and new packs were run over eight real diagnostics sets with the tracker's
|
||||
assembly semantics; the match sets are identical apart from the removed `PS-SCRIPT-CONTEXT`
|
||||
false positives. `--run-ime-matching` (the agent's offline matcher) now assembles multiline
|
||||
entries like the tracker so it reproduces the same result.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.LogProcessing.cs` — `CheckLogFilesAsync`, `MatchLine`, `HoldBackTail`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/BoundedLineReader.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.cs` — `MaxEntryBytes`, `HeldTailSettle`, `CompilePatterns`
|
||||
* `rules/ime-log-patterns/*.json`, `rules/scripts/combine.js`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/ImeLogPatternPackLinearityTests.cs`
|
||||
* [CMTrace Time Resolution](cmtrace-time-resolution.md) — the linear CMTrace line parser this loop feeds
|
||||
@@ -1,69 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: IME Process Watcher — Identity Before Attach, Re-Arm After Exit
|
||||
description: Why ImeProcessWatcher attaches only to a process that runs in session 0 with its image under the IME install root — a process name is not an identity — and why a reported exit re-arms discovery instead of ending it.
|
||||
resource: src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeProcessWatcher.cs
|
||||
tags: [agent, ime, process, security, cwe-807, reliability]
|
||||
timestamp: 2026-08-30
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
`ImeProcessWatcher` reports `ime_process_exited` (Warning — read as an IME crash indicator in the
|
||||
timeline) by subscribing to `Process.Exited` of `IntuneManagementExtension.exe`. Until 2026-08-30 it
|
||||
attached to `Process.GetProcessesByName(...)[0]` and stopped discovery for good once attached.
|
||||
|
||||
A process name is not a protected namespace. During AccountSetup (and on Device Preparation flows)
|
||||
a standard-user session is live and can start any binary renamed to `IntuneManagementExtension.exe`.
|
||||
Two consequences (CWE-807 — the same class as the agent instance guard and the local-admin dynamic
|
||||
allowance):
|
||||
|
||||
* **Forged signal** — the fake exits, the SYSTEM agent emits a Warning with the fake's PID/exit
|
||||
code, and failure diagnosis is misdirected.
|
||||
* **Muted signal** — attached to the fake, discovery is stopped; the real service is never watched,
|
||||
and a genuine IME crash later in the enrollment is never reported. Even without an attacker the
|
||||
one-shot design missed every IME exit after the first (an IME service restart is not rare).
|
||||
|
||||
# Schema
|
||||
|
||||
`ImeProcessIdentity` establishes identity by facts a standard user cannot forge:
|
||||
|
||||
| Check | Source | Why it holds |
|
||||
|---|---|---|
|
||||
| `SessionId == 0` | `Process.SessionId` | Only services and their children run in session 0; a user session cannot start a process there. |
|
||||
| image under `%ProgramFiles(x86)%\Microsoft Intune Management Extension\` (also `%ProgramW6432%`/`%ProgramFiles%` variants) | `QueryFullProcessImageName` (cross-bitness — IME is 32-bit, the agent may be 64-bit; `MainModule` throws there), `MainModule.FileName` fallback | Program Files is admin-writable only. Segment-aware prefix (`...Extension\`), `GetFullPath`-normalised, so a sibling folder or `..` does not pass. |
|
||||
| file name exactly `IntuneManagementExtension.exe` | image path | `GetProcessesByName` matches the name without extension. |
|
||||
|
||||
Unknown facts (session `-1`, unresolved path) are **untrusted** — a false negative only delays
|
||||
attach by one 5 s discovery tick, a false positive mutes the signal for the session.
|
||||
|
||||
Among trusted candidates the **oldest** by `StartTime` wins (the SCM-started service predates
|
||||
anything transient). Untrusted name matches are logged once per PID as a Warning in the agent log
|
||||
(`ignoring untrusted IntuneManagementExtension.exe candidate (PID, session, image)`) and never
|
||||
attached; no timeline event is raised for them.
|
||||
|
||||
**Re-arm:** `OnImeExited` detaches, emits the event, and restarts the discovery timer. Only the
|
||||
currently attached `Process` object is honoured (`ReferenceEquals`), so a late `Exited` from a
|
||||
replaced or disposed handle cannot produce a duplicate event. A process that exits between probe
|
||||
and subscription is handled by an explicit `HasExited` check after subscribing.
|
||||
|
||||
The watcher owns the `Process` handles its source returns (like `GetProcessesByName`) and disposes
|
||||
them — tests must hand it fresh handles (`Process.GetProcessById`), not the ones they hold.
|
||||
|
||||
# Examples
|
||||
|
||||
* Fake started in session 1 from `Downloads\IntuneManagementExtension.exe` before the service is up →
|
||||
never attached; its exit emits nothing; the service arriving later is attached normally.
|
||||
* Service exits (crash or restart) → `ime_process_exited` (PID, exit code, uptime), discovery
|
||||
re-armed, the restarted service is attached on the next tick and its later exit is reported too.
|
||||
* Fake and service both running, fake listed first → the service is attached.
|
||||
|
||||
Covered by `ImeProcessWatcherIdentityTests` (pure identity table + stand-in `cmd.exe` processes
|
||||
with injected identity facts).
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeProcessIdentity.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeProcessWatcher.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/ImeProcessWatcherIdentityTests.cs`
|
||||
* Related: [Local Admin Analyzer](local-admin-analyzer.md) (CWE-807 dynamic allowance).
|
||||
@@ -1,17 +0,0 @@
|
||||
# V2 Agent
|
||||
|
||||
How the on-device agent works:
|
||||
|
||||
* [Runtime Overview](overview.md) - lifecycle (install → boot guards → runtime → termination), collector hosts, the single-rail event pipeline to the backend, config & kill switch, on-disk layout.
|
||||
* [Decision Engine](decision-engine.md) - the pure-reducer state machine (DecisionCore): signals, state, effects, the A/B/C completion arms, invariants, threading.
|
||||
* [Logs, Persistence & Crash Recovery](logs-and-persistence.md) - every persisted file and why it exists; signal log vs spool; snapshot/journal; the replay-based recovery flow and quarantine.
|
||||
* [Diagnostics Package](diagnostics-package.md) - what the diagnostics ZIP contains: the built-in section catalog shared with the portal (Shared `DiagnosticsBuiltInSections`, served by GET diagnostics/paths), the RealmJoin Watcher and Device Preparation collection gates, why built-ins bypass the configured-path guards, global ∪ tenant path dedup, caps and the package manifest.
|
||||
* [Hello Wizard Un-Skip](hello-wizard-unskip.md) - why a single "WHfB disabled" policy read never decides completion: the HelloWizardStarted signal (prevention + cure) and the HelloTracker confirmation second read.
|
||||
* [Zero User-Targeted Apps](zero-user-apps-completion-gate.md) - why a user with no targeted Intune apps starved every user-phase completion path (IME never writes its "Completed user session" line in the zero-app branch), and why a completion gate that is still holding the session must outrank the 30-minute backstop.
|
||||
* [Autopilot ZTD Diagnostics](autopilot-ztd-diagnostics.md) - Windows' own diagnostic surfaces for the profile-download flow: ModernDeployment event IDs (807/815/164/...), the Diagnostics\Autopilot registry key, deployment-service endpoints, and the known-issue error-code map.
|
||||
* [MDM Reboot Coalescing](mdm-reboot-coalescing.md) - attributing the mid-ESP coalesced reboot ("second sign-in") to the device-assigned policy URIs that forced it: aggregated 2800 observations (neutral, flush-time watermark) + ANALYZE-ESP-005 gated on an actually-observed reboot.
|
||||
* [Power-State Watcher](power-state-watcher.md) - live AC↔battery transitions and 50/30/15% downward threshold crossings via a WMI push subscription (no polling): power_state_change semantics (latch/debounce/cap, never armed without a battery) and the single-condition analyze rules ANALYZE-DEV-009/-010 on top.
|
||||
* [DO GroupId Auto-Set](do-group-id-auto-set.md) - the optional Delivery Optimization peering-group seed: deterministic GUID from default gateway IPv4 + MAC (RealmJoin-compatible), written to the DOGroupId policy value with readback, ownership-tracked so foreign Intune/GPO values are never overwritten; skip reasons and the do_group_id_auto_set event.
|
||||
* [System Timeline Watcher](system-timeline-watcher.md) - clock steps (system_clock_changed: oldTime/newTime/timeDeltaMs/process, micro-slew floor) and completed sleep episodes (system_sleep_episode: sleep/hibernate/Modern Standby with authoritative enteredAt/exitedAt) from one System-channel EventLogWatcher with pre-agent backfill; feeds the web clock-set jump cause, the standby duration note, time-attribution SleepSpans and ANALYZE-DEV-012.
|
||||
|
||||
Reading order for newcomers: overview → decision engine → persistence.
|
||||
@@ -1,139 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Local Admin Analyzer — account inventory, dormant accounts and Administrators membership
|
||||
description: What the local_admin_analysis event asserts about a device at enrollment start and completion, why disabled accounts are inventoried (a /active:no backdoor re-enabled after enrollment is dormant at both scans), how Administrators-group membership is read from the SAM by well-known SID (NetLocalGroupGetMembers, independent of WMI and account state), why built-ins are matched by well-known RID (localized Guest/Administrator), why the shutdown-time dynamic allowance never covers a local SAM account (CWE-807 — the login state is attacker-controlled), which folders are not profiles, which members are scored versus only listed, and why the built-in Administrator's enabled state is reported but not scored.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Analyzers/LocalAdminAnalyzer.cs
|
||||
tags:
|
||||
- agent
|
||||
- security
|
||||
- local-admin
|
||||
- analyze-rules
|
||||
- bypass-detection
|
||||
timestamp: 2026-08-30T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Local Admin Analyzer
|
||||
|
||||
`LocalAdminAnalyzer` emits one `local_admin_analysis` event at agent startup and one at
|
||||
shutdown. The pair is a before/after inventory of local accounts on the device; the backend
|
||||
rule `ANALYZE-ID-002` alerts when the finding is `unexpected_local_admins_detected`.
|
||||
The threat model is the classic Autopilot bypass: a person with a Shift+F10 SYSTEM console
|
||||
during OOBE/ESP who creates a persistent local administrator.
|
||||
|
||||
# Schema
|
||||
|
||||
## Checks
|
||||
|
||||
| Check | Source | What it yields |
|
||||
|---|---|---|
|
||||
| `bypass_nro` | `HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\OOBE\BypassNRO` | value + `flagged` |
|
||||
| Local accounts | WMI `Win32_UserAccount WHERE LocalAccount = True` (Name, Disabled, SID) | `accounts_checked`, `account_details` (name / disabled / administrators_member), `unexpected_accounts`, `builtin_administrator_enabled` |
|
||||
| Administrators membership | `NetLocalGroupGetMembers` level 2 on the group resolved from `S-1-5-32-544` (`LocalGroupNativeMethods`) | `administrators_group.{enumerated,error_code,members}`, `unexpected_admin_members` |
|
||||
| Profile folders | `C:\Users` top-level directories, junctions and `$<32 hex>` folders skipped | `profiles_found`, `unexpected_profiles` |
|
||||
|
||||
## Allowed list
|
||||
|
||||
Built-in names (`Administrator`, `Guest`, `DefaultAccount`, `WDAGUtilityAccount`,
|
||||
`defaultuser0/1/2`, `kioskUser0` — the Assigned Access autologon account of kiosk profiles —
|
||||
and the non-account folders `Public`, `Default`, `Default User`, `All Users`) plus the
|
||||
tenant's *Allowed Local Accounts* (glob `*`/`?`, case-insensitive).
|
||||
`defaultuser1/2` stay allowlisted deliberately: they do appear during OOBE enrollment.
|
||||
|
||||
**Well-known RIDs.** Built-in accounts are additionally recognised by RID on the machine SID
|
||||
(`S-1-5-21-…-500` Administrator, `-501` Guest, `-503` DefaultAccount, `-504`
|
||||
WDAGUtilityAccount), for the WMI inventory and for the SAM-member fallback alike. Localized
|
||||
installs name them `Gast`, `Invité`, `Гость`, `Administrateur`, … — disabled by default and
|
||||
therefore invisible before disabled accounts were inventoried; without the RID match every
|
||||
non-English device would flag its own Guest. Name matching is unchanged (an ordinary-RID
|
||||
account named `Gast` is still unexpected).
|
||||
|
||||
## Dynamic allowance at shutdown
|
||||
|
||||
The shutdown run adds the logged-in users (explorer.exe owners outside session 0) to the
|
||||
allowed list — but **only names that are not a local SAM account**. The WMI inventory is read
|
||||
first; a logged-in name found in it is rejected for both the account and the profile check
|
||||
and reported as `logged_in_local_accounts` (Warning in the agent log).
|
||||
|
||||
Why this is safe for false positives: in real sessions the logged-in identity is an Entra or
|
||||
domain user (user-driven, hybrid, or an Entra user signing in to a self-deploying device
|
||||
after ESP). Those never appear in `Win32_UserAccount`; the allowance only ever mattered for
|
||||
their `C:\Users` folder — and `unexpected_profiles` alone scores 0. A logged-in *local*
|
||||
account is the one case the allowance changed the score, and that is the attack: an account
|
||||
created after the startup scan that signs itself in before the shutdown scan (CWE-807 —
|
||||
exempting on a condition the adversary controls). Kiosk autologon (`kioskUser0`) is covered
|
||||
by the built-in list instead.
|
||||
|
||||
## Profile folders that are not profiles
|
||||
|
||||
Junctions (`All Users`, `Default User` and localized variants such as `Все пользователи`,
|
||||
all pointing at ProgramData / Default) and `$<32 hex>` folders are skipped entirely — neither
|
||||
can be the profile of a local account (a SAM account name is at most 20 characters), so they
|
||||
can never contribute to the profile-overlap score; listing them was payload noise.
|
||||
|
||||
## Evaluation (`EvaluateAccounts`, pure)
|
||||
|
||||
* Every local account is evaluated, **disabled ones included**; the disabled state is reported
|
||||
per account instead of excluding the account.
|
||||
* An account is an Administrators member when its SID matches a group member, or — SID
|
||||
unavailable — its name matches a member of the local machine domain.
|
||||
* `unexpected_accounts` = accounts not matching the allowed list (any state).
|
||||
* `unexpected_admin_members` = unexpected accounts holding Administrators membership, plus
|
||||
local-domain members not on the allowed list that WMI did not return at all (WMI failure
|
||||
or provider gap — the SAM is authoritative for membership).
|
||||
* Members outside the machine domain (Entra role SIDs such as `S-1-12-1-…` on Entra-joined
|
||||
devices, domain groups on hybrid devices, unresolved/deleted SIDs) are **listed for delta
|
||||
comparison but never scored** — they are expected on joined devices.
|
||||
* `builtin_administrator_enabled` = `!Disabled` of the RID-500 account (null when absent).
|
||||
Reported, not scored: whether OOBE enables it transiently is not established; the field
|
||||
collects evidence first.
|
||||
|
||||
## Scoring
|
||||
|
||||
| Signal | Weight |
|
||||
|---|---|
|
||||
| `BypassNRO = 1` | +20 |
|
||||
| any `unexpected_accounts` | +40 |
|
||||
| any `unexpected_admin_members` | +40 |
|
||||
| unexpected account with a matching `C:\Users` folder | +40 |
|
||||
|
||||
Capped at 100. `0` → Info `no_unexpected_admins_detected`; `<40` → Info
|
||||
`bypass_nro_flag_only`; `40–79` → Warning, `≥80` → Error, both
|
||||
`unexpected_local_admins_detected`. A dormant backdoor with admin membership therefore lands
|
||||
at 80 (Error) without ever having a profile folder.
|
||||
|
||||
## Failure behaviour
|
||||
|
||||
WMI and the SAM read are independent; either failing is logged at Warning with the native
|
||||
status (`NET_API_STATUS=…`) and the other source still evaluates. The event carries
|
||||
`administrators_group.enumerated=false` + `error_code` so a "no members" result is never
|
||||
mistaken for an empty group.
|
||||
|
||||
# Examples
|
||||
|
||||
`net user backdoor P@ss /add /active:no` + `net localgroup Administrators backdoor /add`
|
||||
during OOBE, re-enabled by a scheduled task after enrollment: at both scans the account is
|
||||
disabled; before this change it was skipped entirely. Now `unexpected_accounts=[backdoor]`,
|
||||
`unexpected_admin_members=[backdoor]`, `account_details[backdoor].disabled=true` →
|
||||
confidence 80, Error.
|
||||
|
||||
`net user backdoor P@ss /add` + `net localgroup Administrators backdoor /add` during ESP,
|
||||
then an interactive sign-in as `backdoor` (fast user switching) before enrollment completes:
|
||||
the shutdown scan finds `backdoor` in the SAM inventory → not allowed dynamically →
|
||||
`unexpected_accounts=[backdoor]`, `unexpected_admin_members=[backdoor]`, profile overlap →
|
||||
confidence 100, Error, `logged_in_local_accounts=[backdoor]`. Before this change the login
|
||||
alone silenced the finding to 0.
|
||||
|
||||
German device, tenant allowed list empty: `Gast` (RID 501, disabled) and `Administrator`
|
||||
are inventoried and matched by RID → `unexpected_accounts=[]`, confidence 0.
|
||||
|
||||
`net localgroup Administrators adm-helpdesk /add` where `adm-*` is allowlisted: not flagged
|
||||
(the allowed list is the tenant's statement of intent), but `administrators_group.members`
|
||||
shows the new membership in the shutdown event for delta review.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Analyzers/LocalAdminAnalyzer.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/LocalGroupNativeMethods.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Analyzers/LocalAdminAnalyzerEvaluateAccountsTests.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Analyzers/LocalAdminAnalyzerDynamicAllowanceTests.cs`
|
||||
* `rules/analyze/ANALYZE-ID-002.json`
|
||||
@@ -1,141 +0,0 @@
|
||||
---
|
||||
type: Reference
|
||||
title: V2 Agent — Logs, Persistence & Crash Recovery
|
||||
description: Every file the agent persists under %ProgramData%\AutopilotMonitor, why each exists (especially the signal log), and the exact recovery/replay flow after a crash or reboot.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/RecoveryCoordinator.cs
|
||||
tags:
|
||||
- agent
|
||||
- persistence
|
||||
- signal-log
|
||||
- recovery
|
||||
- durability
|
||||
timestamp: 2026-07-11T00:00:00+02:00
|
||||
---
|
||||
|
||||
# V2 Agent — Logs, Persistence & Crash Recovery
|
||||
|
||||
An enrollment spans multiple reboots and can be killed at any moment (app-forced
|
||||
restart, self-update, crash, power loss). The agent therefore persists three distinct
|
||||
tiers of state — each answers a different question:
|
||||
|
||||
1. **Decision truth** (`State\`): *what did the decision engine see and decide?*
|
||||
Signal log + journal + snapshot. Must be deterministic and replayable.
|
||||
2. **Transport** (`Spool\`): *what telemetry still has to reach the backend?*
|
||||
Durable upload queue + cursor. Best-effort; the backend dedups.
|
||||
3. **Diagnostics** (`Logs\`, `Crashes\`): *what happened, for humans?* Trace log and
|
||||
crash dumps. Never allowed to affect decisions.
|
||||
|
||||
Plus small markers (session identity, completion, quarantine) that steer the next
|
||||
process start.
|
||||
|
||||
# Why a signal log AND an event spool?
|
||||
|
||||
They look similar (both JSONL) but serve opposite masters:
|
||||
|
||||
- The **signal log** is the *input* truth of the decision engine. Recovery rebuilds
|
||||
decision state by replaying it; the snapshot is only a cache. An append failure here
|
||||
**blocks the reducer** — a signal that was never durably logged must not change state,
|
||||
or a post-crash replay would diverge from what the live run decided (determinism L.1/L.5).
|
||||
- The **spool** is a *projection* for upload. Writing it is best-effort — a spool
|
||||
failure must never break ingress, and a lost non-critical item is acceptable because
|
||||
the backend dedups on `(PartitionKey, RowKey)`.
|
||||
|
||||
Same data, two files, because durability requirements differ: the signal log fsyncs
|
||||
every line; the spool fsyncs only items flagged `RequiresImmediateFlush`.
|
||||
|
||||
# Artifact inventory
|
||||
|
||||
| Artifact | Path | Format | fsync | Rotation/cap |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Signal log | `State\signal-log.jsonl` | JSONL (`DecisionSignal`/line) | every append (WriteThrough + Flush, bufferSize=1) | none (per-enrollment lifecycle) |
|
||||
| Journal | `State\journal.jsonl` | JSONL (`DecisionTransition`/line) | every append | atomic truncate only (phantom tail) |
|
||||
| Snapshot | `State\snapshot.json` | JSON + SHA-256 checksum envelope | temp-write + `File.Replace` | single file |
|
||||
| Event sequence | `State\event-sequence.json` | `{ LastAssignedSequence }` | atomic | none; survives WhiteGlove Part-2 resume |
|
||||
| Spool | `Spool\spool.jsonl` | JSONL (`TelemetryItem`/line) | only `RequiresImmediateFlush` items | none — never truncated, cursor skips uploaded items |
|
||||
| Upload cursor | `Spool\upload-cursor.json` | `{ LastUploadedItemId }` | atomic | idempotent, never regresses |
|
||||
| Agent trace log | `Logs\agent.log` (segments `agent-NNN.log`) | text `[utc] [LEVEL] msg` | no | 50 MB → suffix rotation; consumed by `/diag-agent-log` |
|
||||
| Crash dump + record | `Crashes\<stem>.dmp` + `.json` | MiniDump + JSON | n/a (process dying) | 5 files / 7 days |
|
||||
| Startup-event gate | `State\startup-event-state.json` | JSON dict | no (atomic) | cross-reboot dedup of one-shot probes |
|
||||
| Bandwidth state | `State\bandwidth-state.json` | JSON samples/counters | no (atomic) | reboot-survivor for the DO bandwidth estimate |
|
||||
| Session identity | `session.id`, `session.created`, `whiteglove.complete` (root) | text | no | drives emergency-break age check + WG resume |
|
||||
| Markers | `clean-exit.marker` (root), `enrollment-complete.marker` (State), `quarantine-requested.marker` (State) | text | no | consumed at next start |
|
||||
| Config cache | `Config\remote-config.json` | JSON | no | security-sensitive fields stripped |
|
||||
|
||||
Per-collector state files (`OfficeInstallStatePersistence`, IME tracker state, script
|
||||
execution state, log-file positions) follow the same fail-soft atomic-replace pattern.
|
||||
Cleanup (`CleanupService`) removes the agent dirs on self-destruct but **keeps `Logs\`**
|
||||
for forensics.
|
||||
|
||||
# The decision-truth triad
|
||||
|
||||
## Signal log — authoritative input
|
||||
|
||||
- Written by the single ingress worker, one line per accepted signal, **before** the
|
||||
reducer runs. Monotonic-ordinal guard on append.
|
||||
- `bufferSize: 1` is load-bearing: session `b9b92d89` (2026-07-09) was killed mid-append
|
||||
by a self-update `Environment.Exit`; the default 4096-byte buffer flushed the same
|
||||
buffered line twice → byte-identical duplicate → non-monotonic ordinal → **recovery
|
||||
crash-loop**. Fixes (commits `57e4334b` + `a0238a4b`):
|
||||
- unbuffered append (nothing left for `Dispose` to re-flush),
|
||||
- read-side tolerance: a byte-identical *consecutive* duplicate line is skipped as a
|
||||
known crash artifact,
|
||||
- self-update restarts now route through graceful shutdown (full drain, bounded 15 s
|
||||
hard-exit fallback) instead of `Environment.Exit`,
|
||||
- poison-replay quarantine (below) so a corrupt log can never loop the agent again.
|
||||
- Corrupt-tail rule: readers stop at the first unparsable line (a torn final write is
|
||||
expected, not fatal).
|
||||
|
||||
## Journal — authoritative applied-transition record
|
||||
|
||||
One `DecisionTransition` per reducer step, appended in `ApplyStep` step 1 — the **only**
|
||||
persistence path that hard-throws. Three consecutive append failures escalate to a
|
||||
quarantine request. Used during recovery to detect BEHIND/AHEAD crash windows against
|
||||
the snapshot; AHEAD phantom entries are truncated to `.quarantine\<ts>\journal-phantom-tail.jsonl`.
|
||||
|
||||
## Snapshot — recovery cache, never truth
|
||||
|
||||
Full `DecisionState` (schema **v4**) in a checksum envelope, written best-effort after
|
||||
meaningful steps (stage transition / non-telemetry effect) or every 100 pass-through
|
||||
steps. Atomic temp-write + `File.Replace`; a crash mid-save leaves the old file intact.
|
||||
Load returns `null` on any doubt (missing, IO error, checksum mismatch) — recovery then
|
||||
falls back to full signal-log replay. Aborted/phantom states are never snapshotted.
|
||||
|
||||
# Recovery flow on restart
|
||||
|
||||
`RecoveryCoordinator.Recover` runs before the live pipeline; order is load-bearing:
|
||||
|
||||
1. **Death-rattle peek**: if the previous exit was unclean, read the prior snapshot
|
||||
*non-mutatingly* (`TryReadRaw`) for the `previous_crash_detected` context — before
|
||||
the pipeline may quarantine it.
|
||||
2. **Quarantine marker** (`quarantine-requested.marker`, written mid-run when the
|
||||
journal became unappendable): quarantine all segments + snapshot *now*, at a moment
|
||||
when no writer holds them. Marker is deleted only on full success.
|
||||
3. **WhiteGlove Part-2 resume**: prior stage `WhiteGloveSealed` → archive the reducer
|
||||
segments to `.part1-<ts>\` but **preserve `event-sequence.json`** (the Web UI splits
|
||||
Part 1/2 on the sequence).
|
||||
4. **Open writers** — each scans its file for the last ordinal/step, stopping at the
|
||||
first unparsable line.
|
||||
5. **Pick seed + replay stream**:
|
||||
- snapshot loads → seed from it, re-stamp `AgentBootUtc` to *now* (replayed signals
|
||||
must not arm past-due deadlines), replay only the signal-log **tail**
|
||||
(`ordinal > snapshot.LastAppliedSignalOrdinal`);
|
||||
- snapshot corrupt → quarantine snapshot only, replay the full signal log;
|
||||
- log head-corrupt → total-loss quarantine of everything, fresh Initial seed.
|
||||
6. **Journal alignment**: truncate AHEAD phantoms to the seed's `StepIndex`.
|
||||
7. **Replay + backfill**: fold the stream through a transient `DecisionEngine`,
|
||||
re-appending any missing journal entries (closes the BEHIND window).
|
||||
8. **Poison guard**: if replay itself throws, quarantine snapshot + all segments and
|
||||
reseed Initial — the agent keeps running with fresh state instead of crash-looping.
|
||||
|
||||
**Duplicate avoidance, end to end**: monotonic append guards (signal log + journal) →
|
||||
consecutive-duplicate skip on read → tail-replay past the snapshot ordinal → AHEAD
|
||||
truncation → backend `(PartitionKey, RowKey)` dedup for re-uploaded spool items.
|
||||
|
||||
**Quarantine semantics**: files are *moved*, never deleted, into
|
||||
`State\.quarantine\<utc-ts>\` with a `reason.txt` sidecar — evidence survives for
|
||||
diagnostics ZIPs while the agent restarts from clean state.
|
||||
|
||||
# Related
|
||||
|
||||
- Pipeline that produces these files: [runtime overview](overview.md)
|
||||
- What replay must reproduce deterministically: [decision engine](decision-engine.md)
|
||||
@@ -1,100 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: MDM Reboot Coalescing — attributing the mid-ESP reboot and second sign-in to policy URIs
|
||||
description: How the agent observes reboot-required MDM policy flags (DM-Enterprise EventID 2800, aggregated) and how ANALYZE-ESP-005 attributes an actually-observed enrollment reboot to them — including why the raw events stay neutral and why the Shell-Core RebootCoalescing match was removed.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals
|
||||
tags:
|
||||
- agent
|
||||
- esp
|
||||
- reboot
|
||||
- policy-attribution
|
||||
- second-sign-in
|
||||
timestamp: 2026-07-20T00:00:00+02:00
|
||||
---
|
||||
|
||||
# MDM Reboot Coalescing
|
||||
|
||||
Device-assigned MDM policies whose CSP URIs match the OS reboot-required catalog
|
||||
(`HKLM\SOFTWARE\Microsoft\Provisioning\SyncML\RebootRequiredURIs`) force a **coalesced
|
||||
reboot at the end of ESP DeviceSetup** — the user sees an unexpected restart and a
|
||||
**second sign-in screen** before AccountSetup. Invisible in the Intune console; most
|
||||
admins live with it, not knowing that reassigning the profiles to user groups removes the
|
||||
reboot entirely. Source research: Patch My PC,
|
||||
["Avoid the second sign-in screen after Autopilot"](https://patchmypc.com/blog/autopilot-unexpected-reboot-autopilot-second-login-screen/).
|
||||
|
||||
The session already counts reboots (`system_reboot_detected` → `session.rebootCount`);
|
||||
this feature adds the **cause attribution**.
|
||||
|
||||
# Verified signal facts (session b2e890c1, 2026-07-20)
|
||||
|
||||
* **EventID 2800**, `Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin`,
|
||||
one record per URI, verified description shape:
|
||||
`The following URI has triggered a reboot: (./Device/Vendor/MSFT/Policy/Config/ServiceControlManager/SvchostProcessMitigation).`
|
||||
Records arrive in sub-second bursts during policy sync.
|
||||
* **Timing semantics (the critical nuance)**: 2800 fires at EVERY policy sync. Only URIs
|
||||
applied **during ESP DeviceSetup** are coalesced into the forced mid-ESP restart. In the
|
||||
verified session the flags fired during AccountSetup (after DeviceSetup completed) —
|
||||
**no reboot occurred**, and `omadmRebootRequiredFlag` read `false`. A detector that
|
||||
claims "reboot incoming" on every 2800 is therefore wrong.
|
||||
* **Shell-Core 62407 "RebootCoalescing" — REMOVED as a signal.** The token appears in the
|
||||
ROUTINE bootstrap marker on every enrollment:
|
||||
`Name: 'CommercialOOBE_BootstrapStatusCategory_SubcategoryProcessing_Started', Value: '{"message":"BootstrapStatus: Starting subcategory DeviceSetup.RebootCoalescing...","errorCode":0}'`
|
||||
— the ESP merely *processes* the RebootCoalescing subcategory (which checks whether a
|
||||
reboot is needed). A substring match on it is a false positive by construction. If a
|
||||
real coalesced-reboot record is ever captured on the E2E VM, a precise pattern can be
|
||||
added then.
|
||||
|
||||
# Detector — `MdmRebootPolicyTracker` (`MdmRebootPolicyWatcherHost`)
|
||||
|
||||
Live `EventLogWatcher` on the DM-Enterprise Admin channel (EventID 2800) + startup
|
||||
backfill (default 60 min lookback). Records are **buffered and debounced (10 s)**, then
|
||||
flushed as **ONE aggregated `mdm_policy_reboot_required` event** per burst:
|
||||
|
||||
* `Severity=Info`, neutral message ("Windows flagged N MDM policy URIs as
|
||||
reboot-required: … (+N more)") — the raw event never claims a reboot.
|
||||
* `Data`: `rebootUris` (distinct, sorted, capped 20), `uriCount` (uncapped distinct),
|
||||
`firstRebootUri` (record-time order, drives rule interpolation), `recordCount`,
|
||||
`unparsedCount`/`sampleDescription` (extraction diagnostics), `backfilled` (all-backfill),
|
||||
fail-soft `omadmRebootRequiredFlag` probe of
|
||||
`HKLM\SOFTWARE\Microsoft\Provisioning\OMADM\SyncML\RebootRequired` (null = unknown).
|
||||
* `Timestamp` = earliest record time (backfilled bursts land at their historical instant);
|
||||
`ImmediateUpload=true` (one event per burst is cheap, and a DeviceSetup burst precedes a
|
||||
reboot that kills the agent).
|
||||
* **Watermark is persisted at FLUSH, not at claim** (`mdm-reboot-watermark.json`): if the
|
||||
coalesced reboot kills the process before the debounced emit, nothing is marked — the
|
||||
post-restart backfill re-reads and emits the burst. A flushed burst is never repeated.
|
||||
|
||||
URI extraction: EventData values starting with `./` preferred, permissive regex over the
|
||||
description as fallback (trailing punctuation trimmed — the verified text wraps the URI in
|
||||
parens and ends with a period). Unparsed records still count into the aggregate.
|
||||
|
||||
Config (internal steering only, no admin UI — agent-native collector, NOT an
|
||||
`IAgentAnalyzer`): `Collectors.MdmRebootPolicyWatcherEnabled` (default true),
|
||||
`Collectors.MdmRebootPolicyBackfillLookbackMinutes` (default 60). ConfigVersion 35.
|
||||
|
||||
# Advisory — ANALYZE-ESP-005
|
||||
|
||||
The reboot **claim** lives here, and only here. Required conditions (BOTH):
|
||||
|
||||
1. `mdm_policy_reboot_required.firstRebootUri` exists (→ `{{firstRebootUri}}` interpolation)
|
||||
2. `system_reboot_detected` count ≥ 1 — **an actually-observed reboot**
|
||||
|
||||
Policy flags alone (e.g. AccountSetup user-policy sync) stay advisory-silent. Optional
|
||||
confidence: `uriCount ≥ 2` (+10), `omadmRebootRequiredFlag == true` (+15) on base 70 /
|
||||
threshold 70. Remediation: map URIs to Intune profiles, reassign device → user groups (or
|
||||
exclude the Autopilot device group), or accept + communicate the restart.
|
||||
|
||||
# Non-goals / invariants
|
||||
|
||||
* Event type **deliberately distinct from `system_reboot_detected`** — the backend
|
||||
RebootCount matches that string exactly in two places
|
||||
(`EventIngestProcessor.Classification` + terminal recount); attribution must never
|
||||
inflate the reboot count.
|
||||
* No session-level field / SessionInfoCard change in this iteration (follow-up candidate).
|
||||
* The `RebootRequiredURIs` catalog itself is not read; only the 2800 events (which carry
|
||||
the actually-triggered URIs) are consumed.
|
||||
|
||||
# Citations
|
||||
|
||||
* [Patch My PC — Avoid the second sign-in screen after Autopilot](https://patchmypc.com/blog/autopilot-unexpected-reboot-autopilot-second-login-screen/)
|
||||
* [Microsoft Learn — Enrollment Status Page tracking](https://learn.microsoft.com/en-us/autopilot/enrollment-status)
|
||||
@@ -1,155 +0,0 @@
|
||||
---
|
||||
type: Component Overview
|
||||
title: V2 Agent — Runtime Overview
|
||||
description: How the V2 agent boots during Autopilot enrollment, which collectors it runs, how telemetry flows to the backend, and how it decides to stop.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2/Program.cs
|
||||
tags:
|
||||
- agent
|
||||
- runtime
|
||||
- collectors
|
||||
- telemetry
|
||||
timestamp: 2026-07-11T00:00:00+02:00
|
||||
---
|
||||
|
||||
# V2 Agent — Runtime Overview
|
||||
|
||||
The agent is a .NET Framework 4.8 executable (`AutopilotMonitor.Agent.exe`) that runs as
|
||||
SYSTEM on Windows devices during Autopilot enrollment. It observes the enrollment
|
||||
(ESP phases, IME app installs, Hello, desktop arrival, …), streams telemetry to the
|
||||
backend, and decides locally — via the [decision engine](decision-engine.md) —
|
||||
when the enrollment is complete or failed. When it reaches a terminal decision it shows
|
||||
the optional summary dialog, uploads diagnostics, and removes itself.
|
||||
|
||||
Projects: `src/Agent/AutopilotMonitor.Agent.V2` (exe) + `AutopilotMonitor.Agent.V2.Core`
|
||||
(library) + `src/Shared/AutopilotMonitor.DecisionCore` (portable decision engine).
|
||||
|
||||
# Lifecycle
|
||||
|
||||
## Install (bootstrap during OOBE)
|
||||
|
||||
`AutopilotMonitor.Agent.exe --install` is invoked by the Intune Platform Script
|
||||
(bootstrap) and never runs the monitoring runtime
|
||||
(`src/Agent/AutopilotMonitor.Agent.V2/Program.InstallMode.cs`):
|
||||
|
||||
1. Copy payload to `%ProgramData%\AutopilotMonitor\Agent\`.
|
||||
2. Persist `bootstrap-config.json` (bootstrap token, tenant id) and `await-enrollment.json`.
|
||||
3. Register the `AutopilotMonitor-Agent` scheduled task (BootTrigger, SYSTEM,
|
||||
`ExecutionTimeLimit=PT0S`, survives battery conditions).
|
||||
4. Immediate runtime handoff: WMI `Win32_Process.Create` → `schtasks /Run` fallback →
|
||||
defer to BootTrigger on next reboot. `--install` exits 0 even if both immediate
|
||||
launches fail (fail-soft; the BootTrigger is the net).
|
||||
5. Write `HKLM\SOFTWARE\AutopilotMonitor\Deployed` = UTC timestamp — the **bootstrap
|
||||
re-entry lock** read by the install script. Never consulted by the agent itself;
|
||||
intentionally preserved across cleanup.
|
||||
|
||||
## Runtime start (each process start / boot)
|
||||
|
||||
`Program.Main` → `RunAgent` runs numbered phases (V1-parity exit codes 0–7):
|
||||
|
||||
1. **Guards & bootstrap** (`AgentBootstrap.Run`): classify previous exit
|
||||
(`clean` / `exception_crash` / `hard_kill` / `reboot_kill` via `clean-exit.marker`,
|
||||
crash logs, boot event 6009); ghost-restart guard (`enrollment-complete.marker`
|
||||
→ cleanup + exit 0); absolute session-age emergency break; optional waits
|
||||
(`--await-enrollment` MDM cert poll, `--tenant-id-wait` registry wait); resolve
|
||||
TenantId from `HKLM:\SOFTWARE\Microsoft\Enrollments`; get-or-create SessionId.
|
||||
2. **Auth clients** (mTLS with the Intune MDM device client certificate).
|
||||
3. **Remote config** (`GET /api/agent/config`): live fetch → cached
|
||||
`Config\remote-config.json` → built-in defaults. Config-channel **kill switch**
|
||||
checked here, honoured only from a *live* fetch (cache strips all security-sensitive
|
||||
fields).
|
||||
4. **Telemetry clients**, **backend session registration**.
|
||||
5. **`AgentRuntimeHost.Run`**: builds the `EnrollmentOrchestrator`, starts collector
|
||||
hosts, decision pipeline and upload drain loop, then blocks until shutdown.
|
||||
|
||||
## Stop triggers
|
||||
|
||||
- **Terminal decision** of the engine (Completed / Failed / WhiteGloveSealed) — the normal path.
|
||||
- **Max-lifetime watchdog**: `AgentMaxLifetimeMinutes` (default 360) — hard cap per session.
|
||||
- Backend **kill switch** (config channel at start; telemetry channel via upload responses
|
||||
→ synthesized `terminate_session` action).
|
||||
- Auth-failure threshold, self-update restart, Ctrl+C / ProcessExit.
|
||||
|
||||
Distinct from agent lifetime: `CollectorIdleTimeoutMinutes` (default 15) stops only the
|
||||
periodic collectors (Performance, AgentSelfMetrics) when nothing "real" happens; they
|
||||
restart on new activity.
|
||||
|
||||
## Termination sequence
|
||||
|
||||
On terminal decision (`EnrollmentTerminationHandler`): stop collectors → build
|
||||
`FinalStatus` → write `final-status.json` + launch `SummaryDialog` (separate WPF exe in
|
||||
the user session, opt-in) → 2 s late-event grace → diagnostics ZIP upload →
|
||||
write `enrollment-complete.marker` → optional reboot → self-destruct cleanup
|
||||
(skipped for WhiteGlove Part 1, which instead writes the `whiteglove.complete` marker
|
||||
and leaves state for Part 2).
|
||||
|
||||
# Collector model
|
||||
|
||||
Collectors are wrapped in **collector hosts** (`ICollectorHost`, built by
|
||||
`DefaultComponentFactory.CreateCollectorHosts`). Each host owns collector/tracker
|
||||
classes plus a **signal adapter** that translates raw callbacks into decision signals.
|
||||
Main hosts (gating in parentheses; several more are opt-in via remote config):
|
||||
|
||||
| Host | Watches |
|
||||
| --- | --- |
|
||||
| EspAndHelloHost | ESP phases, Shell-Core events, provisioning status, Hello (always) |
|
||||
| ImeLogHost | IME log tailer (CMTrace) + IME process watcher — app installs (always) |
|
||||
| DesktopArrivalHost | real-user desktop arrival (always) |
|
||||
| AadJoinHost | AAD join events + hybrid login pending (always) |
|
||||
| DeviceInfoHost | one-shot hardware/OS/network inventory (always) |
|
||||
| StallProbeHost | idle/stall probes (config) |
|
||||
| WindowsUpdateWatcherHost / OsBuildChangeHost | WU during OOBE, build changes (config / always) |
|
||||
| DeliveryOptimizationHost | DO counters for bandwidth estimate (config, dormant until installs run) |
|
||||
| OfficeInstallDetectorHost, ConsoleBypassHost, ProvisioningPackageHost, NetworkChangeHost, RealmJoinHost, GatherRuleExecutorHost, … | feature-specific (mixed gating) |
|
||||
|
||||
# Event pipeline (single rail)
|
||||
|
||||
Everything a collector observes flows through **one** path — no collector emits
|
||||
telemetry directly:
|
||||
|
||||
```
|
||||
Collector callback
|
||||
→ SignalAdapter (dual emission: decision signal + InformationalEvent)
|
||||
→ SignalIngress.Post() (bounded queue 256, back-pressure blocks producer;
|
||||
posts from the worker thread itself bypass the
|
||||
queue via a worker-local follow-up queue)
|
||||
→ single worker thread:
|
||||
assign ordinals → SignalLog.Append (durable) ← source of truth
|
||||
→ DecisionEngine.Reduce → apply step/effects
|
||||
→ telemetry projection (best-effort)
|
||||
→ TelemetrySpool → batch (≤100) → gzip → POST /api/agent/telemetry
|
||||
```
|
||||
|
||||
- Wire model: `EnrollmentEvent` (`EventType` from `Constants.EventTypes`, monotonic
|
||||
`Sequence` = canonical order, `Phase`, `ImmediateUpload` for lifecycle-critical items),
|
||||
wrapped in `TelemetryItem` (Kind = Event / Signal / DecisionTransition, monotonic
|
||||
`TelemetryItemId` as upload cursor).
|
||||
- Auth per request: client-cert mTLS + hardware headers (`X-Tenant-Id`,
|
||||
`X-Device-Manufacturer/-Model/-SerialNumber`, `X-Bootstrap-Token` pre-MDM-cert,
|
||||
`X-Agent-Version`).
|
||||
- Retry: 100/400/1600 ms for transient (408/429/5xx); 413 → split batch and halve batch
|
||||
size; unknown 4xx → items retained (cursor holds) unless the backend flags explicit
|
||||
poison row keys; offline → spool keeps everything, backend dedups on re-upload;
|
||||
`DeviceBlocked` pauses the drain until `UnblockAt`.
|
||||
|
||||
# Decision integration
|
||||
|
||||
Adapters post `DecisionSignal`s; the engine's terminal stage fires
|
||||
`Terminated` (off the worker thread) → termination sequence above. Engine *effects*
|
||||
(timeline events like `enrollment_complete`, deadline scheduling, classifier runs,
|
||||
snapshots) are executed by the `EffectRunner` — the reducer itself is pure. Details:
|
||||
[decision engine](decision-engine.md); durability and replay:
|
||||
[logs & persistence](logs-and-persistence.md).
|
||||
|
||||
# On-disk layout (`%ProgramData%\AutopilotMonitor\`)
|
||||
|
||||
| Path | Contents |
|
||||
| --- | --- |
|
||||
| `\` | `bootstrap-config.json`, `await-enrollment.json`, `clean-exit.marker`, session-id persistence |
|
||||
| `\Agent\` | deployed binaries |
|
||||
| `\State\` | decision persistence (SignalLog, Journal, Snapshot), `enrollment-complete.marker`, `final-status.json`, self-update state, `.quarantine/`, `.part1-<ts>/` |
|
||||
| `\Spool\` | telemetry spool (offline queue) |
|
||||
| `\Logs\` | agent trace log (`agent.log`), `crash-*.log`, `ime-pattern-matches.log` |
|
||||
| `\Config\` | `remote-config.json` cache |
|
||||
| `\Agent-Update\`, `\Updates\` | self-update staging/download |
|
||||
|
||||
Per-file details and the recovery flow: [logs & persistence](logs-and-persistence.md).
|
||||
@@ -1,200 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ESP Completion Starvation — the sits-d Cloud-PC complex
|
||||
description: Why five Windows 365 Cloud PCs finished enrolling but never completed in the product — a SkipUser=true flow produces none of the signals every completion-gate evaluation site hangs off — plus the two adjacent defects fixed alongside (edge-only user-apps-settled synthesis, orphaned Hello-wizard restart replay) and what a Shell-Core replay deliberately does not do.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals
|
||||
tags:
|
||||
- agent
|
||||
- esp
|
||||
- completion
|
||||
- skip-user
|
||||
- reboot
|
||||
- recovery
|
||||
timestamp: 2026-08-20T00:00:00+02:00
|
||||
---
|
||||
|
||||
# ESP Completion Starvation — the sits-d Cloud-PC complex
|
||||
|
||||
Seven Windows 365 Cloud PCs, same tenant, first sign-in the same morning. Two sessions
|
||||
succeeded within minutes; five sat at `Completion is waiting on: hello_resolution` from
|
||||
second five onward and ran into the six-hour max-lifetime watchdog — verdict `Incomplete`
|
||||
on devices whose users were working normally the whole time.
|
||||
|
||||
The first diagnosis blamed the mid-ESP reboot (DeviceGuard/DmaGuard, see
|
||||
[MDM Reboot Coalescing](mdm-reboot-coalescing.md)): the agent is killed in
|
||||
`Stage=EspAccountSetup` and, so the theory went, loses the Shell-Core ESP exit to the
|
||||
downtime. The evidence disproved it. `8110e262` ran **six hours uninterrupted** after its
|
||||
last reboot with the Shell-Core watcher armed — and Windows never wrote a 62407 in all
|
||||
that time, because on these machines **the user ESP page never existed**.
|
||||
|
||||
# Schema
|
||||
|
||||
## The actual defect — a satisfied gate nobody evaluates
|
||||
|
||||
The five stuck machines carried `SkipUserStatusPage=True` in the enrollment `FirstSync`
|
||||
key (read 10+ times across three agent processes, `True` on every read — not a stale
|
||||
single read). Windows honours it: no user ESP page, hence
|
||||
|
||||
* no Shell-Core 62407 (`esp_exiting`) — the page that would exit does not exist,
|
||||
* no `AccountSetupCategory` provisioning registry — nothing writes categories for a page
|
||||
that never renders,
|
||||
* desktop within seconds of sign-in.
|
||||
|
||||
The reducer is *built* for this flow: **arm B** of `ShouldTransitionToAwaitingHello`
|
||||
(`SkipUserEsp == true`) declares the gate open, and the missing-prerequisites bookkeeping
|
||||
already exempts the AccountSetup gate — which is why the sessions reported only
|
||||
`hello_resolution` as missing. But the gate is a predicate, not a process. Something has
|
||||
to evaluate it, and every evaluation site hangs off a signal a skip-user flow structurally
|
||||
cannot produce:
|
||||
|
||||
| Evaluation site | Carrier signal | On SkipUser=true |
|
||||
|---|---|---|
|
||||
| `HandleEspExitingV1` | Shell-Core 62407 | never fires |
|
||||
| `HandleEspPhaseChangedV1(FinalizingSetup)` | Shell-Core 62407 via coordinator | never fires |
|
||||
| `HandleDesktopArrivedV1` fast-path | requires Hello policy **disabled** | Hello was enabled |
|
||||
| `HandleImeUserSessionCompletedV1` arm-C attempt | requires the recorded final exit | unsatisfiable |
|
||||
|
||||
No knock, no promotion, no `HelloSafety` — and the `AdvisoryCompletion` backstop is armed
|
||||
exclusively from an `EspExiting` signal, so it never armed either. The session idles to
|
||||
the watchdog with an open gate.
|
||||
|
||||
## The fix — the observed skip is the exit evidence
|
||||
|
||||
`HandleImeUserSessionCompletedV1`'s completion attempt now accepts the observed skip in
|
||||
place of the final exit (`skipUserExitEquivalent`): on a flow where no page exists, no
|
||||
exit can ever say more than the `FirstSync` read already did. Everything else stays
|
||||
mandatory — AccountSetup anchor, genuine (at-or-after-anchor) IME user-session completion,
|
||||
DAD-validated real-user desktop — which makes this knock **stricter** than the existing
|
||||
Shell-Core-carried arm-B promotion (pinned in `ClassicAwaitingHelloGuardTests`, which
|
||||
requires no IME or desktop evidence at all). Hello semantics ride the existing rails:
|
||||
enabled-or-unknown promotes to `AwaitingHello` with `HelloSafety` armed (on Cloud PCs
|
||||
reached via RDP the Hello wizard never appears, so the synthetic timeout resolves the
|
||||
wait), disabled completes directly with the synthetic `Skipped` outcome.
|
||||
|
||||
The knock deliberately lives on the IME user-session edge, not on desktop arrival:
|
||||
|
||||
* **Ordering-robust against Fix 10.** An `EspPhaseChanged(AccountSetup)` arriving on
|
||||
`AwaitingHello` bounces the stage back and cancels `HelloSafety` (the premature-promotion
|
||||
guard). The IME AccountSetup phase line lands *after* the desktop on this flow, so a
|
||||
desktop-side knock would be undone two signals later. The IME phase line always precedes
|
||||
the user-session-complete line in log order, so a bounced promotion is re-knocked by the
|
||||
very next re-emission — including the restart re-parse.
|
||||
* **Right semantics.** The IME user session completes when the required-app processing is
|
||||
done; a desktop-side knock would mark the session `Succeeded` mid-installation.
|
||||
* Device Preparation is excluded — WDP keeps its Hello+Desktop conjunction and the
|
||||
`DevicePrepCompletion` backstop.
|
||||
|
||||
`ClassicSkipUserEspUserSessionCompletionTests` replays the real `8110e262` signal ordering
|
||||
end to end (bootstrap `EspConfigDetected` → Hello enabled → desktop → AccountSetup anchor →
|
||||
IME completion → HelloSafety → `enrollment_complete`), including the Fix-10
|
||||
bounce-then-re-knock cycle, and pins the negative space (no skip observed / no desktop /
|
||||
ghost pre-anchor IME completion / WDP).
|
||||
|
||||
Why five of seven identically administered Cloud PCs carried `SkipUserStatusPage=True` is
|
||||
a tenant-side question (ESP profile assignment state at each machine's provisioning
|
||||
moment); the value did not change during any observed session, and machine age does not
|
||||
cleanly explain the split. The agent's job is to complete the flow Windows actually ran —
|
||||
which it now does.
|
||||
|
||||
## Adjacent defect — the synthesis was edge-triggered
|
||||
|
||||
This one is real but belongs to the `SkipUser=false` flow (the two sessions that
|
||||
succeeded). The user-apps-settled synthesis
|
||||
(`MaybeSynthesizeAccountSetupCompleteFromSettledUserApps`, session `caa6cf50`) ran **only**
|
||||
from `OnEspExited`. On a tenant with ~138 required apps the ESP page exits while apps are
|
||||
still in flight: the single attempt misses and nothing ever re-checks, even when every
|
||||
tracked app reaches a terminal state minutes later.
|
||||
|
||||
The ESP exit is an **edge** (happens once); settled user-ESP apps are a **level** (reached
|
||||
later). `EspAndHelloTracker` records the edge in `_espExitObserved` and exposes
|
||||
`ReevaluateUserAppsSettledSynthesis()`, chained into `ImeLogTracker.OnAppStateChanged` and
|
||||
into the post-restore `onStateRestored` callback (a live exit observed before the IME
|
||||
state restore completes would otherwise never get a second look). Gate conditions are
|
||||
unchanged — no new completion path, only a second opportunity for the existing one. Only a
|
||||
**live, confirmed post-AccountSetup** exit may be remembered
|
||||
(`IsConfirmedPostAccountSetupExit`, stricter than the `IsIntermediateDeviceEspExit`
|
||||
forward guard: unknown counts as not confirmed).
|
||||
|
||||
## Adjacent defect — the restart replay was never wired up
|
||||
|
||||
`ShellCoreTracker`'s backfill had existed since session `772fe502` with **no caller**; the
|
||||
Shell-Core `EventLogWatcher` only delivers records written after `Start()`.
|
||||
`EspAndHelloHost.Start()` now calls it as `BackfillRecentHelloWizardStart` — which is
|
||||
exactly what it does and all it does:
|
||||
|
||||
| Record | Replayed | Why |
|
||||
|---|---|---|
|
||||
| 62404 Hello-wizard start | **yes** | Conservative: vetoes a premature "Hello is disabled" skip, can never complete a session by itself. The observation `772fe502` was about. |
|
||||
| 62407 ESP exit | **no** | Cannot be placed in time — see below. |
|
||||
| 62407 ESP failure | **no** | Re-injecting a historic failure as fresh can fail a session that recovered on retry (`ANALYZE-ESP-006`). |
|
||||
|
||||
A replayed exit is unusable, and every candidate ordering mechanism fails independently:
|
||||
|
||||
1. Windows writes the **identical** description `CommercialOOBE_ESPProgress_Page_Exiting`
|
||||
for the intermediate DeviceSetup→AccountSetup transition and the final exit.
|
||||
2. Every post-hoc ordering probe (AccountSetup registry, settled-apps) reads state as it
|
||||
is **now**, not as it was at the event's time.
|
||||
3. The reducer orders exits by **ingest ordinal**, not timestamp
|
||||
(`IsPostAccountSetupFinalExit`) — a historic exit replayed today gets a fresher ordinal
|
||||
than reality and reads as post-AccountSetup by construction.
|
||||
4. `HandleEspExitingV1` passes `espFinalExitInFlight: true` for every arriving exit; with
|
||||
restored state, arm C would open on a historic intermediate exit.
|
||||
|
||||
`ClassicEspExitingOnRestoredStateTests` pins that the reducer *does* open on any arriving
|
||||
exit; `ShellCoreTrackerReplayScopeTests` pins that the replay never produces one. Skipped
|
||||
62407 records are counted and reported once as an `agent_trace`
|
||||
(`reason: replayed_62407_not_orderable`) — silently dropping evidence is how this class of
|
||||
bug survives.
|
||||
|
||||
**Residual gap, named honestly:** a `SkipUser=false` session whose final exit falls
|
||||
exactly into an agent downtime *and* whose AccountSetup category never resolves
|
||||
(`1ec8f4c6` shape) still has no completion carrier after the restart. The normal case is
|
||||
covered — `ProvisioningStatusTracker` reads the current registry at startup, so a
|
||||
category that resolved during the downtime fires arm A. The residual shape is rare,
|
||||
surfaced by `ANALYZE-ESP-005` via the `session_stalled` interim trigger, and deliberately
|
||||
not patched with a replay that cannot be ordered.
|
||||
|
||||
### Sizing the replay window
|
||||
|
||||
A constant is provably wrong: the scheduled task has a `BootTrigger` only
|
||||
(`Program.InstallMode.BuildScheduledTaskXml`) — after an `exception_crash` the agent does
|
||||
not return until the next boot. `ResolveEspExitBackfillLookbackMinutes` reaches back to
|
||||
the last moment the previous run is known to have been alive (max of `snapshot.json`
|
||||
mtime and `LastBootUtc` for kill-type exits), floored at 5 minutes, capped at 360, off on
|
||||
`first_run`, and immune to cross-reboot clock skew (a future timestamp is ignored).
|
||||
|
||||
# Examples
|
||||
|
||||
Tenant `sits-d.cloud`, 2026-08-19, seven Windows 365 Cloud PCs, first sign-ins the same
|
||||
morning. The discriminator is a single registry value:
|
||||
|
||||
| Session | SkipUser | `esp_exiting` | AccountSetup registry | Reboots | Outcome |
|
||||
|---|---|---|---|---|---|
|
||||
| `08fc6bda` | False | yes | yes | 0 | Succeeded (10 min) |
|
||||
| `3b9291aa` | False | yes | yes | 0 | Succeeded (2.1 h) |
|
||||
| `8110e262` | True | — | — | 2 | Incomplete (6 h watchdog) |
|
||||
| `a89aac2d` | True | — | — | 1 | Incomplete |
|
||||
| `e7ba63c9` | True | — | — | 1 | Incomplete |
|
||||
| `cb4a485a` | True | — | — | 2 | Incomplete |
|
||||
| `3d6278fb` | True | — | — | 1 | Incomplete |
|
||||
|
||||
The reboot correlation was a red herring: `8110e262` ran six hours live after its final
|
||||
reboot and would have seen every signal. `8110e262` and `a89aac2d` both reached
|
||||
`138/138 completed, 0 failed` and a genuine IME user-session completion — with the fix,
|
||||
all five complete a few minutes after their IME user session settles.
|
||||
|
||||
`08fc6bda` exercises the edge-level fix on the same day: ESP page exited normally while
|
||||
`categorySucceeded` never confirmed — completed via the user-apps-settled synthesis.
|
||||
`3b9291aa` shows the independent starvation visibility gap: 99 `app_install_starved`
|
||||
events on a session recorded as `Succeeded` — surfaced from now on by `ANALYZE-APP-016`.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Classic.cs` — `HandleImeUserSessionCompletedV1`, `skipUserExitEquivalent`
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Shared.cs` — `ShouldTransitionToAwaitingHello` (arm B)
|
||||
* `src/Agent/AutopilotMonitor.DecisionCore.Tests/ClassicSkipUserEspUserSessionCompletionTests.cs` — end-to-end replay of the 8110e262 ordering
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ShellCoreTracker.cs` — `BackfillRecentHelloWizardStart`, `ReplayBackfillRecords`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs` — `_espExitObserved`, `ReevaluateUserAppsSettledSynthesis`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs` — lookback sizing, `OnAppStateChanged` chaining
|
||||
* [MDM Reboot Coalescing](mdm-reboot-coalescing.md) — the policy attribution for the reboots themselves
|
||||
* [Decision Engine](decision-engine.md) — the AccountSetup gate and completion arms
|
||||
@@ -1,106 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Power-State Watcher — live AC/battery transitions and threshold events during enrollment
|
||||
description: How the agent observes AC↔battery switches and downward battery-threshold crossings mid-enrollment via a WMI push subscription (no polling), the power_state_change event semantics (transition field, 50/30/15 ladder, latch/debounce/cap), and the two analyze rules on top (ANALYZE-DEV-009/-010).
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/PowerStateWatcherHost.cs
|
||||
tags:
|
||||
- agent
|
||||
- power
|
||||
- battery
|
||||
- collector
|
||||
- wmi
|
||||
- analyze-rules
|
||||
timestamp: 2026-08-21T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Power-State Watcher
|
||||
|
||||
The startup probe (`power_state_check`, one-shot from `StartupEnvironmentProbes` via
|
||||
`PowerStateProbe`/`GetSystemPowerStatus`) only captures the power state at agent start. A
|
||||
device unplugged mid-enrollment — or draining its battery across a multi-hour ESP — was
|
||||
invisible. The watcher closes that gap **event-driven, with zero polling**.
|
||||
|
||||
# Mechanism
|
||||
|
||||
- `PowerStateWatcherHost` (`Orchestration/Hosts`) arms a WMI event subscription with a
|
||||
two-step chain:
|
||||
1. `SELECT * FROM Win32_PowerManagementEvent WHERE EventCode = 10` (PBT_APMPOWERSTATUSCHANGE:
|
||||
true kernel push, fires on AC↔DC switches and battery-percentage steps).
|
||||
2. **Fallback** `SELECT * FROM __InstanceModificationEvent WITHIN 30 WHERE TargetInstance
|
||||
ISA 'Win32_Battery'` — required because on Windows 11 24H2/25H2 (observed on builds
|
||||
26200 and 26220, session 161b838c) the `MS_Power_Management_Event_Provider` is still
|
||||
registered in the repository but its activation fails with `WBEM_E_NOT_FOUND`: the
|
||||
legacy push provider no longer loads on exactly the OS builds new devices enroll with.
|
||||
The WITHIN sampling runs inside WinMgmt (one Win32_Battery snapshot every 30 s), not in
|
||||
the agent; worst-case detection latency ≈ 35 s incl. debounce. `collector_degraded`
|
||||
(`watcher_arm_failed`) is emitted only when BOTH attempts fail; the agent log's arm
|
||||
line names the active mode.
|
||||
|
||||
Either way the arriving event's payload is ignored — each arrival re-probes via the
|
||||
existing `PowerStateProbe.Probe()` (cheap P/Invoke).
|
||||
- Snapshots are diffed by the pure `PowerStateTransitionTracker`
|
||||
(`Monitoring/Enrollment/SystemSignals`), which decides what to emit. The host only owns
|
||||
WMI, a 5 s trailing-edge debounce timer (dock-flap AC→battery→AC inside the window
|
||||
collapses to a no-diff) and the emit path (`InformationalEventPost`, single rail).
|
||||
- **No battery / probe error ⇒ WMI is never armed** — desktops and VMs pay nothing. WMI
|
||||
arming runs on a background thread (WinMgmt can hang in OOBE; `ConsoleBypassWatcher`
|
||||
precedent); an arm failure emits one `collector_degraded` (`watcher_arm_failed`).
|
||||
- Registered as a plain always-on peripheral host after `NetworkChangeHost`, deliberately
|
||||
NOT under `PeriodicCollectorLifecycleHost`: it is push-based (zero idle cost) and must be
|
||||
alive precisely when the device idles on battery. Kill-switch:
|
||||
`Collectors.EnablePowerStateWatcher` (default true, compile-time default like the other
|
||||
collector toggles; ConfigVersion 38).
|
||||
|
||||
# Event: power_state_change
|
||||
|
||||
`data.transition` discriminates three shapes (payload always carries `onAcPower`,
|
||||
`batteryPercent`, `isCharging`, `batteryLifeMinutes`; unknowns as `"unknown"`):
|
||||
|
||||
| transition | Severity | Immediate | Notes |
|
||||
|---|---|---|---|
|
||||
| `ac_to_battery` | Warning | yes | one per accepted edge |
|
||||
| `battery_to_ac` | Info | no | recovery |
|
||||
| `threshold_crossed` | Info (50) / Warning (30) / Error (15) | 30+15 only | extra field `thresholdPercent` |
|
||||
|
||||
Threshold semantics (tracker):
|
||||
|
||||
- Ladder 50/30/15, evaluated only while ON battery; **each level latches once per agent
|
||||
lifetime** (charge-up + re-drain does not re-emit — no plug-cycle alarm spam).
|
||||
- A multi-level jump emits only the LOWEST newly-crossed level (60%→10% ⇒ one `15` event).
|
||||
- Entering battery power already below a level counts as crossing it; the arm-time baseline
|
||||
does too — an enrollment STARTING at 12% battery emits its `thresholdPercent: 15` event
|
||||
immediately (the analyze rule would otherwise never see sessions that began low).
|
||||
- Lifetime emission cap 20 (storm backstop against a flapping dock / dying battery
|
||||
controller); the host logs one warning when it engages.
|
||||
|
||||
Backend/agent hygiene: `power_state_change` is classified as non-activity on both sides —
|
||||
`SignalActivityClassifier.NonActivityEventTypes` (agent: must not reset the
|
||||
PeriodicCollectorLifecycleHost/StallProbeHost idle clocks) and
|
||||
`EventIngestProcessor.IsPeriodicOrStallEvent` (backend: must not heal a Stalled session —
|
||||
battery drain is not enrollment progress).
|
||||
|
||||
# Analyze Rules
|
||||
|
||||
- **ANALYZE-DEV-009** (high): battery crossed 15% while on battery — matches the
|
||||
agent-stamped `thresholdPercent equals 15`.
|
||||
- **ANALYZE-DEV-010** (warning): AC→battery switch — matches `transition equals
|
||||
ac_to_battery`.
|
||||
- Both: `evaluateOn: ["enrollment_end", "whiteglove_sealed", "on_event:power_state_change"]`
|
||||
(live interim finding + terminal finalization), `markSessionAsFailedDefault: false`.
|
||||
- Both are deliberately **single-required-condition**: `event_data` conditions scan events
|
||||
independently (no same-instance join), so a two-condition rule (e.g. `transition ==
|
||||
threshold_crossed AND batteryPercent < 15`) could false-positive across two different
|
||||
power events. `thresholdPercent` exists only on threshold events and its value pins the
|
||||
ladder level, so one condition fully identifies the instance.
|
||||
- `{{token}}` caveat: `AddDataFieldsToEvidence` whitelists only generic identifiers
|
||||
(appId/errorCode/…), so `{{batteryPercent}}` would render literally. DEV-009 interpolates
|
||||
`{{thresholdPercent}}` (the matched condition's own dataField); the exact percentage lives
|
||||
on the linked event in the timeline.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/PowerStateWatcherHost.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/PowerStateTransitionTracker.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Runtime/PowerStateProbe.cs` (probe, unchanged)
|
||||
- `rules/analyze/ANALYZE-DEV-009.json`, `rules/analyze/ANALYZE-DEV-010.json`
|
||||
- [analyze-rule-triggers](../rules/analyze-rule-triggers.md) (interim-trigger design; the WG battery rule was its motivating case)
|
||||
@@ -1,89 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Registry App-State Second Pillar
|
||||
description: Why the agent reads IME's authoritative per-app state straight from the registry (Win32Apps / EspTrackingWin32Apps / StatusServiceReports), how the snapshot-and-diff observer works, and how registry/log reconciliation doubles as the built-in IME log-pattern drift alarm.
|
||||
resource: agent
|
||||
tags: [ime, registry, win32apps, observability, reconciliation, pattern-drift, wdp]
|
||||
timestamp: 2026-08-18
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
## Why a second pillar
|
||||
|
||||
The 2026-08-17 audit of the decompiled IME (1.97.107 / 1.104.102) proved that IME log
|
||||
wording is an unstable contract: Microsoft removed the old V1/V2 processing pipeline
|
||||
(killing 9 patterns silently), reworded the post-remediation line, changed the
|
||||
report-delta quoting between 1.97 and 1.104, and deleted `[DO TEL]` entirely. The
|
||||
registry, by contrast, is IME's *internal product contract* — the ESP page and Intune
|
||||
reporting consume it — and cannot drift casually.
|
||||
|
||||
Architecture decision (Oliver, 2026-08-17): **hybrid**. Logs stay the narrative
|
||||
(downloads, retries, exit codes, scripts — the timeline). The registry is the stable
|
||||
truth for terminal state. Divergence between the two IS the alarm that log patterns
|
||||
drifted — turning silent data loss into a Warning event.
|
||||
|
||||
## Observed surfaces
|
||||
|
||||
All under `HKLM\SOFTWARE\Microsoft\IntuneManagementExtension` (64-bit view), verified
|
||||
against decompiled IME source:
|
||||
|
||||
| Surface | Content |
|
||||
|---|---|
|
||||
| `Win32Apps\<userGuid>\<appGuid>_<rev>` | `EnforcementStateMessage` JSON (EnforcementState + ErrorCode), `ExitCode`, `Intent`. Device context = `Guid.Empty` user. Non-GUID subkeys (GRS, OperationalState, Reporting, ProvisioningProgress) are skipped. |
|
||||
| `EspTrackingWin32Apps\<userGuid>\<appGuid>_<rev>` | Which apps IME registered for ESP tracking, `EspTrackingWin32AppPhase`. |
|
||||
| `SideCarPolicies\StatusServiceReports\<userGuid>\<appId>` | The `AppInstallStatusReport` the ESP page renders (`Status`: 1000=Installed, 1001=Installing, 3000=Failed, 2000er=NotApplicable). |
|
||||
|
||||
## Mechanism
|
||||
|
||||
`ImeRegistryAppStateObserver` + `ImeRegistryAppStateHost` (always-on, no config gate —
|
||||
precedent `EspPolicyProviderStallHost`):
|
||||
|
||||
- **One recursive `RegistryWatcher`** on the IME root covers all three surfaces;
|
||||
changes are debounced (2 s trailing edge) into a tick. A 60-s periodic tick is both
|
||||
the reconciliation-settle evaluator and the poll-only fallback when the watcher
|
||||
fails to arm (`collector_degraded`, once).
|
||||
- **Snapshot-and-diff**: `RegNotifyChangeKeyValue` gives coalesced key-scope edges
|
||||
only, so every tick re-reads the surfaces and diffs against the previous snapshot.
|
||||
- **Silent baseline**: the state present at agent start is captured without events.
|
||||
Win32Apps keys survive re-enrollments — replaying them would be the registry twin
|
||||
of the historic-IME-replay bug. Only apps that *change during the session* are
|
||||
reported or judged.
|
||||
- **`registry_app_state`** (Info): emitted per real field change
|
||||
(enforcementState/errorCode/exitCode/statusServiceStatus/espTracked/espPhase),
|
||||
capped at 200/session (cap announced once as Warning).
|
||||
- **`app_state_reconciliation`** (Warning, once per app, immediate upload): after a
|
||||
changed entry has been terminal (EnforcementState 1000er=success / 5000er=error,
|
||||
or StatusService 1000er/3000) for ≥ 90 s, its outcome is compared against the
|
||||
log-derived `AppPackageState` from `ImeLogHost.AllKnownPackageStates`. Divergence
|
||||
reasons: `registry_error_log_installed`, `registry_success_log_error`,
|
||||
`app_unknown_to_log_tracking` (only when the tracker is actively tracking apps —
|
||||
an idle tracker is not judgeable).
|
||||
- **Pure observability**: nothing feeds the DecisionEngine (same rule as
|
||||
oobe-state observability).
|
||||
|
||||
## WDP reuse (Paket 3)
|
||||
|
||||
The same host archetype (RegistryWatcher subtree + debounce + snapshot-diff) is the
|
||||
intended vehicle for WDP live progress: watch
|
||||
`HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotSettings\DevicePreparation\BootstrapperAgent`
|
||||
(`ProgressUpdates` rows, `ResumeContext.ProvisioningResult` terminal marker) with the
|
||||
RealmJoinWatcher appearance-watcher pattern for not-yet-existing keys, plus an
|
||||
Event-142 channel watcher (`Microsoft-Autopilot-BootstrapperAgent/BootstrapperAgentServiceLogProvider`,
|
||||
channel handling precedent in `DiagnosticsPackageService`). Win32Apps keys are written
|
||||
on WDP devices too — the pillar works on both rails unchanged.
|
||||
|
||||
# Examples
|
||||
|
||||
Timeline evidence of a drifted pattern set: apps install fine (registry says
|
||||
EnforcementState 1000) but the timeline shows no `app_install_completed` — each app
|
||||
raises one `app_state_reconciliation` with `reason=app_unknown_to_log_tracking`.
|
||||
Operator response: run `/ime-pattern-validate` against the session, fix patterns,
|
||||
backend-deploy (no agent release needed for pattern fixes).
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ImeRegistryAppStateObserver.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/ImeRegistryAppStateHost.cs`
|
||||
- `tasks/ime-v1-code-audit-2026-08-17.md` (decompiled-source audit; registry surface catalog)
|
||||
- Decompile archive: `okieselbach/ime-decompiles` (private), `C:\Code\GitHubRepos\ime-decompiles\<version>\decompiled`
|
||||
@@ -1,100 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Secure Boot CA-2023 Detection — firmware evidence vs. Windows servicing status
|
||||
description: Why ANALYZE-SEC-001 v3 reads the UEFI db/KEK variables directly (GetFirmwareEnvironmentVariableExW + SeSystemEnvironmentPrivilege) instead of trusting the SecureBoot\Servicing registry key, and how the one-sided uefiCa2023FirmwareConfirmed marker plus a not_exists precondition give absence-tolerant rule suppression across old and new agents.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/UefiSecureBootCertReader.cs
|
||||
tags:
|
||||
- agent
|
||||
- secureboot
|
||||
- ca2023
|
||||
- analyze-rules
|
||||
- backward-compatibility
|
||||
timestamp: 2026-07-23T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Secure Boot CA-2023 Detection
|
||||
|
||||
## Schema
|
||||
|
||||
Two independent evidence sources describe the Windows UEFI CA 2023 rollout state:
|
||||
|
||||
1. **Servicing registry** (`HKLM\SYSTEM\CurrentControlSet\Control\SecureBoot\Servicing\UEFICA2023Status`)
|
||||
— reflects whether the *Windows Update* rollout ran and self-verified. Lags reality:
|
||||
devices whose firmware received the certificate another way (OEM factory image, WinCS
|
||||
CLI, manual deployment) or where WU simply has not run yet show `unknown`/`notfound`.
|
||||
2. **Firmware itself** — the UEFI variables `db` (signature database, GUID
|
||||
`{d719b2cb-3d3a-4596-a3bc-dad00e67656f}`) and `KEK` (GUID
|
||||
`{8be4df61-93ca-11d2-aa0d-00e098032b8c}`). Authoritative. Certificate subject CNs are
|
||||
ASCII bytes inside the DER signature lists, so a byte-substring search for
|
||||
`Windows UEFI CA 2023` / `Microsoft Corporation KEK 2K CA 2023` decides presence —
|
||||
the same check `Get-SecureBootUEFI` enables from PowerShell.
|
||||
|
||||
`UefiSecureBootCertReader` (agent, `Monitoring/Interop`) reads both variables natively via
|
||||
`GetFirmwareEnvironmentVariableExW` — no PowerShell runspace during OOBE. The call needs
|
||||
`SeSystemEnvironmentPrivilege`: SYSTEM holds it default-disabled, so the reader enables it
|
||||
on the process token (`AdjustTokenPrivileges`) and restores the previous state afterwards.
|
||||
Fail-soft contract like `OobeStateReader`: never throws; failures map to status strings
|
||||
(`not_uefi` = legacy BIOS via `ERROR_INVALID_FUNCTION`, `variable_not_found`,
|
||||
`privilege_denied`, `error_<code>`).
|
||||
|
||||
This is the repo's only privilege-adjustment code, so it also carries the ownership rules
|
||||
any future one should copy:
|
||||
|
||||
* The process token lives in a `SafeHandle` assigned by the P/Invoke marshaler, so the CLR
|
||||
owns it from the moment `OpenProcessToken` returns — the critical finalizer releases it
|
||||
even if `Dispose` never runs (async exception before the instance reaches its `using`).
|
||||
* The restore obligation is recorded immediately after the successful `AdjustTokenPrivileges`,
|
||||
and only when the privilege was not *already* enabled; nothing that can throw sits between
|
||||
the adjustment and that assignment. `Dispose` is idempotent, swallows a failed restore, and
|
||||
releases the handle in a `finally`.
|
||||
* The privilege is adjusted on the **process** token, so enable → read → restore is a
|
||||
process-wide side effect and runs under a static gate. Without it a concurrent reader would
|
||||
see "already enabled", correctly skip its own restore, and then lose the privilege mid-read
|
||||
when the first reader disposes — a spurious `privilege_denied` on a machine that holds it.
|
||||
|
||||
Fields land on the existing `secureboot_status` event (once per agent start,
|
||||
restart-deduped by the StartupEventGate): `uefiFirmwareReadStatus`, per-cert booleans
|
||||
(`uefiDbHasWindowsUefiCa2023`, `uefiDbHasWindowsProductionPca2011`,
|
||||
`uefiKekHasMicrosoftKek2kCa2023`, …) — emitted only when the respective read succeeded —
|
||||
plus the marker below.
|
||||
|
||||
## The one-sided marker (absence-tolerant suppression)
|
||||
|
||||
Rule-engine constraints (pinned by `RuleEnginePreconditionTests`):
|
||||
|
||||
* **Conditions** never match on an absent field (null guard) — new fields must stay
|
||||
`required: false` or the rule dies for every old agent.
|
||||
* **Preconditions**: `not_exists` passes while the field is absent/empty and fails as soon
|
||||
as the field is present with *any* value; comparison operators fail closed on absence.
|
||||
|
||||
A two-sided boolean therefore cannot drive suppression: `not_exists` on
|
||||
`uefiDbHasWindowsUefiCa2023` would also suppress the rule when the field is present with
|
||||
`false`. Hence the **one-sided marker** `uefiCa2023FirmwareConfirmed = true`, emitted
|
||||
ONLY when the db read succeeded AND the 2023 certificate was found — never as `false`.
|
||||
ANALYZE-SEC-001 v3 adds a `not_exists` precondition on it:
|
||||
|
||||
| Device state | Marker | Rule |
|
||||
|---|---|---|
|
||||
| Old agent (no firmware fields) | absent | evaluates exactly like v2 |
|
||||
| New agent, cert missing or firmware unreadable | absent | evaluates; optional conditions `uefiDbHasWindowsUefiCa2023=false` (+20) / `uefiKekHasMicrosoftKek2kCa2023=false` (+10) raise confidence |
|
||||
| New agent, firmware confirms cert | present | silently suppressed (the customer-reported false positive) |
|
||||
|
||||
Because the marker exists nowhere until the new agent ships, the v3 rule is safe to seed
|
||||
**before** the agent rollout.
|
||||
|
||||
## Examples
|
||||
|
||||
Customer-equivalent verification from PowerShell (authoritative, matches the reader):
|
||||
|
||||
```powershell
|
||||
[Text.Encoding]::ASCII.GetString((Get-SecureBootUEFI -Name db).Bytes).Contains('Windows UEFI CA 2023')
|
||||
[Text.Encoding]::ASCII.GetString((Get-SecureBootUEFI -Name KEK).Bytes).Contains('Microsoft Corporation KEK 2K CA 2023')
|
||||
```
|
||||
|
||||
## Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/UefiSecureBootCertReader.cs` — native reader
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/DeviceInfo/DeviceInfoCollector.NetworkAndSecurity.cs` — `CollectSecureBootStatus` / `AppendFirmwareCertFields`
|
||||
* `rules/analyze/ANALYZE-SEC-001.json` — v3.0.0 rule definition
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/RuleEnginePreconditionTests.cs` — pinned precondition semantics + v3 mirror tests
|
||||
* Microsoft: [Act now — Secure Boot certificates expire June 2026](https://techcommunity.microsoft.com/blog/windows-itpro-blog/act-now-secure-boot-certificates-expire-in-june-2026/4426856)
|
||||
@@ -1,110 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: System Timeline Watcher — clock steps and sleep episodes from the System event log
|
||||
description: How the agent turns two enrollment blind spots into timeline ground truth via one EventLogWatcher on the System channel — system_clock_changed (Kernel-General 1, oldTime/newTime/timeDeltaMs/process) and system_sleep_episode (Power-Troubleshooter 1 classic sleep/hibernate, Kernel-Power 507 Modern Standby) — including backfill of pre-agent records, the payload-times-are-authoritative rule, SleepSpans in time attribution, the web clock-set jump cause, and ANALYZE-DEV-012.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/SystemTimelineTracker.cs
|
||||
tags:
|
||||
- agent
|
||||
- eventlog
|
||||
- time
|
||||
- clock-skew
|
||||
- standby
|
||||
- collector
|
||||
- time-attribution
|
||||
- analyze-rules
|
||||
timestamp: 2026-08-21T00:00:00+02:00
|
||||
---
|
||||
|
||||
# System Timeline Watcher
|
||||
|
||||
Two things silently distort every session timeline and its wall-clock duration:
|
||||
|
||||
1. **The OS clock gets stepped** — classically w32time correcting a wrong BIOS clock early
|
||||
in OOBE, but also tzutil/MDM actions. Before this watcher the jump was only *inferred*
|
||||
(TimeJumpBadge, `clock_skew`/DEV-008); Windows records the ground truth.
|
||||
2. **The device sleeps mid-enrollment** — the timeline shows an unexplained gap, the
|
||||
duration looks inflated, and nothing says "the device was simply asleep for 56 minutes".
|
||||
|
||||
`SystemTimelineTracker` (hosted by `SystemTimelineWatcherHost`, config flag
|
||||
`Collectors.EnableSystemTimelineWatcher`, ConfigVersion 39) subscribes to the **System**
|
||||
channel with one provider-qualified XPath and forwards both facts as events.
|
||||
|
||||
# Sources and events
|
||||
|
||||
| Source (System channel) | Emitted event | Key payload |
|
||||
|---|---|---|
|
||||
| `Microsoft-Windows-Kernel-General` EventID 1 | `system_clock_changed` | `oldTime`, `newTime` (ISO-8601), signed `timeDeltaMs`, `reason`/`reasonText` (1=`application_set`, 2=`hardware_clock_sync`), `processName`/`processId` |
|
||||
| `Microsoft-Windows-Power-Troubleshooter` EventID 1 | `system_sleep_episode` | `kind`=`sleep`/`hibernate` (EffectiveState 5 ⇒ hibernate), `enteredAt`=SleepTime, `exitedAt`=WakeTime, `durationSeconds`, `wakeSourceType`/`wakeSourceText` |
|
||||
| `Microsoft-Windows-Kernel-Power` EventID 507 | `system_sleep_episode` | `kind`=`modern_standby`, `exitedAt`=TimeCreated, `enteredAt`=exit − DurationInUs, `durationSeconds` (scenario), `sleepDurationSeconds` (real sleep), `reason`, `onAcPower`, `batteryRemainingCapacityOnExit` |
|
||||
|
||||
**506 is deliberately not consumed.** 507 carries the full episode; pairing 506/507 would
|
||||
need an orphan-tolerant state machine across agent restarts for zero information gain — an
|
||||
episode without a 507 never completed and would not be emitted under any design.
|
||||
|
||||
# Thresholds (constants, not knobs)
|
||||
|
||||
- `MinClockDeltaMs = 2000` — w32time logs Kernel-General 1 for routine **1 ms micro-slews**
|
||||
(verified live, several per hour); only genuine steps are emitted. Suppressed records are
|
||||
still claimed by the dedup watermark.
|
||||
- `ClockDeltaWarningMs = 5 min` — larger steps are Warning + immediate upload (can break
|
||||
token/cert validation windows; matches the web TimeJumpBadge threshold).
|
||||
- `MinSleepDurationSeconds = 60` — **Modern Standby only**: every screen-off scenario logs a
|
||||
506/507 pair, including seconds-long ones with `SleepDurationInUs 0` (`SleepEntered=false`);
|
||||
the single duration gate suppresses both. Classic Power-Troubleshooter episodes have no
|
||||
floor — they only exist for genuinely completed S3/S4 transitions.
|
||||
|
||||
# Payload times are authoritative
|
||||
|
||||
The decisive clock correction happens exactly when the device clock is untrustworthy, and
|
||||
the backend clamps implausible event timestamps (±168 h/24 h) to server time. Every semantic
|
||||
instant therefore travels as an explicit ISO-8601 payload field (`oldTime`/`newTime`,
|
||||
`enteredAt`/`exitedAt`); consumers must never read the event's own `Timestamp` for these.
|
||||
The event `Timestamp` is set to the record's TimeCreated (WindowsUpdateTracker backfill
|
||||
convention) so rows land where the wake/step happened.
|
||||
|
||||
# Backfill and dedup
|
||||
|
||||
Template: `WindowsUpdateTracker`. `LoadWatermark → arm watcher → backfill` with lookback
|
||||
`SystemEventBackfillLookbackMinutes` (default 1440 = 24 h; 0 disables). The System .evtx
|
||||
persists across OOBE reboots, so the pre-agent w32time correction and pre-agent standby
|
||||
episodes are recovered with `backfilled=true`. Cross-restart dedup via RecordId watermark
|
||||
(`system-timeline-watermark.json` in the state directory) plus an intra-run `HashSet` —
|
||||
deliberately **not** a high-water mark, which would drop exactly the older, never-emitted
|
||||
pre-agent records (the live watcher arms before the backfill scan).
|
||||
|
||||
# Non-activity classification (both sides)
|
||||
|
||||
Both event types are environment observation, not enrollment progress:
|
||||
`SignalActivityClassifier.NonActivityEventTypes` (agent — must not reset idle/stall clocks)
|
||||
and `EventIngestProcessor.IsPeriodicOrStallEvent` (backend — a device that slept through its
|
||||
stall must stay Stalled).
|
||||
|
||||
# Consumers
|
||||
|
||||
- **Web timeline** (`EventTimeline.tsx`): badge blocks for both events; the TimeJumpBadge
|
||||
gains the ground-truth cause `clock-set` — `classifyTimeJump` receives the session's
|
||||
`system_clock_changed` deltas and matches a backward display step against a backward clock
|
||||
set within max(60 s, 20 %) (`lib/timeProvenance.ts`).
|
||||
- **Duration story** (`SessionInfoCard` via `useSessionDerivedData.standbySeconds`): the
|
||||
wall-clock duration deliberately keeps the pause (the enrollment really took that long);
|
||||
a `· 🌙 56m standby` note explains it. Episodes are dedup'd on `enteredAt`.
|
||||
- **Time attribution** (`TimeAttributionCalculator.BuildSleepSpans`, AttributionVersion 3):
|
||||
`SleepSpan`s mirror `RebootSpan`s — cross-cutting annotations clipped to the observation
|
||||
windows (a WhiteGlove-pause episode contributes only its in-window flanks), never a slice
|
||||
of the wall-clock partition. Persisted as `SleepSeconds`/`SleepSpansJson`; rendered as the
|
||||
standby chip in `TimeAttributionLane`.
|
||||
- **ANALYZE-DEV-012** (info, `trigger: single`, `on_event:system_sleep_episode`): fires once
|
||||
per session for episodes ≥ 5 min. Deliberately the ONLY new rule: clock issues already
|
||||
have DEV-008 (`clock_skew`) — a second clock rule would double-report the same action item
|
||||
(user decision 2026-08-21); clock steps stay timeline ground truth with badges instead.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/SystemTimelineTracker.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/SystemTimelineWatcherHost.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs` (`BuildSleepSpans`)
|
||||
- `src/Web/autopilot-monitor-web/lib/timeProvenance.ts` (`clock-set`, `findExplainingClockChange`)
|
||||
- `rules/analyze/ANALYZE-DEV-012.json`
|
||||
- Live payload evidence (2026-08-21, Win11 26220): Kernel-General 1 with `TimeDeltaInMs=1`
|
||||
micro-slews; Kernel-Power 507 with `SleepDurationInUs=2324095807`; Power-Troubleshooter 1
|
||||
with `SleepTime`/`WakeTime` across a hibernate.
|
||||
@@ -1,108 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: WDP Scenario Gates & Completion Backstop
|
||||
description: How the agent gates Autopilot/ESP-only machinery off on Windows Device Preparation (WDP) sessions and how a WDP session resolves without any ESP signal.
|
||||
resource: agent
|
||||
tags: [wdp, device-preparation, decision-engine, collectors, completion]
|
||||
timestamp: 2026-08-18
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Windows Device Preparation (Autopilot v2, "WDP") has **no deployment profile and no ESP**.
|
||||
Before the afee7ae0 audit (2026-08-18) the agent ran the full Classic apparatus anyway:
|
||||
misleading `autopilot_profile_missing` warnings, a ZTD event-log query + HTTP probe,
|
||||
`esp_config_detected` "unknown/unknown" noise, an Autopilot-channel noise flood
|
||||
(EventIDs 100/417/418/1005, 900+ occurrences per session), a subtree registry watcher
|
||||
woken by every WDP progress write, and — worst — no completion path when Hello never
|
||||
resolved (park until the 360-min watchdog).
|
||||
|
||||
## Detection: deterministic vs. fallback
|
||||
|
||||
`EnrollmentRegistryDetector` classifies v2 through three rules. Only the first is
|
||||
**deterministic WDP**:
|
||||
|
||||
1. `AutopilotSettings\DevicePreparation\BootstrapperAgent` subkey with non-empty
|
||||
`ExecutionContext` → the WDP policy orchestration tree, present before the agent
|
||||
starts. Exposed as `EnrollmentRegistryDetector.IsDeterministicDevicePreparation()`
|
||||
and the `DevicePreparationExecutionContextRule` constant.
|
||||
2./3. `CloudAssignedDeviceRegistration == 2` / `CloudAssignedEspEnabled == 0` —
|
||||
profile-derived legacy signals. They imply a REAL Autopilot profile (just ESP-less),
|
||||
so every gate below deliberately ignores them: **gates require the deterministic
|
||||
marker and fail toward Classic behavior.**
|
||||
|
||||
## Agent-side gates (all keyed to the deterministic marker)
|
||||
|
||||
* `DeviceInfoCollector.CollectAutopilotProfile`: no `autopilot_profile` event, no
|
||||
`autopilot_profile_missing` warning, no ZTD event-log query, no ZTD HTTP probe.
|
||||
`enrollment_type_detected` still fires. (WDP devices DO carry an
|
||||
`AutopilotPolicyCache` with `ProfileAvailable=0` / error 807 `ZtdDeviceIsNotRegistered`
|
||||
— Windows runs the ZTD download regardless — so the probe fired for real before.)
|
||||
* `CollectEspConfigurationLocked`: `esp_config_detected` is emitted only when at least
|
||||
one FirstSync value or ESP tracking list exists (generic null-gate, mirrors the
|
||||
orchestrator bootstrap's signal gate) — fixes "unknown/unknown" for WDP AND plain
|
||||
Entra joins.
|
||||
* `DefaultComponentFactory`: `EspPolicyProviderStallHost` (60-s poll of the ESP-only
|
||||
`EnrollmentStatusTracking` key) is not created on WDP.
|
||||
* `EspAndHelloTracker.Start`: `ProvisioningStatusTracker` is not started on WDP — its
|
||||
subtree watcher sits on `AutopilotSettings`, exactly where WDP's BootstrapperAgent
|
||||
writes progress, so it woke on every write only to read three ESP category values
|
||||
that never exist there.
|
||||
* `ModernDeploymentTracker`: `DevicePreparationNoiseEventIds` {100, 417, 418, 1005}
|
||||
are suppressed entirely on WDP (before `FormatDescription()`); Critical (level 1)
|
||||
always passes; genuine WDP diagnostics (e.g. 408) keep flowing. The Autopilot error
|
||||
backfill (807/809/815/908 replay) is skipped — 807 is the EXPECTED state on WDP.
|
||||
The watcher itself stays on: WDP's provisioning stack logs into the same channels.
|
||||
* `StallProbeCollector`: skips the ESP category registry scan on WDP and unions the
|
||||
noise IDs into its harmless set; the channel scans remain (they carry genuine WDP
|
||||
diagnostics during a stall).
|
||||
* `EnrollmentTerminationHandler`: the `app_install_starved` terminal sweep is skipped
|
||||
on WDP — post-desktop installs are the DESIGNED order there (only DPP-policy apps
|
||||
run during provisioning); per-app data still travels via `app_tracking_summary` and
|
||||
final-status.json.
|
||||
|
||||
## DecisionCore: WDP completion without ESP facts
|
||||
|
||||
* **Seeding**: `EnrollmentFactsObserved` carries `enrollmentTypeDeterministic`;
|
||||
the deterministic marker seeds `DevicePreparation` at **High** confidence (same
|
||||
registry-determinism standard as the self-deploying OobeConfig bits), the fallback
|
||||
rules stay Medium. The three explicit downgrade guards from d0ab3eee remain as
|
||||
defence in depth.
|
||||
* **Arm D** (`ShouldTransitionToAwaitingHello`): on WDP the DAD-validated real-user
|
||||
desktop is a legitimate promotion basis — arms A–C are unsatisfiable by construction
|
||||
(no provisioning categories, no Shell-Core 62407 final exit). With a disabled Hello
|
||||
policy the fast-path now completes at desktop arrival.
|
||||
* **Prerequisites**: `completion_waiting` never lists
|
||||
`account_setup_provisioning_complete` on WDP (unsatisfiable; mirrors the
|
||||
`skipUserEsp` exemption).
|
||||
* **Backstop**: `DeadlineNames.DevicePrepCompletion` (30 min, replay-safe base) is
|
||||
armed by `HandleDesktopArrivedV1` when a WDP session parks desktop-first with Hello
|
||||
pending. On fire it mirrors the HelloSafety timeout: synthetic
|
||||
`HelloOutcome=Timeout` + completion through Finalizing. This closes the
|
||||
"no Hello → park until watchdog" hole; a timer surviving into a terminal stage is
|
||||
swallowed by the post-terminal dispatch guard.
|
||||
* **Tripwire**: `DecisionStepProcessor.IsParkedWithoutDeadline` has a WDP branch
|
||||
(dead-end zone = desktop arrived) so a regression in the backstop becomes visible
|
||||
as `session_parked_without_deadline`.
|
||||
|
||||
# Examples
|
||||
|
||||
Session afee7ae0 (the audit trigger): Succeeded in 9:32 min, but its timeline carried
|
||||
`autopilot_profile` (807), an `autopilot_profile_missing` WARN, `esp_config_detected`
|
||||
unknown/unknown, 30 `modern_deployment_log` rollup events and 10 `app_install_starved`
|
||||
entries — all structurally meaningless on WDP. After the gates, the same session shape
|
||||
emits none of these; completion rode Hello-skipped + desktop as before.
|
||||
|
||||
Deliberately NOT changed (open follow-ups, see the WDP audit topic memory): the ESP
|
||||
phase vocabulary (`esp_phase_changed`/stage names ride the Classic rails on WDP),
|
||||
subscription to the `BootstrapperAgentServiceLogProvider` event channel, and
|
||||
`registry_app_state` as the primary WDP app source.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Security/EnrollmentRegistryDetector.cs` — deterministic rule + helper
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/DeviceInfo/DeviceInfoCollector.NetworkAndSecurity.cs` — profile/ESP emission gates
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ModernDeploymentTracker.cs` — `DevicePreparationNoiseEventIds`
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.DevicePreparation.cs` — backstop deadline
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/State/EnrollmentScenarioProfileUpdater.cs` — High/Medium seeding
|
||||
* Tests: `DevicePreparationEngineTests`, `ModernDeploymentTrackerDevicePrepTests`, `EnrollmentTerminationHandlerTests.StarvedApps_skipped_on_device_preparation`, `DeviceInfoCollectorEspRefreshTests.NoEspEvidenceAtAll_SuppressesTheEvent`
|
||||
@@ -1,185 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: WiFi SSID and the Windows 24H2 Location Gate
|
||||
description: Why wifi_signal_info silently disappeared on Windows 11 24H2+ devices, why the netsh fallback could never fix it, and the tiered reader that restores the SSID from the SYSTEM context.
|
||||
resource: src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/WifiInfoProvider.cs
|
||||
tags:
|
||||
- agent
|
||||
- interop
|
||||
- wifi
|
||||
- telemetry
|
||||
timestamp: 2026-08-26T14:05:00+02:00
|
||||
---
|
||||
|
||||
# Summary
|
||||
|
||||
The agent reports the enrollment WiFi as a `wifi_signal_info` event (SSID, signal, PHY, channel).
|
||||
On Windows 11 24H2 and later the event went missing on a large share of devices — no error, no
|
||||
partial payload, simply no event. The cause is a Windows privacy change, not a bug in the reader:
|
||||
**`WlanQueryInterface` with `wlan_intf_opcode_current_connection` is gated behind precise-location
|
||||
consent**, and the agent runs as SYSTEM in session 0 where that consent usually does not exist.
|
||||
|
||||
The fix is a tiered reader ([WifiInfoProvider](../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/WifiInfoProvider.cs))
|
||||
whose middle tier uses the one SSID API Microsoft left ungated.
|
||||
|
||||
# The Gate
|
||||
|
||||
Microsoft restricted the Win32 WLAN APIs that expose BSSIDs, because nearby-BSSID lists are a
|
||||
precise-location oracle. Without consent these return `ERROR_ACCESS_DENIED` (5):
|
||||
|
||||
* `WlanGetAvailableNetworkList`
|
||||
* `WlanGetNetworkBssList`
|
||||
* `WlanScan`
|
||||
* **`WlanQueryInterface` when `OpCode` is `wlan_intf_opcode_current_connection`** — the only opcode
|
||||
the agent needs, and the one that carries SSID, signal quality, PHY type.
|
||||
|
||||
Two properties of the gate decide the agent's fate:
|
||||
|
||||
1. **The consent prompt only appears for a process running in the user's context and outside
|
||||
`C:\Windows\System32`.** The agent is neither during ESP: it is SYSTEM, in session 0, before any
|
||||
user has seen the OOBE privacy page. It is therefore denied *silently* — no prompt, no event
|
||||
log, nothing that surfaces without asking the API for its return code.
|
||||
2. **Consent resolves per user hive, with the device-level setting as the floor.** An interactive
|
||||
user typically has
|
||||
`HKCU\...\CapabilityAccessManager\ConsentStore\location\NonPackaged\Value = Allow`
|
||||
("Let desktop apps access your location"). The SYSTEM account (`HKU\S-1-5-18`) normally has
|
||||
**nothing** set there and falls back to
|
||||
`HKLM\...\ConsentStore\location\Value`. Where that device-level value is `Deny`, SYSTEM is
|
||||
denied even though the logged-on user on the very same machine reads the SSID fine.
|
||||
|
||||
That asymmetry is why the failure looks random from the outside: it tracks the **image's
|
||||
device-level location setting**, which varies per fleet, not per hardware, driver, or language.
|
||||
|
||||
# Why netsh Was Never the Answer
|
||||
|
||||
`NetshWifiFallback` was added in 2026-07 on the theory that inbox `netsh` still reported the
|
||||
connection where the WLAN API did not, and it was later extended with localized label variants for
|
||||
every enrollment-geography language. Field data says otherwise: **`netsh wlan` hangs off the same
|
||||
location gate** ("Network shell commands need location permission to access WLAN information"), so
|
||||
it fails in lockstep with tier 1 rather than covering for it. In the reported session the agent
|
||||
build already contained the netsh fallback and still produced no event.
|
||||
|
||||
Consequence for future work: **adding more netsh language labels cannot fix a missing SSID.** Every
|
||||
`wifi_signal_info` event that does exist carries a fully populated SSID — the failure is binary at
|
||||
the reader level, never a half-parsed or mis-decoded payload. A missing SSID is an access problem,
|
||||
not a parsing problem.
|
||||
|
||||
# The Tiered Reader
|
||||
|
||||
`WifiInfoProvider.TryRead` owns the order so the two collectors (`DeviceInfoCollector`,
|
||||
`NetworkChangeDetector`) cannot drift apart. First non-null wins:
|
||||
|
||||
| # | Reader | Yields | Location-gated |
|
||||
|---|---|---|---|
|
||||
| 1 | `WifiInfoReader` (native `wlanapi.dll`) | SSID, signal %, PHY, channel | yes |
|
||||
| 2 | `WinRtWifiSsidReader` (WinRT `GetConnectedSsid`) | SSID only | **no** |
|
||||
| 3 | `NetshWifiFallback` (process spawn, localized parse) | SSID, signal %, radio type, channel | yes |
|
||||
|
||||
Tier 2 is Microsoft's own documented replacement for this exact scenario:
|
||||
`Windows.Networking.Connectivity.WlanConnectionProfileDetails.GetConnectedSsid()`. It is not part of
|
||||
the BSSID-bearing surface, so it needs no consent, and it has existed since Windows 10 1507.
|
||||
|
||||
It is bound through the CLR's built-in WinRT projection with
|
||||
`Type.GetType("…, ContentType=WindowsRuntime")` — no NuGet package, no winmd reference, no csproj
|
||||
change — the same mechanism [OobeStateReader](../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/OobeStateReader.cs)
|
||||
already uses from the SYSTEM service context. Every step degrades to null rather than throwing,
|
||||
because the caller runs fire-and-forget inside `Task.Run`.
|
||||
|
||||
# Telling the Operator
|
||||
|
||||
A missing signal reading is indistinguishable from a bug unless the payload says why, and the
|
||||
device-level location setting is not something an admin would think to check. When tier 2 wins
|
||||
*and* the native tier reported `rc=5`, the `wifi_signal_info` payload therefore carries
|
||||
|
||||
```
|
||||
wifiDataLimitedReason = "location_services_off"
|
||||
```
|
||||
|
||||
and the event message gains `- signal unavailable, Location services off`, so the reason is visible
|
||||
in the timeline itself. The portal's WiFi card (`DeviceDetailsCard`) matches that literal and
|
||||
renders the explanation plus the Intune setting that fixes it. The value is a published contract —
|
||||
agent, portal and the MCP device-property catalog all match the same string.
|
||||
|
||||
The claim is only made on observed evidence: any other native failure (no WLAN service, no
|
||||
connected interface) leaves the field null rather than blaming a privacy setting we did not
|
||||
measure. `WifiInfoProvider.IsLocationGateDenied` is the single predicate, pinned by tests at its
|
||||
boundaries (`rc=5` yes; `rc=50`, `rc=5023`, `rc=1062` no).
|
||||
|
||||
The network timeline reads the same field. It needs no payload of its own: `buildSegments`
|
||||
already merges every `wifi_signal_info` into the WiFi segment covering its timestamp, so the
|
||||
reason rides in with the sample. A segment that never got a sample (a mid-session reconnect emits
|
||||
no `wifi_signal_info`) inherits it — the gate is a device-level setting and cannot flip during an
|
||||
enrollment — while a segment that *did* get a real reading is never overwritten. The segment
|
||||
tooltip then reads `Signal: unavailable - Location services off on this device` where the
|
||||
percentage would be, and the "Avg WiFi signal" stat says `n/a (Location services off)` instead of
|
||||
disappearing.
|
||||
|
||||
`network_state_change` deliberately does NOT carry the reason: that event has never carried
|
||||
`signalPercent` or `radioType` at all (`NetworkChangeDetector` collects them into its snapshot for
|
||||
change detection but only `before_/after_wifiSsid` reaches the payload), so there is no absent
|
||||
field there for a reason to explain. Adding it would ship a field nothing reads. The SSID it does
|
||||
carry is unaffected by the gate.
|
||||
|
||||
Fleet-wide the same field is queryable through
|
||||
`search_sessions(deviceProperties: { "wifi_signal_info.wifiDataLimitedReason": "location_services_off" })`,
|
||||
which sizes how much of a fleet is affected before anyone changes a policy.
|
||||
|
||||
Tier 2 returns the SSID alone. Signal quality, PHY type and channel have no ungated source; they
|
||||
stay null when the native tier is denied. That is deliberate — deriving a percentage from
|
||||
`ConnectionProfile.GetSignalBars()` (0–5) would fabricate precision the API does not have.
|
||||
|
||||
# Diagnostics
|
||||
|
||||
The step-level native return codes were already collected, but they were logged with
|
||||
`_logger.Debug` — and the agent's default log level is `Info`, so the diagnostic built to explain
|
||||
this failure never reached a single field log. Both collectors now log at **Warning** when all tiers
|
||||
fail, which they only reach when the active NIC really is WiFi. A native `rc=5` is spelled out as
|
||||
`ERROR_ACCESS_DENIED` with the location-gate explanation, so the log line diagnoses itself:
|
||||
|
||||
```
|
||||
EnrollmentTracker: no wifi_signal_info — no WiFi info from any reader — native: [<guid>]
|
||||
query(current_connection) rc=5; ; WinRT: no SSID; netsh: no parsable output. rc=5 is
|
||||
ERROR_ACCESS_DENIED — Windows 11 24H2+ gates the current-connection opcode (and netsh wlan)
|
||||
behind precise-location consent, which is off in this image; enable Location services to
|
||||
restore signal/PHY/channel.
|
||||
```
|
||||
|
||||
# Evidence
|
||||
|
||||
Session `7305e0ad` (HP EliteBook X Flip G1i, build 26200.9168, agent 2.0.1407, reported as
|
||||
`WIFI SSID fehlt`): `network_interface_info` shows the active NIC as `Intel(R) Wi-Fi 7 BE201`,
|
||||
`Wireless80211`, `connectionType=WiFi`, so the `isWifi` gate passed and collection ran — yet zero
|
||||
`wifi_signal_info` events across 214 events and 22 minutes, with the netsh fallback already shipped
|
||||
in that build.
|
||||
|
||||
Fleet correlation over WiFi sessions from 2026-08-01 (n=276), share with a `wifi_signal_info` event:
|
||||
|
||||
| OS build | WiFi sessions | with SSID |
|
||||
|---|---|---|
|
||||
| 22631 (23H2, pre-gate) | 3 | 100% |
|
||||
| 26100 (24H2) | 178 | 21% |
|
||||
| 26200 (25H2) | 95 | 45% |
|
||||
|
||||
Within a single model the outcome splits by fleet, not by hardware — ThinkPad T14 Gen 6: 7/91;
|
||||
ThinkPad E14 Gen 7: 9/9 — which is what a per-image privacy setting looks like.
|
||||
|
||||
Local verification (build 26220.9022): `HKLM\…\ConsentStore\location = Deny` while
|
||||
`HKCU\…\ConsentStore\location\NonPackaged = Allow`; the interactive user reads the SSID through
|
||||
both the native API and netsh, `HKU\S-1-5-18` has no consent value at all, and the WinRT chain
|
||||
resolves and returns the SSID under net48.
|
||||
|
||||
# Operator Guidance
|
||||
|
||||
A device that reports no SSID is not misconfigured for enrollment — only the telemetry field is
|
||||
missing. To get the full payload (signal, PHY, channel) back, the **device-level** location setting
|
||||
must be on: *Settings → Privacy & security → Location → Location services*, or via Intune
|
||||
(`Privacy/LetAppsAccessLocation`). Without it the SSID still arrives via tier 2.
|
||||
|
||||
# Citations
|
||||
|
||||
* [Changes to API behavior for Wi-Fi access and location](https://learn.microsoft.com/en-us/windows/win32/nativewifi/wi-fi-access-location-changes)
|
||||
* [WlanConnectionProfileDetails.GetConnectedSsid](https://learn.microsoft.com/en-us/uwp/api/windows.networking.connectivity.wlanconnectionprofiledetails.getconnectedssid)
|
||||
* [WifiInfoProvider.cs](../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/WifiInfoProvider.cs)
|
||||
* [WinRtWifiSsidReader.cs](../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/WinRtWifiSsidReader.cs)
|
||||
* [WifiInfoReader.cs](../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/WifiInfoReader.cs)
|
||||
* [NetshWifiFallback.cs](../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/NetshWifiFallback.cs)
|
||||
@@ -1,136 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Zero User-Targeted Apps — the starved user-phase completion gate
|
||||
description: Why a device with no user-targeted Intune apps could never satisfy the IME user-session gate, how that turned into a 30-minute esp_exit_without_completion_evidence false positive on a third of one tenant's enrollments, and the two fixes — reading the line IME actually writes in its zero-app branch, and refusing to fail a session while a completion gate is deliberately holding it.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime
|
||||
tags:
|
||||
- agent
|
||||
- ime
|
||||
- esp
|
||||
- completion
|
||||
- realmjoin
|
||||
- false-positive
|
||||
timestamp: 2026-08-25
|
||||
---
|
||||
|
||||
# Summary
|
||||
|
||||
A Classic user-driven enrollment whose user has **zero targeted Intune Win32 apps** could not
|
||||
produce the evidence every user-phase completion path required. The session parked, and the
|
||||
30-minute `advisory_completion` backstop resolved it as
|
||||
`enrollment_failed: esp_exit_without_completion_evidence` — while the user was at a working
|
||||
desktop, Windows Hello was provisioned, and RealmJoin was still installing packages.
|
||||
|
||||
One tenant hit this on **10 of 30 enrollments in a week**. Every one of them was a false
|
||||
positive; the devices were healthy in Intune and RealmJoin.
|
||||
|
||||
# The starvation chain
|
||||
|
||||
The IME writes `Completed user session N, userId: …` at the end of a user-session processing
|
||||
pass. `IME-USER-SESSION-COMPLETED` matches it and the adapter posts
|
||||
`ImeUserSessionCompleted`, which sets `DecisionState.ImeUserSessionCompletedUtc`.
|
||||
|
||||
But the IME never reaches that statement when the user has no assigned apps. Decompiled IME
|
||||
`1.104.102.0`, `Win32AppPlugIn/ApplicationPoller.cs`:
|
||||
|
||||
```csharp
|
||||
if (!appPolicies.Any())
|
||||
{
|
||||
AppWorkloadLog.TraceInformation($"[Win32App] Get 0 apps for user session {user.RDSSessionId}, user id = {user.IntuneId}");
|
||||
RegisterApplicationsInIfInESP(sender, user, delegate {
|
||||
espLockInProcessor.AttemptToLockInApplications(user, new List<ProcessorSubgraph>());
|
||||
});
|
||||
continue; // skips the "Completed user session" statement below
|
||||
}
|
||||
```
|
||||
|
||||
Identical shape in `1.97.107.0`, `1.83.107.0`, `1.80.132.0`. With that fact missing, every door
|
||||
out of the user phase is shut at once:
|
||||
|
||||
| Path | Requirement | Why it is dead |
|
||||
|---|---|---|
|
||||
| `ShouldTransitionToAwaitingHello` arm A | `AccountSetupProvisioningSucceededUtc` | ESP registry froze at `AccountSetup: 1 of 5`; neither `categorySucceeded` nor the all-subcategories fallback can ever hold |
|
||||
| arm B | `SkipUserEsp` | Full ESP, not a skip-user flow |
|
||||
| arm C | `IsImeUserSessionGenuine` | needs the missing IME fact |
|
||||
| arm D | `EnrollmentMode.DevicePreparation` | Autopilot v1, not WDP |
|
||||
| `MaybeSynthesizeAccountSetupCompleteFromSettledUserApps` | `AreUserEspAppsSettled` | returned `false` on an empty app list |
|
||||
| `HandleAdvisoryCompletionDeadlineFired` conjunction | `imeUserSessionGenuine \|\| isObservationAdvisory` | both false |
|
||||
|
||||
The last row is the one that produced the verdict: the backstop that exists to resolve a
|
||||
dead-end resolved it as a failure.
|
||||
|
||||
`AreUserEspAppsSettled` deserves its own note. Its "at least one required app" rule is
|
||||
deliberately conservative — an empty list normally means *the phase just cleared* or *apps have
|
||||
not surfaced yet*. But it made **zero apps**, the most settled state possible, indistinguishable
|
||||
from *unknown*, and there was no evidence available to tell them apart.
|
||||
|
||||
# Fix 1 — read the line IME does write
|
||||
|
||||
`IME-USER-SESSION-ZERO-APPS` matches the zero-app line and routes it through the **same**
|
||||
`OnUserSessionCompleted` callback. Downstream this genuinely *is* a completed user session, so
|
||||
the adapter's pending-apps deferral and its fire-once flag apply unchanged.
|
||||
|
||||
This is not a weakened gate — it is IME's own verdict. The zero-app branch locks the ESP in with
|
||||
an *empty* application list, and the sibling emit site in `AppWorkloadAbstraction.ExecuteUserCheckIn`
|
||||
returns `ProviderResult.ProvisioningComplete` straight after logging it. All three emit sites are
|
||||
user-session-scoped (`user.RDSSessionId`, `user.IntuneId`); none is device-scoped.
|
||||
|
||||
Guards that stay in force:
|
||||
|
||||
* **Phase guard** — the observation only fires while the tracker is in `AccountSetup`. Firing
|
||||
earlier would burn the adapter's fire-once flag on a pre-sign-in (defaultuser0) timestamp that
|
||||
`IsImeUserSessionGenuine` (`>= AccountSetupEnteredUtc`) can never accept. An observation made
|
||||
before the phase was known is remembered and replayed on the transition into it, because IME
|
||||
writes its check-in lines independently of when the agent first parses the phase marker.
|
||||
* **Engine guards unchanged** — arm C still requires a post-AccountSetup final ESP exit *and* a
|
||||
DAD-validated real-user desktop.
|
||||
* `AreUserEspAppsSettled` returns `true` for an empty list **only** with this explicit
|
||||
observation — never for a merely empty list.
|
||||
|
||||
# Fix 2 — do not fail while a completion gate is holding
|
||||
|
||||
An independent defect, and the decisive one for the affected tenant. `advisory_completion` runs
|
||||
for 30 minutes. The RealmJoin completion gate blocks completion until phase 110 and carries its
|
||||
*own* bounded resolution: 60 minutes from detection, re-armed on deployment activity, hard-capped
|
||||
at 4 hours. The shorter timer always won.
|
||||
|
||||
Session `75d6ae8e` installed **25 RealmJoin packages successfully inside the window**, the last
|
||||
one 2 minutes before the failure, while the engine's own `completion_waiting` still read
|
||||
`waiting on: realmjoin_resolution`.
|
||||
|
||||
`HandleAdvisoryCompletionDeadlineFired` now re-arms instead of failing when the RealmJoin gate is
|
||||
closed *and* its deadline is still armed, emitting `completion_waiting` with trigger
|
||||
`DeadlineFired:advisory_completion:CompletionGateHolding`.
|
||||
|
||||
Convergence comes from the gate itself: `Resolved`, `FirstDeploymentIncomplete` and `Timeout` all
|
||||
set `RealmJoinFacts.Outcome`, which opens `RealmJoinGateOpen` and stops the guard applying.
|
||||
Requiring the gate's deadline to still be armed keeps a gate that can no longer resolve itself
|
||||
from parking the session; `AgentMaxLifetimeMinutes` bounds it regardless.
|
||||
|
||||
# Timeline evidence
|
||||
|
||||
`ime_user_session_completed` carries an `evidence` field so the basis of the verdict is legible
|
||||
without the diagnostics archive:
|
||||
|
||||
* `user_session_completed` — IME wrote its completion line and no required app was pending.
|
||||
* `zero_user_apps_enumerated` — IME reported zero targeted apps; the user phase had nothing to
|
||||
enforce.
|
||||
|
||||
No new event type: the fact is the same, only the observation behind it differs.
|
||||
|
||||
# Deployment
|
||||
|
||||
The pattern ships through the config API and reaches existing agents without a rebuild; an agent
|
||||
that does not know the `userSessionZeroApps` action logs one Debug line and ignores it. The
|
||||
*behaviour* needs an agent release. Sessions already stamped `Failed` are not reclassified — the
|
||||
backend mirrors the agent verdict (`VerdictPaths.AgentFailed`) and has no independent judgement
|
||||
here by design.
|
||||
|
||||
# Citations
|
||||
|
||||
* `rules/ime-log-patterns/IME-USER-SESSION-ZERO-APPS.json`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.Handlers.cs` — `HandleUserSessionZeroApps`, the phase replay in `HandleEspPhaseDetected`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.cs` — `AreUserEspAppsSettled`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs` — `EmitUserSessionCompleted`, evidence labels
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs` — `HandleAdvisoryCompletionDeadlineFired`
|
||||
* [decision-engine.md](decision-engine.md), [post-reboot-completion-recovery.md](post-reboot-completion-recovery.md)
|
||||
@@ -1,135 +0,0 @@
|
||||
---
|
||||
type: Architecture Guide
|
||||
title: Autopilot Monitor – Architecture Guide
|
||||
description: High-level architecture overview and solution layout for contributors.
|
||||
tags:
|
||||
- architecture
|
||||
- agent
|
||||
- backend
|
||||
- web
|
||||
- mcp
|
||||
timestamp: 2026-07-06T21:39:29+02:00
|
||||
---
|
||||
|
||||
# Autopilot Monitor – Architecture Guide
|
||||
|
||||
> High-level architecture overview and solution layout for contributors.
|
||||
> Full product documentation lives at https://docs.autopilotmonitor.com.
|
||||
|
||||
---
|
||||
|
||||
## High-Level Overview
|
||||
|
||||
```text
|
||||
+----------------------------------------------------------------------------------+
|
||||
| Autopilot Monitor - Architecture |
|
||||
+----------------------------------------------------------------------------------+
|
||||
|
||||
[Admin/User Browser]
|
||||
|
|
||||
| HTTPS
|
||||
v
|
||||
+-----------------------------+ OIDC +-------------------------------+
|
||||
| Web App (Next.js) |<---------------->| Microsoft Identity (Entra ID) |
|
||||
| Dashboard / Progress | | Auth / Authorization |
|
||||
+-----------------------------+ +-------------------------------+
|
||||
^ ^
|
||||
| |
|
||||
| | Live updates
|
||||
| |
|
||||
| | +---------------------------+
|
||||
| +-------------------------| SignalR Hub |
|
||||
| | Real-time event stream |
|
||||
| +---------------------------+
|
||||
| ^
|
||||
| REST / API |
|
||||
v | events
|
||||
+---------------------------+ Data & |
|
||||
| Azure Functions Backend |------> Integrations ---+
|
||||
| APIs / Ingestion / Rules | \
|
||||
+---------------------------+ \ +---------------------------+
|
||||
^ +----->| Storage Tables |
|
||||
| mTLS | Sessions / Events / Rules |
|
||||
| telemetry / events +---------------------------+
|
||||
| +---------------------------+
|
||||
+---------------------------+ | Notifications |
|
||||
| Agent on Windows Devices | | Teams / Slack / Webhooks |
|
||||
| Enrollment + app progress | +---------------------------+
|
||||
+---------------------------+
|
||||
+---------------------------+
|
||||
| MCP Server |
|
||||
| Azure Container App |
|
||||
+---------------------------+
|
||||
```
|
||||
|
||||
**Core Data Flow:**
|
||||
```
|
||||
Device (Agent) ──JSON telemetry+gzip──► Azure Functions ──► Azure Table Storage
|
||||
│ │
|
||||
├── SignalR ─────────►│
|
||||
│ │
|
||||
Web Dashboard ◄── REST API ◄──┘
|
||||
```
|
||||
|
||||
| Component | Technology | Purpose |
|
||||
|-----------|------------|---------|
|
||||
| **Agent** | .NET Framework 4.8 (Windows) | Runs on devices during enrollment, collects telemetry |
|
||||
| **SummaryDialog** | .NET Framework 4.8 (WPF) | Post-enrollment summary UI (outcome, app timeline, auto-close) |
|
||||
| **Backend** | Azure Functions .NET 8 (Isolated Worker) | REST API, event processing, rule engine, storage |
|
||||
| **Web** | Next.js 15 + TypeScript + React 18 | Dashboard, settings, analytics, real-time UI |
|
||||
| **Shared** | .NET Standard 2.0 | DTOs, models, enums, constants shared between Agent & Backend |
|
||||
| **MCP Server** | Azure Container App | Model Context Protocol server for AI-assisted analysis |
|
||||
|
||||
---
|
||||
|
||||
## Solution Structure
|
||||
|
||||
```
|
||||
AutopilotMonitor.sln
|
||||
├── src/
|
||||
│ ├── Shared/
|
||||
│ │ ├── AutopilotMonitor.Shared/ (netstandard2.0 – DTOs, models, enums, constants)
|
||||
│ │ └── AutopilotMonitor.DecisionCore/ (netstandard2.0 – completion decision engine)
|
||||
│ ├── Agent/
|
||||
│ │ ├── AutopilotMonitor.Agent.V2/ (net48, Exe – entry point; output named AutopilotMonitor.Agent.exe)
|
||||
│ │ ├── AutopilotMonitor.Agent.V2.Core/ (net48, Lib – core logic)
|
||||
│ │ ├── AutopilotMonitor.Agent.V2.Core.Tests/ (net48, xUnit – agent unit tests)
|
||||
│ │ ├── AutopilotMonitor.DecisionCore.Tests/ (net48, xUnit – decision engine tests)
|
||||
│ │ ├── AutopilotMonitor.SummaryDialog/ (net48, WPF – enrollment summary UI)
|
||||
│ │ └── AutopilotMonitor.SummaryDialog.Tests/ (net48, xUnit)
|
||||
│ ├── Backend/
|
||||
│ │ ├── AutopilotMonitor.Functions/ (net8.0, Azure Functions v4)
|
||||
│ │ └── AutopilotMonitor.Functions.Tests/ (net8.0, xUnit)
|
||||
│ ├── McpServer/
|
||||
│ │ └── autopilot-monitor-mcp/ (TypeScript/Node – MCP server; deployed as Azure Container App)
|
||||
│ └── Web/
|
||||
│ └── autopilot-monitor-web/ (Next.js 15, TypeScript)
|
||||
├── rules/
|
||||
│ ├── gather/ Individual gather rule JSONs
|
||||
│ ├── analyze/ Individual analyze rule JSONs
|
||||
│ ├── ime-log-patterns/ IME regex pattern JSONs
|
||||
│ ├── schema/ JSON Schema definitions
|
||||
│ ├── guardrails.json Gather-rule guardrail allow-lists (source)
|
||||
│ ├── scripts/ combine.js
|
||||
│ └── dist/ Combined output (embedded in Functions)
|
||||
├── infra/ Bicep templates (MCP server)
|
||||
├── scripts/
|
||||
│ ├── Backup/ Table backup/restore tooling
|
||||
│ ├── Bootstrap/ Intune deployment scripts
|
||||
│ ├── CustomerSetup/ Customer onboarding helpers
|
||||
│ └── Deployment/ CI/CD build scripts
|
||||
└── .github/workflows/ CI/CD pipelines
|
||||
```
|
||||
|
||||
**Project References:**
|
||||
```
|
||||
Shared + DecisionCore ◄── Agent.V2.Core ◄── Agent.V2
|
||||
◄── Agent.V2.Core.Tests
|
||||
Shared + DecisionCore ◄── DecisionCore.Tests
|
||||
Shared + DecisionCore ◄── Functions ◄── Functions.Tests
|
||||
SummaryDialog ◄── SummaryDialog.Tests (standalone; compiled alongside the agent build)
|
||||
Web (independent – communicates via REST + SignalR)
|
||||
McpServer (independent – talks to the backend REST API)
|
||||
```
|
||||
|
||||
---
|
||||
@@ -1,184 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Admin Identity Binding — tid + oid behind every cross-tenant-role UPN
|
||||
description: Why a GlobalAdmins, DelegatedAdmins or McpUsers row keyed on a UPN string must never confer access by itself in an API that accepts tokens from every Entra tenant, the AdminIdentityBindings table that pins each such UPN to its immutable home tenant id and object id, the fail-closed resolution order in the role services and the MCP whitelist, the first-sign-in object-id pin, and the explicit rebind surface.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/AdminIdentityBindingService.cs
|
||||
tags:
|
||||
- backend
|
||||
- security
|
||||
- authorization
|
||||
- multi-tenant
|
||||
- global-admin
|
||||
- delegated-admin
|
||||
- mcp
|
||||
timestamp: 2026-08-30T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
The API is a multi-tenant Entra application on purpose: `AuthenticationMiddleware` accepts a token from
|
||||
any tenant (issuer prefix `login.microsoftonline.com/` or `sts.windows.net/`), and signature, audience,
|
||||
lifetime and algorithm are validated strictly. Tenant-member roles (`TenantAdmins`) are keyed on
|
||||
`(tid, upn)` and therefore bound to the verified tenant.
|
||||
|
||||
The two cross-tenant tiers were not. `GlobalAdmins` (GlobalAdmin / GlobalReader) and the delegated MSP
|
||||
tier (`DelegatedAdmins` + `TenantGroupAssignments`) resolved their role from the **UPN string alone**
|
||||
(`upn`, falling back to `preferred_username`). Both claims are mutable and reusable across tenants:
|
||||
|
||||
* **Domain re-registration**: if the domain behind a platform admin's UPN lapses or moves, whoever
|
||||
verifies it in a fresh tenant can mint the identical UPN; the token is genuine, only `tid` differs.
|
||||
* **UPN recycling**: a User Administrator in the home tenant can re-assign the UPN to another account;
|
||||
same `tid`, different `oid`.
|
||||
* **MSP variant**: the delegated tier was gated only on the caller's home tenant being Pro — which any
|
||||
tenant can self-provision via the trial endpoint.
|
||||
|
||||
Any of these yielded full platform scope: raw table dumps including `TenantConfiguration` secrets,
|
||||
tenant offboarding, GA management. Microsoft's own guidance (post-nOAuth) is explicit: authorization
|
||||
data must be keyed on the immutable `tid` + `oid` pair, never on `upn` / `preferred_username`.
|
||||
|
||||
# Mechanism
|
||||
|
||||
## One binding per UPN
|
||||
|
||||
`AdminIdentityBindings` (PK `Bindings`, RK = lowercase UPN) holds, for every UPN that carries a
|
||||
cross-tenant role, the identity that may use it:
|
||||
|
||||
| Column | Meaning |
|
||||
| --- | --- |
|
||||
| `TenantId` | The admin's **home** Entra tenant — must equal the JWT `tid`. Mandatory at grant time. |
|
||||
| `ObjectId` | The admin's Entra object id — must equal the JWT `oid`. Empty until pinned. |
|
||||
| `BoundBy`, `BoundDate` | Operator provenance. |
|
||||
| `ObjectIdPinnedDate` | When the object id was fixed (grant time, or the first matching sign-in). |
|
||||
|
||||
A single table rather than columns on three role tables: a UPN string belongs to exactly one identity
|
||||
at a time, a person may hold a GlobalAdmins row *and* delegated assignments, and the role rows stay
|
||||
what they are — grants. The binding says whose grants they are.
|
||||
|
||||
## Resolution order (both role services)
|
||||
|
||||
`AdminIdentity` is the record `(Upn, TenantId, ObjectId)` built from a validated principal; it is
|
||||
`null` when any of the three claims is missing (app-only tokens, foreign IdP shapes). Both
|
||||
`GlobalAdminService.GetGlobalRoleAsync(AdminIdentity?)` and
|
||||
`DelegatedAdminService.GetScopeAsync(AdminIdentity?)` follow the same order:
|
||||
|
||||
1. `null` identity → no role, no storage read.
|
||||
2. Resolve the **row** by UPN (cached 30 s, unchanged from before).
|
||||
3. No row / empty scope → return; the binding is not even read. Ordinary tenant users cost one cached
|
||||
read and never produce a binding log line.
|
||||
4. `AdminIdentityBindingService.IsBoundAsync(identity)` → false ⇒ no role / empty scope.
|
||||
5. Delegated only: the Pro home-tenant entitlement gate, **after** the binding check — an unbound caller
|
||||
never triggers an edition lookup.
|
||||
|
||||
`IsBoundAsync` is the single choke-point: no binding ⇒ false; `TenantId` mismatch ⇒ false (checked
|
||||
before anything else, so a foreign-tenant token can never claim the pin below); pinned `ObjectId`
|
||||
mismatch ⇒ false; unpinned binding + matching tenant ⇒ **pin** the caller's `oid` and verify against
|
||||
what was actually stored afterwards. Every false outcome is logged at Warning with its reason
|
||||
(`[IdentityBinding] TENANT MISMATCH …` / `OBJECT-ID MISMATCH …` / `holds no identity binding`), the
|
||||
one level the worker forwards to Application Insights; matches are silent. Pins are logged at Warning
|
||||
once — they are one-off, operator-relevant events.
|
||||
|
||||
The pin is a conditional update in the repository (`TryPinIdentityObjectIdAsync`: only onto an unpinned
|
||||
row homed in the caller's tenant, ETag-guarded; a 412 re-reads and the caller compares against the
|
||||
winner). Two accounts racing the first sign-in inside the home tenant therefore leave exactly one bound.
|
||||
|
||||
Row lookups stay cached by UPN; the binding has its own 30 s cache keyed by UPN and is invalidated on
|
||||
pin, rebind and removal. A rebind on one scaled-out instance converges on the others within seconds.
|
||||
|
||||
## The MCP whitelist is a third UPN-keyed grant
|
||||
|
||||
`McpUsers` (PK `McpUsers`, RK = lowercase UPN) is the WhitelistOnly allow-list consulted by
|
||||
`GET auth/mcp`, which runs under the AuthenticatedUser policy — a token from any Entra tenant reaches it.
|
||||
Before 2026-08-30 the row was honoured by UPN alone, so a same-UPN identity from another tenant
|
||||
received `allowed=true` / `accessGrant=McpUser`, inherited the row's `UsagePlan` (a paid quota tier
|
||||
provisioned for another organization's user), and was also hit by the row's Disabled kill-switch. The
|
||||
data reach stayed tenant-bound (the MCP server routes a non-global caller on its own JWT `tid`), but the
|
||||
GA-curated whitelist and the per-user entitlement were bypassable.
|
||||
|
||||
`McpUserService` now resolves the whitelist exactly like the role services: row by UPN (cached 30 s) →
|
||||
no row ⇒ `NotPresent` without touching the binding → `IsBoundAsync(identity)` false ⇒ `NotPresent`. Grant,
|
||||
usage-plan override (`GetBoundMcpUserAsync`, used by `McpQuotaService.ResolvePlanAsync` and the
|
||||
self-service usage view) and the Disabled kill-switch all belong to the bound identity only; an unbound
|
||||
legacy row is inert. `POST global/mcp-users` binds first via the same resolver / 422 / 409 contract as the
|
||||
other grant endpoints, and the MCP users settings section shows the same home-tenant picker on 422. The
|
||||
AllMembers path is unaffected — it is tenant-bound by the auth middleware itself.
|
||||
|
||||
## Grants bind first — and resolve the identity themselves
|
||||
|
||||
Granting is "type the UPN". The four grant endpoints (`POST auth/global-admins`,
|
||||
`POST global/delegated-admins`, `POST global/tenant-groups/{id}/assignees`, `POST global/mcp-users`) accept `homeTenantId` /
|
||||
`objectId` only as optional overrides; normally `AdminIdentityResolver` supplies them:
|
||||
|
||||
1. **Sign-in history** (`UserActivity`, filtered on the UPN): the person has signed in before ⇒ tid + oid
|
||||
are known, both from validated tokens. The most recent sign-in wins for the oid. A UPN seen under
|
||||
**more than one tenant** is never auto-resolved — that is the attack shape the binding exists for.
|
||||
2. **UPN domain → onboarded tenant** (`TenantConfiguration.DomainName`, exactly one match): tid only; the
|
||||
oid is pinned on the first sign-in.
|
||||
3. Neither ⇒ **422 `HomeTenantUnresolved`**; the portal then shows a home-tenant picker (onboarded
|
||||
tenants) and retries with `homeTenantId`. This is the only manual step, and only for a UPN whose
|
||||
domain the platform has never seen.
|
||||
|
||||
`AddGlobalAdminAsync`, `DelegatedAdminService.UpsertAsync`, `AssignGroupAsync` and `AddMcpUserAsync` then take the resolved
|
||||
`homeTenantId` / `objectId` and call `EnsureBoundAsync` **before** writing the role row. `EnsureBoundAsync` creates the binding, keeps a compatible one (same tenant; the
|
||||
supplied object id equals the pin or upgrades an unpinned one), and throws
|
||||
`IdentityBindingConflictException` for a different tenant or a different pinned object id — the
|
||||
functions map that to **409**. A grant can therefore never silently re-home a UPN.
|
||||
|
||||
Why the object id may be unknown: a person who has never signed in has no history to resolve from, and
|
||||
the operator granting an external MSP admin has no directory access to the partner tenant. The first
|
||||
sign-in from the bound tenant pins it; from then on a recycled UPN in the same tenant is refused.
|
||||
|
||||
## Explicit rebind surface
|
||||
|
||||
| Route | Policy | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `GET global/identity-bindings` | GlobalReadOrAdmin | Audit which tenant / object id each grant is usable from. |
|
||||
| `PUT global/identity-bindings/{upn}` | GlobalAdminOnly | Replace the binding: move a UPN to another tenant, or re-pin after a legitimate account re-creation (omit `objectId` to clear the pin). |
|
||||
| `DELETE global/identity-bindings/{upn}` | GlobalAdminOnly | Make every role row of the UPN inert without touching the rows. Self-removal is refused. |
|
||||
|
||||
Both mutations are audited under the binding's home tenant (`AdminIdentityBinding` entity, previous
|
||||
values in the details) and logged at Warning. The portal surfaces this in **Tenant Management → Edit
|
||||
Tenant Configuration → Identity bindings** (collapsed): the bindings homed in that tenant, with inline
|
||||
edit (re-home / re-pin; empty object id ⇒ re-pinned on the next sign-in) and remove. The grant forms
|
||||
themselves carry no identity fields — the binding is a background property of the grant.
|
||||
|
||||
## Lifecycle and secondary effects
|
||||
|
||||
* `AdminIdentityBindings` is in `TableNames.All`, in `CriticalBackupTables.All` and in
|
||||
`AuthTablesFullRestoreForbidden` (single-row restore only, like the other auth tables).
|
||||
* Tenant offboarding wipes bindings whose `TenantId` is the offboarded tenant (property-only bucket):
|
||||
once a home tenant is gone, its admins' grants become inert.
|
||||
* `RequestContext.ObjectId` carries the JWT `oid`; `AdminIdentity.FromRequestContext` serves the
|
||||
handlers that resolve roles outside the middleware (SignalR group join/leave, health checks).
|
||||
* The throttle identity (`GetCallerId`) now prefers `oid` over the UPN, so two accounts in different
|
||||
tenants that share a UPN string no longer share a rate-limit bucket. Audit and presence rows keep the
|
||||
UPN as their display key; they already carry the tenant id.
|
||||
* Legacy role rows without a binding are **inert, not grandfathered**. Rolling this out therefore means
|
||||
seeding the binding rows for the existing admin UPNs (their `tid` and `oid` are recorded per login in
|
||||
`UserActivity`) *before* the backend deploy — an unseeded platform admin locks themselves out of every
|
||||
GlobalAdminOnly route, including the rebind surface.
|
||||
|
||||
# Examples
|
||||
|
||||
A GlobalAdmin row for `admin@vendor.example` bound to tenant `1111…` with object id `aaaa…`:
|
||||
|
||||
| Token | Result |
|
||||
| --- | --- |
|
||||
| `tid=1111… oid=aaaa… upn=admin@vendor.example` | GlobalAdmin |
|
||||
| `tid=2222… oid=aaaa… upn=admin@vendor.example` | no role — `TENANT MISMATCH` warning |
|
||||
| `tid=1111… oid=bbbb… upn=admin@vendor.example` | no role — `OBJECT-ID MISMATCH` warning |
|
||||
| `tid=1111… upn=admin@vendor.example` (no `oid`) | no role — identity incomplete |
|
||||
|
||||
Same row, binding without an object id: the first `tid=1111…` sign-in pins its `oid` and is admitted; a
|
||||
`tid=2222…` sign-in before that is refused **and does not pin**.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/AdminIdentity.cs` — the identity record and its factories.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/AdminIdentityBindingService.cs` — verdict, pin, grant-time binding, rebind; the table entity.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/AdminIdentityResolver.cs` — sign-in-history / UPN-domain resolution at grant time (`AdminIdentityResolverTests`).
|
||||
* `src/Web/autopilot-monitor-web/app/admin/components/IdentityBindingsSection.tsx` — the per-tenant inspect/correct panel.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/GlobalAdminService.cs`, `DelegatedAdminService.cs`, `McpUserService.cs` — resolution order (`McpUserServiceIdentityBindingTests` for the whitelist matrix).
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableAdminRepository.cs` — `TryPinIdentityObjectIdAsync` and the binding CRUD.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Admin/IdentityBindingManagementFunction.cs` — the rebind surface and the shared `IdentityBindingRequest.Validate`.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/AdminIdentityBindingServiceTests.cs`, `AdminIdentityBindingAuthorizationTests.cs` — verdict matrix, race semantics, and the middleware-level denials for foreign-tenant and recycled-UPN tokens.
|
||||
* [Client-Certificate Tenant Binding](client-cert-tenant-binding.md) — the same "chain-valid is not tenant-bound" argument on the device side.
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Attempt-Scoped App Install Durations
|
||||
description: AppInstallSummary.DurationSeconds measures the status-defining (last) install attempt via a LastAttemptStartedAt max-fold anchor — replacing the first-observation-to-last-terminal span that billed multi-pass IME processing gaps to trivial apps; time attribution counts the union of active attempt segments.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.AppInstall.cs
|
||||
tags: [backend, agent, ingest, app-installs, metrics, time-attribution, durations]
|
||||
timestamp: 2026-08-21
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
The IME processes the ESP app list in multiple passes: a device-ESP evaluation pass can
|
||||
touch an app (often ending `Skipped`) long before the real install runs in a later pass.
|
||||
`AppInstallSummary` is keyed by `{SessionId}_{AppName}` and folded from ALL of the app's
|
||||
events, so `DurationSeconds = last terminal − first observation` spanned those passes.
|
||||
Field case (session `44862cd3`, "Set TimeZone"): evaluation pass 09:16, real install
|
||||
10:22:58 → 10:23:21 — reported 4002 s for a 23-second install. Fleet averages, the
|
||||
Slowest-Apps ranking, p95s, and the version-regression radar all inherited the inflation;
|
||||
the rule engine's `app_install_duration` (already attempt-scoped) contradicted the
|
||||
dashboard for the same install.
|
||||
|
||||
# Schema
|
||||
|
||||
Three durable anchors on the row — each with a distinct, non-interchangeable job:
|
||||
|
||||
| Column | Fold | Role |
|
||||
|---|---|---|
|
||||
| `StartedAt` | min (earliest wins) | First observation. Server-side window/bucket filter of every AppInstallSummaries scan and the radar's version-ordering key. Never re-anchored. |
|
||||
| `LastAttemptStartedAt` | **max** (in-batch and in `ReconcileAppInstallSummaryWithExisting`) | Anchor of `DurationSeconds`. Null = sentinel (row predates 2026-08, or no start observed) → duration falls back to the historical `StartedAt` span. Replay of an older batch can never regress it. |
|
||||
| `InstallPassCount` | additive, guarded by `LastAttemptStartedAt` monotonicity (a batch whose newest start is not newer than the stored anchor adds 0) | Explains "finished long after first seen" (web ×N chip); 2+ is normal. |
|
||||
|
||||
`DurationSeconds = CompletedAt − LastAttemptStartedAt`, computed at the terminal event and
|
||||
re-derived in one central reconcile site covering in-order and Q4 out-of-order batches.
|
||||
A `LastAttemptStartedAt` newer than `CompletedAt` means a fresh pass is in flight — the
|
||||
completed attempt's stored duration stands (the 0 value omits the column; Merge preserves).
|
||||
`DownloadDurationSeconds` pairs the latest `app_download_started` with the NEXT
|
||||
`app_install_started` (attempt-scoped, in-batch). The full span stays derivable as
|
||||
`CompletedAt − StartedAt` — deliberately not a column.
|
||||
|
||||
Weaker-terminal guard (both in-batch and cross-batch): `app_install_completed` with
|
||||
state `Skipped`/`Postponed` never overrides an `Installed`/`Error` terminal, nor its
|
||||
`CompletedAt`/`DurationSeconds` — mirroring the long-pinned `app_install_skipped` rule.
|
||||
|
||||
# Consumers
|
||||
|
||||
* **Fleet metrics / apps dashboard / PDF** (`MetricsMath`, `AppsAnalyticsHelper`): values
|
||||
drop to attempt scale; labels say "install time (final attempt)". The 6 h
|
||||
`MaxPlausibleInstallDurationSeconds` gate remains (an attempt longer than the agent's
|
||||
lifetime is still unobservable).
|
||||
* **AppVersionRegressionRadar**: cutover gate in `MeasuredAppGroups` — only rows with
|
||||
`LastAttemptStartedAt` enter the duration population, on BOTH comparison sides. A mixed
|
||||
population would mask real regressions and falsely re-arm active episodes for up to the
|
||||
35-day horizon. `MinMeasuredInstalls` keeps it silent until enough post-cutover rows
|
||||
exist. Thresholds (2.0× / 300 s) were tuned on span data — re-evaluate ~4 weeks after
|
||||
rollout.
|
||||
* **Time attribution** (`TimeAttributionCalculator.BuildBlockingAppIntervals`):
|
||||
sequence-ordered start→terminal pairing yields one segment per pass/attempt. Per-app
|
||||
seconds = SUM of clamped segments (failed attempts and evaluation passes count — they
|
||||
occupied the path); occupancy merges the segments, never the per-app hulls; hull
|
||||
endpoints remain first-start→last-terminal for display and what-if. Deliberately
|
||||
BROADER than the metrics duration: active time across all passes vs. final attempt.
|
||||
* **Agent payloads** (`AppInstallTiming` / `ImeLogTrackerAdapter.UpdateAppTiming`): a
|
||||
terminal→active transition re-arms BOTH endpoints, so event `durationSeconds` also
|
||||
describes the attempt (needs the agent release carrying this change).
|
||||
* **Rule engine** `app_install_duration`: was already attempt-scoped — now consistent.
|
||||
|
||||
# Rollout
|
||||
|
||||
No backfill (user decision, analog to the CMTrace legacy): pre-cutover rows keep span
|
||||
values, so time-series charts step down visibly at the changeover (documented in the
|
||||
public platform changelog). Sentinel-gated columns follow the standard chain:
|
||||
`BuildAppInstallSummaryEntity` + `MapToAppInstallSummary` + `AppsDashboardProjection`
|
||||
(the only projection whose consumers read the new columns).
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.AppInstall.cs` — fold + weaker-terminal guard + attempt pairing
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Metrics.cs` — reconcile max-fold, pass-count guard, central duration recompute
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/AppVersionRegressionRadar.cs` — cutover gate
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs` — per-pass segments + occupancy
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs` — payload timing re-arm
|
||||
* Tests: `AppInstallTerminalStateAndReconcileTests`, `EventTimestampValidationTests`, `AppVersionRegressionRadarTests`, `TimeAttributionCalculatorTests`
|
||||
@@ -1,98 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: App Version Duration Regression Radar
|
||||
description: How the radar detects app versions whose median install duration regressed vs. the previous version (median lift ≥2x plus an absolute floor over first-seen-ordered versions), how alert episodes live in the appversionregression tracker keyspace, and which surfaces show them.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/AppVersionRegressionRadar.cs
|
||||
tags:
|
||||
- backend
|
||||
- apps
|
||||
- regression-radar
|
||||
- insights
|
||||
timestamp: 2026-08-13T12:00:00+02:00
|
||||
---
|
||||
|
||||
# App Version Duration Regression Radar
|
||||
|
||||
Detects "version X made this app's installs much slower" — the concrete, quotable insight
|
||||
("median install duration rose from 11 to 29 min after version X") — from the per-(session,
|
||||
app) `AppInstallSummaries` rows. Duration only: failure-rate per version is already visible
|
||||
in the apps dashboard's `versionBreakdown`, and rate regressions belong to Wilson-interval
|
||||
statistics, not medians.
|
||||
|
||||
# Schema
|
||||
|
||||
## Signal & detection
|
||||
|
||||
Per (tenant, app), the MEASURED install population is: `Status == "Succeeded"`, terminal
|
||||
state not Skipped/Postponed, `MetricsMath.HasMeasuredDuration` (1s..6h — 0s means the start
|
||||
was unobserved, above 6h means back-stamped batches), no `AppIdCollision`, non-empty
|
||||
`AppVersion`. Versions are ordered by FIRST-SEEN install time (min `StartedAt`), never by
|
||||
string comparison ("9.1" vs "2024.10" would lie). The comparison pair is always the newest
|
||||
version (the one with the most recent measured install) vs. the version with the latest
|
||||
first-seen strictly before it.
|
||||
|
||||
MEDIANS, not means (nearest-rank, shared with the apps dashboard via
|
||||
`AppsAnalyticsHelper.Percentile`) — one back-stamped straggler must not fire a fleet alert.
|
||||
A regression fires only when ALL hold over the trailing 35-day horizon:
|
||||
|
||||
1. both versions have ≥ 10 measured installs;
|
||||
2. current median ≥ 2× the previous version's median;
|
||||
3. absolute increase ≥ 300 s (a 20s→45s app is noise, not an incident).
|
||||
|
||||
## Suppression & limits
|
||||
|
||||
* Parallel ring rollouts can make "previous" a concurrently-deployed version — accepted in
|
||||
v1; the episode key (app, current version) still caps it at one bell per version.
|
||||
* Platform kill switch: app setting `AppVersionRegressionRadarDisabled=true` skips the
|
||||
radar (fail-open — it only notifies, never mutates data).
|
||||
* One cross-tenant install-summary read per pass (the same query the global apps dashboard
|
||||
runs), grouped by tenant; per-tenant failures are non-fatal.
|
||||
|
||||
## Alert episodes (tracker keyspace)
|
||||
|
||||
An episode lives as `appversionregression|{app-lower}|{version-lower}` (table-key-sanitized;
|
||||
raw AppName/CurrentVersion live in columns, the mapper never parses the RowKey) in the
|
||||
notification-tracker table, next to the `ruleregression|` keyspace: the row IS the dedup
|
||||
(bell + ops event fire exactly once per episode) and the `versionRegressions[]` payload —
|
||||
both medians, both sample counts, lift. Numbers refresh on every radar pass while the
|
||||
episode stays active; `FirstNotifiedAt` never moves. The episode closes (dedup re-arms) when
|
||||
the alerted version drains under 10 measured installs in the horizon or its recomputed
|
||||
median falls back under 1.5× the previous version's median; the tracker's 30-day retention
|
||||
sweep also re-arms, so a month-old still-burning regression rings once more by design.
|
||||
|
||||
## Notification chain (v1: bell + ops event)
|
||||
|
||||
* Tenant bell `app_version_duration_regression` (audience: Admin), href
|
||||
`/apps/detail?name={app}&days=35` (canonical `appDetailUrl` shape). Message carries the
|
||||
full numbers so the admin can verify without a portal round-trip.
|
||||
* Ops event `AppVersionDurationRegression` (Tenant category, Warning) — dual-registered in
|
||||
the web `OPS_EVENT_TYPES` catalog so operators can wire Telegram/Teams/Slack alert rules.
|
||||
|
||||
## Surfaces
|
||||
|
||||
* `apps/{appName}/analytics` (+ global twin with `?tenantId=`) responds with
|
||||
`versionRegressions[]` (empty for the cross-tenant aggregate — episodes are per-tenant
|
||||
rows), and `versionBreakdown[]` rows now carry `measuredInstalls` /
|
||||
`medianDurationSeconds` / `p95DurationSeconds`.
|
||||
* Apps detail page: amber "↑ Duration regression" banner above the version cards and a
|
||||
"Median Install Duration by Version" bar chart.
|
||||
|
||||
# Examples
|
||||
|
||||
A tenant ships Contoso VPN 2.4.0 after 2.3.9. 2.3.9 has 40 measured installs, median 660s;
|
||||
2.4.0 accumulates 12 measured installs with median 1740s. Lift 2.6×, absolute +1080s →
|
||||
episode `appversionregression|contoso vpn|2.4.0` opens, one bell + one ops event fire.
|
||||
Subsequent passes refresh the numbers only. When 2.4.1 replaces 2.4.0 and 2.4.0 drains
|
||||
below 10 measured installs in the horizon, the episode deletes and the dedup re-arms.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/AppVersionRegressionRadar.cs` — pure core
|
||||
(gates, first-seen ordering, re-arm).
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.AppVersionRegression.cs`
|
||||
— orchestration, kill switch, bell/ops-event emission, wording contract.
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableHardwareRejectionNotificationTracker.cs`
|
||||
— episode keyspace, entity round-trip.
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppVersionRegressionAlert.cs` — persisted
|
||||
alert payload.
|
||||
* [Rule Regression Radar](rule-regression-radar.md) — the sibling pattern this radar mirrors.
|
||||
@@ -1,95 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Business Timestamps vs. the Azure Tables System Timestamp
|
||||
description: Why the system Timestamp is never authoritative, how the 2026-07-18 migration exposed that, and the OccurredUtc + RowKey-decode compensation (mappers, RowKey-range date filters, retention, backfill).
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Helpers/BusinessTimestamp.cs
|
||||
tags:
|
||||
- backend
|
||||
- table-storage
|
||||
- audit-log
|
||||
- ops-events
|
||||
- events
|
||||
- migration
|
||||
timestamp: 2026-07-20T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
Azure Table Storage's `Timestamp` is a **system property**: the server ignores any supplied
|
||||
value on write and stamps the row's *write* time; every row rewrite (storage-account
|
||||
migration, MERGE) resets it. Three tables nevertheless treated it as the business event
|
||||
time — writes like `["Timestamp"] = evt.Timestamp` were silent no-ops, unnoticed because
|
||||
write time ≈ event time in normal operation.
|
||||
|
||||
The 2026-07-18 EU storage migration rewrote every row, so all migrated rows in
|
||||
**AuditLogs**, **OpsEvents**, and **Events** read as "2026-07-18 ~13:58 UTC": displays were
|
||||
wrong, date-window filters matched the wrong rows, and retention froze (nothing older than
|
||||
the migration moment) with a mass-delete cliff one retention period later.
|
||||
|
||||
# Rule
|
||||
|
||||
**The system Timestamp is never authoritative.** Every table that needs an event time
|
||||
stores it in the custom column `OccurredUtc` (`BusinessTimestamp.OccurredUtcColumn`).
|
||||
Custom columns are copied verbatim by `scripts/Migration/Copy-TableStorage.ps1`, so future
|
||||
migrations preserve chronology. Migration checklist: any table whose reads/filters depend
|
||||
on a timestamp must carry it as a custom column — audit this before every storage move.
|
||||
|
||||
# Recovery sources per table
|
||||
|
||||
The original time survives migrations inside the RowKey:
|
||||
|
||||
| Table | RowKey scheme | Decodable |
|
||||
|---|---|---|
|
||||
| AuditLogs | `!{MaxTicks-ticks:D19}_{guid:N}` (since 2026-05-06) | Yes; legacy bare-GUID rows (pre-2026-05) are lost |
|
||||
| OpsEvents | `{MaxTicks-ticks:D19}` | Yes, all rows |
|
||||
| Events | `{ts:yyyyMMddHHmmssfff}_{seq:D10}` (sanitized agent time) | Yes, all rows (ms precision) |
|
||||
|
||||
# Mechanism
|
||||
|
||||
`BusinessTimestamp` (Functions/Helpers) is the single source of truth: RowKey decoders,
|
||||
the OData clause builders, and `GetUtcDateTime` (datetime columns materialize as
|
||||
`DateTimeOffset` *or* `DateTime` depending on payload shape — always read via this helper).
|
||||
|
||||
1. **Read mappers** resolve `OccurredUtc` → RowKey decode → system Timestamp (last
|
||||
resort; only audit legacy GUID rows end there). Resolvers:
|
||||
`TableStorageService.ResolveAuditTimestamp`, `TableOpsEventRepository.ResolveTimestamp`,
|
||||
`TableStorageService.ResolveEventTimestamp` (also used by
|
||||
`GetEarliestSessionEventTimestampAsync`, which feeds StartedAt/Duration reconciliation —
|
||||
a system-Timestamp read there would rewrite migrated sessions' derived fields).
|
||||
2. **Date-window filters and retention use RowKey ranges**, not property comparisons:
|
||||
OData excludes rows missing a compared property (an `OccurredUtc` filter would drop
|
||||
every not-yet-backfilled row), and RowKey ranges are index-backed. Boundary semantics
|
||||
are tick-exact equivalents of the old `Timestamp ge/le` clauses.
|
||||
**Trap:** audit legacy GUID RowKeys (hex first char) sort *after* all `!`-rows, so
|
||||
every lower bound `RowKey ge '!…'` must be paired with `RowKey lt '"'`
|
||||
(`AuditTimeEncodedUpperBound`) — without the guard, a retention sweep would delete all
|
||||
legacy rows. Pinned by `BusinessTimestampFilterClauseTests`.
|
||||
3. **Writes** in all three tables now set `OccurredUtc`; the no-op `["Timestamp"]`
|
||||
entries were removed.
|
||||
4. **Backfill** (`POST /api/maintenance/backfill-occurred-utc`, GlobalAdminOnly, dry-run
|
||||
by default, `table=audit|ops`, batched via `maxRows`/`continuation`): MERGE that adds
|
||||
only `OccurredUtc` decoded from the RowKey — deterministic and idempotent. **Events
|
||||
rows are deliberately never backfilled** (read-path decode is sufficient; no bulk
|
||||
mutation of enrollment evidence). Note: a MERGE resets the row's system Timestamp —
|
||||
irrelevant once nothing depends on it.
|
||||
5. **Audit retention** runs two passes until ~2027-01: RowKey-range for `!`-rows plus a
|
||||
`RowKey ge '0' and Timestamp lt …` pass that ages legacy GUID rows out via their frozen
|
||||
migration-date system Timestamp (remove after 2027-03-01).
|
||||
|
||||
# Consequences
|
||||
|
||||
* Audit legacy GUID rows have "unknown date" semantics: excluded from date-filtered
|
||||
views, shown with the (wrong but honest) 2026-07-18 fallback in unfiltered views.
|
||||
* Raw endpoints (`/api/raw/*`, `query_table`) intentionally keep showing the raw system
|
||||
Timestamp — they are storage-state inspectors, not business views.
|
||||
* Event timestamps for pre-cutover rows resolve to the RowKey's sanitized agent time
|
||||
(ms precision); `ReceivedAt` (server ingest time, rows since 2026-03-19) and
|
||||
`OriginalTimestamp` (pre-clamp agent time, clamped rows only) remain separate fields.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/BusinessTimestamp.cs` — decoders + clause builders
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs` — audit write/mapper/filters/retention
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableOpsEventRepository.cs` — ops write/mapper/filters/retention
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs` — event writes, `ResolveEventTimestamp`, earliest-event read
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/OccurredUtcBackfillService.cs` + `Functions/Admin/BackfillOccurredUtcFunction.cs` — backfill
|
||||
@@ -1,270 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Client-Certificate Tenant Binding — Closing the Cross-Tenant Gap
|
||||
description: Why a chain-valid Intune MDM certificate does not prove the caller belongs to the tenant it claims, which certificate extension carries the Entra tenant id (and which two look like it but do not), and the shadow-then-enforce rollout that measures field coverage before it can lock devices out.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Security/CertTenantBinding.cs
|
||||
tags:
|
||||
- backend
|
||||
- security
|
||||
- authentication
|
||||
- certificates
|
||||
- intune
|
||||
- multi-tenant
|
||||
timestamp: 2026-08-29T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
Agent requests authenticate with mTLS. `CertificateValidator` pins the chain to the embedded
|
||||
Intune roots (`X509ChainTrustMode.CustomRootTrust`), checks the validity window and requires the
|
||||
Client-Authentication EKU. All of that together proves exactly one thing: **the certificate was
|
||||
issued by the Microsoft Intune MDM Device CA to some tenant**.
|
||||
|
||||
Those roots are shared by every Intune tenant on the planet. Nothing in the certificate stage ties
|
||||
the caller to the tenant in the request. The tenant scoping came entirely from the downstream
|
||||
device validators — the serial number must appear in that tenant's Autopilot / imported-device /
|
||||
Cloud PC inventory. Device serial numbers are not secrets; they are printed on the chassis.
|
||||
|
||||
So the pre-existing attack was: enroll a device in your own Intune tenant, obtain a legitimate
|
||||
device certificate, learn one serial number of the victim tenant, and submit telemetry as that
|
||||
tenant. Certificate validation passes, device validation passes.
|
||||
|
||||
# Mechanism
|
||||
|
||||
The Intune MDM Device CA stamps the customer's identifiers into the leaf certificate. Since the
|
||||
agent presents this certificate in the TLS handshake, its contents are proof of possession and the
|
||||
backend can compare them against the requested tenant — no Graph call, no additional consent, and
|
||||
nothing the agent has to send.
|
||||
|
||||
## Which extension carries what
|
||||
|
||||
Verified by decoding a real field certificate (issued 2026-05-08, `Functions.Tests/device-cert-sample.pem`):
|
||||
|
||||
| OID | Encoding | Content |
|
||||
| --- | --- | --- |
|
||||
| `1.2.840.113556.5.14` | nested OCTET STRING (`04 10` + 16 bytes) | **Entra tenant id** — the one to compare |
|
||||
| `1.2.840.113556.5.6` | nested OCTET STRING (`04 10` + 16 bytes) | Intune **Account** id — a *different* GUID, never equal to the tenant id |
|
||||
| `1.2.840.113556.5.4` | **16 raw bytes, no wrapper** | Intune device id, identical to the certificate's Subject CN |
|
||||
|
||||
Two traps are worth stating explicitly, because both cost real debugging time:
|
||||
|
||||
1. `…5.6` is widely described as "the tenant id (AccountID)". It is not the Entra tenant id.
|
||||
Comparing it against the requested tenant would reject every request in production.
|
||||
2. The encodings are not uniform. `…5.4` holds the GUID bare while the other two nest it in a
|
||||
second OCTET STRING, so a decoder written against one shape silently fails on the other.
|
||||
`MsDeviceCertificateOids.TryParseGuid` accepts both, and all GUIDs are Microsoft
|
||||
little-endian (what `new Guid(byte[])` expects).
|
||||
|
||||
A 16-byte input is inherently ambiguous — it is indistinguishable from a truncated wrapper and is
|
||||
always read as a bare GUID. That is harmless here: the value is only ever compared for equality
|
||||
against a known tenant id, never trusted as a standalone claim.
|
||||
|
||||
## Where it is NOT
|
||||
|
||||
`EntraDeviceCertHelper` reads tenant and device ids from the **MS-Organization-Access** certificate
|
||||
(`1.2.840.113556.1.5.284.*`). That certificate is not part of the authentication path at all — the
|
||||
agent selects the Intune MDM certificate by exact issuer match (`CN=Microsoft Intune MDM Device CA`,
|
||||
see `CertificateHelper.SelectMdmCertificate`) and the backend pins only the MDM chain. The Entra
|
||||
certificate is used purely as a local fallback for `TenantIdResolver`; it never leaves the device.
|
||||
|
||||
Sending it in a header instead would prove nothing: only the certificate used in the TLS handshake
|
||||
demonstrates possession of a private key. A certificate copied into a custom header can be replayed
|
||||
by anyone who has seen it once.
|
||||
|
||||
# Rollout
|
||||
|
||||
Enforcing immediately would be reckless: any device population whose certificates predate the
|
||||
extension, or whose CA behaves differently (sovereign clouds), would be locked out at once. The
|
||||
rollout therefore has staged consequences.
|
||||
|
||||
**Stage 1 — shadow (completed, kept as the measurement that justified stage 2).** `SecurityValidator.ObserveCertTenantBinding` evaluates the binding
|
||||
after certificate validation and logs the outcome. It returns `void`, changes no state, and swallows
|
||||
its own exceptions, so it cannot fail a request that would otherwise be accepted. The pure comparison
|
||||
lives in `CertTenantBinding.Evaluate`; the outcome codes (`Match`, `Mismatch`, `ExtensionMissing`,
|
||||
`Unparseable`, `RequestTenantNotAGuid`) are stable and queried by exact match in KQL.
|
||||
`WouldRejectUnderEnforcement` records what stage 2 *would* have done, so the telemetry can be read
|
||||
without re-deriving the rule at query time.
|
||||
|
||||
The telemetry uses two carriers, because the numerator and the denominator have very different
|
||||
costs and answer different questions:
|
||||
|
||||
* **Every** outcome, `Match` included, is stamped onto the request row as the `CertTenantBinding`
|
||||
dimension — `SecurityValidator` writes it into `FunctionContext.Items`
|
||||
(`CertTenantBinding.RequestItemKey`) and `RequestTelemetryMiddleware` copies it onto the
|
||||
`RequestTelemetry` it already emits. That row exists per request and is unsampled, so the
|
||||
denominator costs no additional telemetry at all.
|
||||
* Outcomes that need an enforcement decision (mismatch, missing, undecodable) are **additionally**
|
||||
logged at Warning with the full context — thumbprint, certificate tenant, session.
|
||||
|
||||
`Match` is deliberately not a trace line. Worker-side `LogInformation` never reaches App Insights
|
||||
(the ApplicationInsightsLoggerProvider default rule is Warning+ and host-forwarding of worker logs
|
||||
is disabled), so a `Match` trace is dead code that silently costs the denominator. The first
|
||||
deployment of this feature made exactly that mistake: it emitted zero rows across 32k requests,
|
||||
which is indistinguishable from the code never running.
|
||||
|
||||
**Stage 2 — enforced since 2026-08-25.** `Mismatch` returns 403 and is recorded through the shared
|
||||
`LogRequestRejection` path (`stage=certtenant`) so it lands in the same KQL raster as every other
|
||||
rejection reason. The switch was made against measured data: 35,684 requests across 32 tenants,
|
||||
every one a `Match`, with the Warning channel independently verified to be carrying other traffic so
|
||||
the zero could not be a telemetry artefact.
|
||||
|
||||
The rule lives in one place, `CertTenantBinding.Rejects`, and only `Mismatch` blocks. Everything
|
||||
else means "cannot tell" and is deliberately tolerated:
|
||||
|
||||
* `ExtensionMissing` — the sample only covers devices active in the measurement window, so older
|
||||
certificates or a sovereign-cloud CA could still lack the extension. Rejecting on absence would
|
||||
lock those devices out for something they cannot control. **This is the one to tighten later**,
|
||||
once the `ExtensionMissing` rate is provably zero across a wider window; a test pins the current
|
||||
tolerance so removing it has to be a deliberate decision rather than a drive-by edit.
|
||||
* `Unparseable` — a decoder or encoding surprise is our problem, not the device's.
|
||||
* `RequestTenantNotAGuid` — nothing to compare against, and such a request fails the tenant lookup
|
||||
in §0 long before reaching this point.
|
||||
|
||||
The evaluation fails open on its own exception: this gate exists to stop a cross-tenant certificate,
|
||||
and a defect on our side is not evidence of one. The 403 body deliberately does not echo which
|
||||
tenant the certificate belongs to — that pair is in the rejection log, not in the response.
|
||||
|
||||
Grep marker for every site involved: `CERT-TENANT-BINDING`.
|
||||
|
||||
The bootstrap-token path (`X-Bootstrap-Token`) bypasses certificate validation by design and is
|
||||
unaffected by either stage.
|
||||
|
||||
# Certificate transport and proof of possession
|
||||
|
||||
Every stage above validates certificate *content*: chain, validity window, EKU, tenant extension.
|
||||
None of them proves the caller holds the private key — a certificate is public material (sent in
|
||||
clear in a TLS 1.2 client-auth handshake, readable from the machine store by any local user, present
|
||||
on resold hardware). Proof of possession comes from the TLS handshake alone: the platform runs
|
||||
`clientCertMode=Required` on every path not listed in its `clientCertExclusionPaths`, forwards the
|
||||
handshake certificate as `X-ARR-ClientCert`, and **strips any inbound copy of that header — on
|
||||
excluded paths too** (verified 2026-08-29 with a spoofed header against `/api/bootstrap/config`:
|
||||
the validator saw "No certificate provided"). So `X-ARR-ClientCert` is trustworthy exactly because
|
||||
the caller cannot write it.
|
||||
|
||||
Two consequences shape `SecurityValidator`:
|
||||
|
||||
* **No other certificate header is ever read.** A former local-dev fallback (`X-Client-Certificate`)
|
||||
accepted certificate bytes from a client-writable header, which decouples the certificate from
|
||||
key possession: anyone holding a device's public certificate would authenticate as that device.
|
||||
Removed 2026-08-29. Local development sets `X-ARR-ClientCert` directly — Core Tools does not strip
|
||||
it.
|
||||
* **Bootstrap routes are token-only, fail-closed.** `/api/bootstrap/*` is in the platform's
|
||||
exclusion list so cert-less OOBE agents can reach it — which also means nothing on those paths
|
||||
proves a handshake happened. The token is therefore the sole credential there:
|
||||
`SecurityValidator.IsBootstrapRoute` (path prefix, mirroring the exclusion entry so no function
|
||||
can forget to opt in) rejects a missing or **empty** `X-Bootstrap-Token`, and a missing
|
||||
`BootstrapSessionService`, with 401 before the certificate stage can run. The live gap this
|
||||
closes: the function-level `Headers.Contains` gate accepted an empty header while the validator
|
||||
treated it as "no token" and fell through to the certificate stage, where the fallback header
|
||||
above was parsed. Bootstrap tokens are never accepted as a *replacement* for the certificate on
|
||||
platform-enforced routes either — there the cert requirement is already satisfied by the
|
||||
handshake before the request reaches code.
|
||||
|
||||
The exclusion list is portal-only state (no IaC). Adding `/api/agent` to it, or moving a
|
||||
device-authenticated function under an excluded prefix, would silently reopen the header path;
|
||||
`BootstrapRouteFailClosedTests` pins the code half of this contract.
|
||||
|
||||
**Stage 3 — device binding (Global-Admin-only preview).** Tenant binding proves the certificate was
|
||||
issued to this tenant; it does not prove the certificate belongs to a device the tenant still has.
|
||||
`IntuneDeviceBindingValidator` closes that by resolving the Intune managedDevice id from the
|
||||
certificate's Subject CN against the tenant's own `deviceManagement/managedDevices` inventory
|
||||
(point-GET on `managedDevices/{id}` — no OData filter to inject into, and a 404 is unambiguous).
|
||||
|
||||
It is modelled on `CloudPcDeviceValidator` and shares its cache/retry/transient contract, with one
|
||||
important difference in how it is wired: the lookup is fire-and-forget, like the DevPrep
|
||||
association shadow, because a cold Graph round-trip can take seconds and must never sit in the
|
||||
agent's request path. That rules out the request-row dimension stage 1 uses — the row is written
|
||||
the moment the function returns, before the detached task finishes — so the outcome is logged at
|
||||
Warning instead. Volume stays bounded because only Global Admins can see the toggle
|
||||
(`user?.isGlobalAdmin === true`, the same gate as the DevPrep preview). Widening it beyond preview
|
||||
should move to an inline call plus a request dimension.
|
||||
|
||||
Outcomes: `Match`, `NotFound`, `NoDeviceIdInCert`, `PermissionMissing`, `Transient`. Only
|
||||
`Transient` is retried and left uncached — a missing grant or a foreign device is a state, not an
|
||||
outage.
|
||||
|
||||
Graph cost is not a concern, and that is measured rather than assumed: the first live enrollment
|
||||
produced **136 requests and exactly one Graph call**, because the 30-minute positive cache absorbs
|
||||
the rest. For comparison, the per-request Autopilot serial validator made 25 calls in the same
|
||||
window. Lengthening the positive TTL would buy nothing and would delay noticing a device that has
|
||||
been wiped or removed; shortening the 5-minute negative TTL matters more, since that is how quickly
|
||||
a device object appearing late is picked up.
|
||||
|
||||
What the 136 requests *did* cost was log lines saying the same thing 136 times, so the shadow log is
|
||||
gated on `IntuneDeviceBindingResult.ServedFromCache`: one line per real Graph lookup. Cached repeats
|
||||
add nothing — the reported age just counts up mechanically from the same `enrolledDateTime`.
|
||||
Outcomes that are never cached (`Transient`, `NoDeviceIdInCert`) still log every time, which is the
|
||||
right behaviour for an anomaly.
|
||||
|
||||
The open question this preview exists to answer is a race, not an attack: a device object can in
|
||||
principle appear in Intune *after* the agent's first call, and enforcing before that is understood
|
||||
would reject legitimate enrollments. Every result therefore carries `enrolledDateTime`, and the log
|
||||
line records `enrolledAgeSeconds` — a device whose object was created seconds before the request
|
||||
looks completely different from a certificate that never belonged to the tenant. Grep marker:
|
||||
`CERT-DEVICE-BINDING-SHADOW`.
|
||||
|
||||
Enabling it needs the optional Graph permission `DeviceManagementManagedDevices.Read.All`, granted
|
||||
per tenant with the customer script — it is not in the default-consent set:
|
||||
|
||||
```powershell
|
||||
.\Grant-AutopilotMonitorAddOn.ps1 -ClientId "<client-id>" -Features IntuneDeviceBinding
|
||||
```
|
||||
|
||||
`GraphFeatureCatalog` (feature `IntuneDeviceBinding`) and the script's catalog, `ValidateSet` and
|
||||
help block are pinned to each other by `GraphFeatureCatalogSyncTests`.
|
||||
|
||||
# Examples
|
||||
|
||||
Coverage — the question stage 2 depends on ("would enforcing lock anyone out?"):
|
||||
|
||||
```kusto
|
||||
requests
|
||||
| where isnotempty(customDimensions.CertTenantBinding)
|
||||
| summarize Requests = sum(itemCount)
|
||||
by Outcome = tostring(customDimensions.CertTenantBinding),
|
||||
Tenant = tostring(customDimensions.TenantId)
|
||||
| order by Requests desc
|
||||
```
|
||||
|
||||
Detail on the individually actionable ones:
|
||||
|
||||
```kusto
|
||||
traces
|
||||
| where message has "AgentCertTenantBinding"
|
||||
| extend Outcome = tostring(customDimensions.Outcome),
|
||||
Tenant = tostring(customDimensions.TenantId)
|
||||
| summarize count() by Outcome, Tenant
|
||||
```
|
||||
|
||||
Anything in `Mismatch` is a cross-tenant certificate; anything in `ExtensionMissing` is a device
|
||||
that stage-2 enforcement would have to make a policy decision about. Note that the trace query
|
||||
alone can never show `Match`, so it cannot tell "everything matched" apart from "nothing ran" —
|
||||
use the request-row query for that.
|
||||
|
||||
Stage 3, including the enrollment-race measurement:
|
||||
|
||||
```kusto
|
||||
traces
|
||||
| where message has "AgentCertDeviceBinding"
|
||||
| extend Outcome = tostring(customDimensions.Outcome),
|
||||
AgeSecs = toint(customDimensions.EnrolledAgeSeconds)
|
||||
| summarize Requests = count(),
|
||||
MinAgeSecs = min(AgeSecs),
|
||||
P05AgeSecs = percentile(AgeSecs, 5)
|
||||
by Outcome
|
||||
```
|
||||
|
||||
`NotFound` is the row that decides whether this can ever become a gate. A small `MinAgeSecs` on
|
||||
`Match` means device objects are appearing only just in time — the race is real and enforcement
|
||||
would need a grace path.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Security/MsDeviceCertificateOids.cs` — OID catalog and the
|
||||
shared ASN.1 GUID decoder (one implementation for agent and backend).
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/CertificateValidator.cs` — extension extraction
|
||||
alongside chain validation, cached per thumbprint.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/CertTenantBinding.cs` — the pure comparison.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/SecurityValidator.cs` — shadow observation.
|
||||
* [W365 Cloud PC Device Validation](cloudpc-device-validation.md) — the other consumer of this
|
||||
certificate's identity fields (Subject CN = Intune device id).
|
||||
@@ -1,142 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: W365 Cloud PC Device Validation — Cert-CN-Bound Fallback Validator
|
||||
description: Why Windows 365 Cloud PCs can never pass the Autopilot serial lookup, and how the CloudPc validator admits exactly the tenant's service-provisioned Cloud PCs by binding the MDM certificate's Subject CN (= Intune device id) to the Graph virtualEndpoint/cloudPCs inventory.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Security/CloudPcDeviceValidator.cs
|
||||
tags:
|
||||
- backend
|
||||
- security
|
||||
- device-validation
|
||||
- windows365
|
||||
- cloudpc
|
||||
- graph
|
||||
timestamp: 2026-08-06T00:00:00+02:00
|
||||
---
|
||||
|
||||
# W365 Cloud PC Device Validation
|
||||
|
||||
Windows 365 Cloud PCs are provisioned headless by the Windows 365 service and are
|
||||
**structurally never Autopilot-registered** — the serial lookup against
|
||||
`windowsAutopilotDeviceIdentities` (and the Corporate Identifier fallback, whose
|
||||
serials nobody can pre-import for VMs minted at provisioning time) always misses.
|
||||
Before this validator, a Cloud PC agent died in the 403 "Device not registered"
|
||||
even though its mTLS cert was perfectly valid, so Cloud PC first-connect
|
||||
enrollment (Account Setup, RealmJoin, Hello) could not be monitored.
|
||||
|
||||
# Schema — the validator chain
|
||||
|
||||
`SecurityValidator.ValidateRequestAsync` stage 4 runs the same chain for every
|
||||
device; there is **no device-type routing** and the local W365 markers
|
||||
(Windows365 registry key, CloudManagedDesktopExtension service) are never
|
||||
consulted for auth — they gain an attacker nothing:
|
||||
|
||||
1. **Autopilot serial lookup** (`ValidateAutopilotDevice`) — unchanged primary.
|
||||
2. **Corporate Identifier** (`ValidateCorporateIdentifier`) — unchanged fallback.
|
||||
3. **CloudPc** (`ValidateCloudPcDevice`, default off) — only reached when 1+2
|
||||
missed. Extracts the Intune device id from the **chain-validated** client
|
||||
certificate's Subject CN (`TryGetIntuneDeviceIdFromCertSubject`: CN present
|
||||
AND canonical GUID, else definitive reject) and requires a `cloudPC` object
|
||||
with `managedDeviceId eq <CN>` in the claimed tenant
|
||||
(`GET /v1.0/deviceManagement/virtualEndpoint/cloudPCs?$filter=…`).
|
||||
`ValidatedBy = ValidatorType.CloudPc (5)`.
|
||||
|
||||
Only machines actually provisioned by the Windows 365 service have a `cloudPC`
|
||||
object — a regular Intune-enrolled non-Autopilot device still ends in the 403,
|
||||
so the "Autopilot devices only" contract of stages 1–2 is not widened. The
|
||||
tenant hard gate accepts `ValidateCloudPcDevice` as a third satisfying flag.
|
||||
|
||||
## Why the cert CN, not headers or local markers
|
||||
|
||||
Field-verified 2026-08-06 on a real W365 Enterprise Cloud PC (gktatooine.net):
|
||||
|
||||
- MDM client cert Subject `CN=07623d56-…` == Intune `managedDevice.id` ==
|
||||
`cloudPC.managedDeviceId`. The CN cannot be forged without the Intune MDM
|
||||
Device CA (chain pinned to embedded roots, `CustomRootTrust`).
|
||||
- `$filter=managedDeviceId eq '…'` is supported server-side on **v1.0** — no
|
||||
paging fallback needed.
|
||||
- `managedDevice.serialNumber` matches local `Win32_BIOS.SerialNumber` exactly;
|
||||
the "Cloud PC Enterprise …" model string exists **only** in Graph, never in
|
||||
local WMI (local WMI reports Manufacturer='Microsoft Corporation',
|
||||
Model='Virtual Machine').
|
||||
|
||||
Threat model: a valid Intune cert from a *different* tenant fails (its CN does
|
||||
not exist in the claimed tenant's Cloud PC inventory — this is a *tenant
|
||||
binding* the serial-based validators do not have); spoofed serial headers and
|
||||
faked local W365 markers are irrelevant (never consulted); a rogue enrolled VM
|
||||
with doctored SMBIOS cannot mint a `cloudPC` object (service-side state, not
|
||||
device-reported inventory — the reason the model-string gate was rejected).
|
||||
Retired/deprovisioned Cloud PCs disappear from the inventory and their
|
||||
validation dies with them (compensates the absent cert revocation check).
|
||||
Residual: a compromised *legitimate* Cloud PC of the tenant — same trust level
|
||||
as any tenant device today, rate-limited by cert thumbprint.
|
||||
|
||||
## Graph permission plumbing
|
||||
|
||||
- Optional add-on permission `CloudPC.Read.All` — NOT part of the default
|
||||
consent set. Feature `W365CloudPcValidation` in `GraphFeatureCatalog` →
|
||||
surfaces automatically in the "Optional Graph capabilities" UI (the feature
|
||||
table and grant command are backend-driven) and in
|
||||
`Grant-AutopilotMonitorAddOn.ps1` (`-Features W365CloudPcValidation`).
|
||||
`GraphFeatureCatalogSyncTests` pins C# catalog ↔ grant script (previously
|
||||
comment-only lock-step).
|
||||
- A Graph **403** (permission not granted) is a definitive negative (cached
|
||||
5 min), NOT transient — a 503 Retry-After loop cannot fix a missing grant.
|
||||
The error message names the add-on feature.
|
||||
- Same resilience contract as `AutopilotDeviceValidator`: 30/5 min
|
||||
positive/negative cache, 2 attempts, transient → agent-facing 503.
|
||||
|
||||
## Bootstrap script interplay (v2.3-dev.3)
|
||||
|
||||
The Cloud-PC relax in `Install-AutopilotMonitor-dev.ps1` additionally exempts
|
||||
**guard 4** (12 h uptime window): a Cloud PC runs headless for days before the
|
||||
user's first connect, and the monitorable phase (Account Setup) only starts at
|
||||
that first connect — the single-fresh-profile window (< 15 min) is the time
|
||||
anchor there, not boot uptime. The OOBE-restore trigger keeps the uptime guard.
|
||||
|
||||
## Agent-side IsCloudPc flag + decision-engine expectation
|
||||
|
||||
Same change set, agent side. `CloudPcDetector` (V2.Core) re-uses the bootstrap's
|
||||
field-verified marker AND — `HKLM\SOFTWARE\Microsoft\Windows365` key **plus**
|
||||
installed `CloudManagedDesktopExtension` service (probed via its
|
||||
`HKLM\SYSTEM\CurrentControlSet\Services` key; the agent deliberately has no
|
||||
`System.ServiceProcess` dependency). SKIP-safe: any error resolves `false`.
|
||||
|
||||
Three independent transport rails, never an auth input:
|
||||
|
||||
- **Session metadata** — `SessionRegistration.IsCloudPc` →
|
||||
Sessions table (sticky-true OR across re-registrations, like
|
||||
`IsSelfDeployingProfile`) → SessionsIndex full mirror → `SessionSummary` →
|
||||
search filter `isCloudPc` (portal, `/api/search/sessions`, `/api/raw/sessions`,
|
||||
MCP `search_sessions`/`query_raw_sessions`). Agent-reported and unverified —
|
||||
can legitimately disagree with the server-derived `ValidatedBy = CloudPc`
|
||||
(e.g. Cloud PC admitted via a stage that ran earlier).
|
||||
- **Distress channel** — `DistressReport.IsCloudPc` (captured once at
|
||||
`DistressReporter` construction, `BackendClientFactory`) rides every pre-auth
|
||||
distress report → `DistressReportEntry.IsCloudPc` → the "Devices Not
|
||||
Registered" report aggregates it **sticky-true** per serial
|
||||
(`GetDeviceNotRegisteredFunction`) and the portal insight renders a Cloud PC
|
||||
badge plus a hint pointing at the Cloud PC validation toggle + the
|
||||
`W365CloudPcValidation` add-on. Same UNVERIFIED contract as all distress
|
||||
fields; rows from older agents read false. Fits the existing 1536-byte
|
||||
payload cap without a bump.
|
||||
- **Decision engine** — the marker rides the `EnrollmentFactsObserved` payload
|
||||
(`isCloudPc`), is recorded set-once for BOTH values in
|
||||
`EnrollmentScenarioObservations.CloudPc`, and annotates the scenario profile
|
||||
reason `cloud_pc_first_connect:no_device_esp_expected`. That is the
|
||||
engine-recorded expectation "no Device ESP phase will appear": Device-ESP ran
|
||||
headless at provisioning, the session starts at Account Setup. Deliberately
|
||||
reason+observation only — no completion arm depends on DeviceSetup, `Mode`
|
||||
is still classified by AccountSetup/IME signals (Classic), and `EspConfig`
|
||||
keeps the *configured* FirstSync semantics. The positive marker surfaces as
|
||||
`cloud_pc_marker` in the terminal audit-trail signal census.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Backend/AutopilotMonitor.Functions/Security/CloudPcDeviceValidator.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Security/SecurityValidator.cs` — stage 4 chain, `TryGetIntuneDeviceIdFromCertSubject`
|
||||
- `src/Shared/AutopilotMonitor.Shared/Models/Graph/GraphFeatureCatalog.cs` — `W365CloudPcValidation`
|
||||
- `scripts/CustomerSetup/Grant-AutopilotMonitorAddOn.ps1` — customer-side grant
|
||||
- `scripts/Bootstrap/Install-AutopilotMonitor-dev.ps1` — Cloud-PC relax + guard-4 exemption
|
||||
- `src/Backend/AutopilotMonitor.Functions.Tests/CloudPcDeviceValidatorTests.cs`, `SecurityValidatorTests.cs`, `GraphFeatureCatalogSyncTests.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Security/CloudPcDetector.cs` — agent-side marker AND
|
||||
- `src/Shared/AutopilotMonitor.DecisionCore/State/EnrollmentScenarioObservations.cs` — `CloudPc` fact; reason annotation in `EnrollmentScenarioProfileUpdater`
|
||||
@@ -1,123 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Device Journeys & First-Time-Right (F2)
|
||||
description: How a device's terminal enrollment attempts are grouped into journeys (device key, junk-serial exclusion, first-success end, 30-day gap), where the DeviceHistories chain and the daily FTR aggregates are computed, stored and pruned, and which surfaces serve them.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/DeviceJourneyCalculator.cs
|
||||
tags:
|
||||
- backend
|
||||
- metrics
|
||||
- device-history
|
||||
- first-time-right
|
||||
- insights
|
||||
timestamp: 2026-07-27T12:00:00+02:00
|
||||
---
|
||||
|
||||
# Device Journeys & First-Time-Right (F2)
|
||||
|
||||
Answers "how many devices enrolled right on the first try?" and "show me every attempt
|
||||
this device made". Session success rate hides wipe-and-retry loops — every retry that
|
||||
eventually succeeds counts as a success; First-Time-Right (FTR) exposes the real pain.
|
||||
All grouping semantics live server-side in `DeviceJourneyCalculator` (JourneyVersion 1);
|
||||
no consumer re-derives them.
|
||||
|
||||
# Schema
|
||||
|
||||
## Device key
|
||||
|
||||
`(TenantId, normalized serial)` — normalization is trim + lower-case. Junk/placeholder
|
||||
serials are excluded entirely (never a chain, never in FTR, disclosed as a per-day
|
||||
exclusion count): `System Serial Number`, `To Be Filled By O.E.M.`, `Default string`,
|
||||
`0`, `None`, `INVALID`, `Unknown` (the agent's WMI-failure sentinel), empty, or shorter
|
||||
than 4 characters. A motherboard swap changes the serial and therefore the device key;
|
||||
re-enrollment under a different tenant is a separate key by design.
|
||||
|
||||
## Journey
|
||||
|
||||
Terminal sessions of a device key ordered by StartedAt. The terminal set is
|
||||
**Succeeded / Failed / Incomplete** — `Pending` (a sealed WhiteGlove part 1),
|
||||
`AwaitingUser`, `Stalled` and `InProgress` are OPEN sessions and never attempts.
|
||||
|
||||
* A journey **ends** with the first terminal success; the next session starts a new
|
||||
journey (redeployment).
|
||||
* A gap of **more than 30 days** since the previous attempt's terminal moment
|
||||
(`CompletedAt`, fallback StartedAt) starts a new journey even without a success
|
||||
(device shelved/repurposed). Constant, not a setting — production gaps: median 4.9 h,
|
||||
92 % < 7 d.
|
||||
* WhiteGlove part 1 + part 2 share ONE session row, so a completed WG enrollment is ONE
|
||||
attempt; a WG device still waiting for its user session leaves the journey open —
|
||||
never a failed attempt.
|
||||
* **Attempt count** = terminal sessions in the journey (`Failed` and `Incomplete` are
|
||||
non-successful attempts).
|
||||
* **FTR rate** = completed journeys with attempt count 1 ÷ all completed journeys.
|
||||
Open and gap-abandoned journeys never enter numerator or denominator.
|
||||
|
||||
Durations shown for attempts are the sessions' authoritative `DurationSeconds` verbatim
|
||||
(WhiteGlove pause excluded; `Incomplete` deliberately has none) — never
|
||||
`CompletedAt − StartedAt`, which is later in ~25 % of terminal sessions.
|
||||
|
||||
## Persistence & compute
|
||||
|
||||
* `DeviceHistories` (PK = TenantId, RK = percent-encoded normalized serial): the chain of
|
||||
the device's terminal session refs `{sessionId, startedAt, completedAt, status,
|
||||
enrollmentType, isPreProvisioned, durationSeconds, adminMarked}` capped at the 20 most
|
||||
recent, plus derived `CurrentJourneyAttempts` (the LAST journey's attempts — the
|
||||
banner's "Attempt N"), `JourneyCount` (chain-scoped) and `JourneyVersion`. Written
|
||||
inline at the session-terminal single-writer seam
|
||||
(`TableSessionRepository.UpdateSessionStatusAsync`, all three terminal statuses,
|
||||
DeletionState-guarded) so the session banner is fresh; healed by the maintenance sweep.
|
||||
Deliberately NOT part of the per-session deletion manifest — the row aggregates many
|
||||
sessions. Wiped on tenant offboarding.
|
||||
* Sweep (`SweepDeviceJourneysAsync`, 30-day rolling window, 2h tick): (1) drops chain
|
||||
refs of deleted sessions **tombstone-driven** — every session deletion (cascade and
|
||||
retention both run through the cascade worker) leaves a `SessionTombstones` marker for
|
||||
~7 days, far longer than the sweep cadence, and even an expired-but-unpruned marker is
|
||||
valid deletion evidence; a chain left empty deletes its row. (2) Merges the window's
|
||||
terminal sessions into their chains (backfill, inline-miss healing, re-terminal
|
||||
reclassifications). (3) Recomputes the daily FTR aggregates idempotently.
|
||||
* `DeviceJourneyAggregates` (PK = TenantId or `global`, RK = `{yyyy-MM-dd}`): per day —
|
||||
completed journeys, first-time-right count, attempt histogram, junk-serial exclusion
|
||||
count, `JourneyVersion`. A journey buckets on the StartedAt date of its completing
|
||||
success session (the platform's StartedAt-date convention; late terminals converge via
|
||||
the sweep, which slices the tick's shared projected window scan — see
|
||||
`time-attribution.md`). All counts are **additive**, so a window rate is the ratio of summed daily
|
||||
rows — there is no rolling-window row (unlike the median-based time-attribution
|
||||
aggregates, which cannot merge). 180-day retention. Rows exist below the ≥20 UI gate
|
||||
on purpose (the UI needs the n for "insufficient data (n=…)").
|
||||
|
||||
# Surfaces
|
||||
|
||||
* `GET api/metrics/device-history?serialNumber=&sessionId=` (MemberRead + tenant query
|
||||
param — the same route serves the web banner and MCP; a Global Admin passes `tenantId`)
|
||||
— the chain plus, with `sessionId`, that session's server-computed attempt number
|
||||
(live sessions get their would-be position via a virtual non-successful attempt, so the
|
||||
redeploy and gap rules apply). `history: null` = no recorded history.
|
||||
* `GET api/metrics/device-journeys?days=` (MemberRead) /
|
||||
`GET api/global/metrics/device-journeys?days=&tenantId=` (GlobalReadOrAdmin) —
|
||||
`{ windowDays, totals, daily, repeatDevices }`. The days selector is honored because
|
||||
counts are additive. Without `tenantId` the global variant serves the cross-tenant
|
||||
aggregate rows and `repeatDevices: null` — a per-device list would require scanning
|
||||
every tenant's partition.
|
||||
* Repeat devices (violator list, capped 10): devices whose LAST journey took ≥2 attempts
|
||||
and whose newest terminal session falls inside the window, ordered by attempts then
|
||||
recency; the failure reason comes from the newest failed attempt (bounded point-reads).
|
||||
* Web: "Attempt N for this device · View history" banner on session detail (expandable
|
||||
chain); "First-time-right" section on Fleet Health (rate + weekly trend + attempt
|
||||
histogram + repeat devices). MCP: `get_device_history` (serial mode / fleet FTR mode).
|
||||
|
||||
# Examples
|
||||
|
||||
A device fails ESP twice and succeeds on the third try within one week: one journey,
|
||||
three attempts, completed — counts once in the FTR denominator, not in the numerator,
|
||||
and the histogram's bucket `3` gains one. The same device redeployed a month later and
|
||||
succeeding immediately starts journey 2 with attempt count 1 — first-time-right. A
|
||||
device with `serialNumber = "Unknown"` (WMI failure) never gets a chain; its terminal
|
||||
sessions surface only as that day's disclosed exclusion count.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/DeviceJourneyCalculator.cs` — device key, junk list, grouping, attempt numbers (JourneyVersion 1).
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.DeviceHistory.cs` — persistence + inline seam entry.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.DeviceJourneys.cs` — sweep, tombstone pruning, daily aggregation.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetDeviceJourneyFunctions.cs` — read surfaces.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/DeviceJourneyAndFtrTests.cs` — table-driven grouping/aggregation/attempt-number tests.
|
||||
* `tasks/insights-expansion-spec.md` — F2 specification and source-data audit (§0.5).
|
||||
@@ -1,133 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: IME Installer Archive (version-verified, multi-host)
|
||||
description: How every fleet-observed Intune Management Extension build is archived as its MSI - the ime_agent_version sighting that queues the job, the candidate hosts, the ProductVersion check that keeps a wrong ring's build out of the version folder, the re-queue on later sightings, and the manual skill-side backfill.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs
|
||||
tags:
|
||||
- backend
|
||||
- ime
|
||||
- archive
|
||||
- blob
|
||||
- queue
|
||||
timestamp: 2026-08-30T12:00:00+02:00
|
||||
---
|
||||
|
||||
# IME Installer Archive
|
||||
|
||||
The platform keeps every Intune Management Extension build it has ever seen in the fleet
|
||||
as the original installer, so any IME version can be decompiled and diffed later
|
||||
(`ime-decompiles` repository, `/ime-decompile` skill) even after Microsoft's CDN has moved
|
||||
on. Two stores:
|
||||
|
||||
| Store | Content | Written by |
|
||||
| --- | --- | --- |
|
||||
| Blob container `ime-archive` (private, kept by design, not tenant-scoped) | `{version}/IntuneWindowsAgent.msi` (write-once) + `{version}/provenance.json` (derived, overwritable) | `ImeMsiArchiver` via the `ime-msi-archive` queue; `/ime-decompile` skill as manual backfill |
|
||||
| Table `ImeVersionHistory` (partition `Global`, RowKey = version) | first/last seen, session count, `CorroboratedAt`, `MsiArchiveStatus`, `MsiArchiveUpdatedAt`, `MsiArchiveBlobPath`, `MsiSha256`, `MsiBytes`, `MsiSourceUrl` | `RecordImeVersionAsync` (sighting), `UpdateImeVersionArchiveInfoAsync` (archive outcome / queued stamp) |
|
||||
|
||||
# Schema
|
||||
|
||||
## Sighting → job
|
||||
|
||||
`EventIngestProcessor` looks at the batch's `ime_agent_version` event (`agentVersion`, plus
|
||||
the agent's CSP-registry enrichment `msiDownloadUrl` / `msiMatchedBy` when present).
|
||||
`RecordImeVersionAsync` returns an `ImeVersionSighting`:
|
||||
|
||||
* `IsNew` — the insert succeeded: first fleet-wide sighting → ops event + enqueue.
|
||||
* otherwise the row's `MsiArchiveStatus` / `MsiArchiveUpdatedAt`, read in the same point
|
||||
read that bumps `SessionCount` (no second table read).
|
||||
|
||||
A known version is **re-queued** (`ImeMsiArchiver.ShouldRequeueOnSighting`) only when all
|
||||
of these hold: the event URL passes the allowlist **and** `msiMatchedBy == productVersion`
|
||||
(the CSP row's ProductVersion equals the observed version, so the URL is authoritative),
|
||||
the status is neither `Archived` nor `Failed:BadVersion`, and the last archive activity is
|
||||
older than 24 h (`RequeueBackoff`) or never happened. The ingest path then merges
|
||||
`MsiArchiveStatus = Queued` (stamping `MsiArchiveUpdatedAt`) before enqueueing, so the other
|
||||
sessions of that version see "in flight" and do not pile on. Worst case for a version no
|
||||
host serves any more: one walk over the hosts per day, not one per enrollment.
|
||||
|
||||
## Trust boundary of a sighting
|
||||
|
||||
The version string is **device-supplied** (any device holding a tenant agent credential can
|
||||
send an `ime_agent_version` event) and becomes the RowKey of a caller-independent GLOBAL
|
||||
partition that every tenant reads. Four layers keep one tenant's device from turning a
|
||||
claim into a platform fact:
|
||||
|
||||
| Layer | Where | Effect |
|
||||
| --- | --- | --- |
|
||||
| Plausibility guard | `ImeMsiArchiver.IsPlausibleVersion`, called first in `RecordImeVersionAsync` (and again by the archiver) | plain dotted digits, 2–4 components, inside Windows Installer `ProductVersion` bounds (major/minor ≤ 255, build ≤ 65535). Anything else → `ImeVersionSighting.Rejected`: no row, no ops event, no archive job, no re-queue. |
|
||||
| One sighting per session | `UpdateSessionImeAgentVersionAsync` returns whether the session's stored version changed; only then does the ingest path call `RecordImeVersionAsync` | `SessionCount` moves once per session and version; a re-sent or replayed event, a tombstoned or unregistered session, count nothing. |
|
||||
| Corroboration | merge path of `RecordImeVersionAsync` stamps `CorroboratedAt` on the first sighting from a tenant other than `FirstSeenTenantId` | the first-seen tenant's own devices can never corroborate their own claim. |
|
||||
| Fleet confirmation on read | `GetImeVersionHistoryFunction.IsFleetConfirmed` (MemberRead projection only) | tenant members see a version once it is `Archived` (ProductVersion matched a real package), or corroborated, or first seen before `LegacyTrustCutoffUtc` (2026-08-31 — the 16 rows reviewed on 2026-08-30 are all genuine builds). Global scope sees every row, unconfirmed ones included. |
|
||||
|
||||
What remains possible for a hostile device: a plausible, never-shipped version creates a
|
||||
row visible to Global Admins plus one `NewImeVersionDetected` ops event and one archive walk
|
||||
that ends `Failed:VersionMismatch` (no blob). It is never listed to other tenants and can
|
||||
never occupy the archive folder of a real build.
|
||||
|
||||
## Candidate hosts and the version check
|
||||
|
||||
Microsoft distributes IME from several versionless URLs that carry **different rollout
|
||||
rings at the same time**. Observed 2026-08-29: `imeswdb-afd-secondary` served 1.104.102.0,
|
||||
`imeswda-afd-primary` 1.105.101.0, `imeswda-afd-hotfix` 1.105.103.0. "A new version is by
|
||||
definition what the canonical URL serves right now" is therefore false, and the first
|
||||
version of the archiver filed a 1.104.102.0 package under `1.105.103.0/` (fixed by hand:
|
||||
blob overwritten, row merged).
|
||||
|
||||
The archiver walks `BuildCandidateUrls(eventUrl)`: the allowlisted event URL first (HTTPS,
|
||||
host `*.manage.microsoft.com`, filename exactly `IntuneWindowsAgent.msi`), then
|
||||
`FallbackMsiUrls` = primary, secondary, hotfix — de-duplicated. For each candidate:
|
||||
|
||||
1. Download with the admin size cap (Content-Length preflight + mid-stream cap) into a
|
||||
delete-on-close temp file while SHA-256 hashing.
|
||||
2. Read the package's `ProductVersion` with `MsiProductVersionReader` — a pure managed OLE
|
||||
compound-file reader (v3 512-byte and v4 4096-byte sectors, DIFAT/FAT/mini-FAT, MSI
|
||||
stream-name decoding, `_StringPool`/`_StringData` incl. long strings and holes, the
|
||||
column-major `Property` table with 2- or 3-byte string refs). It never throws; anything
|
||||
unreadable is `null`. Needed because the Function host is Linux — no `msiexec`.
|
||||
3. `VersionsMatch(observed, productVersion)` (component-count tolerant: `1.105.103` ==
|
||||
`1.105.103.0`). Mismatch or unreadable → log, record the attempt, next candidate.
|
||||
4. Match → write-once upload (If-None-Match:*), then `provenance.json` with
|
||||
`productVersion`, the chosen `url`, `urlFromEvent`, hash/size and the full `candidates`
|
||||
walk (`url`, `outcome` ∈ match / version-mismatch / too-large / download-failed /
|
||||
timeout, `productVersion`).
|
||||
|
||||
Outcome precedence when no candidate archived: any transient failure (download/timeout)
|
||||
→ `Failed:Download` / `Failed:Timeout`, retryable via the queue's visibility ladder (the
|
||||
down host might have had it); every candidate over the cap → `Failed:TooLarge`; otherwise
|
||||
`Failed:VersionMismatch` — **not** retryable by the queue (retrying in minutes changes
|
||||
nothing), left to the sighting re-queue above. Storage/unexpected errors stay
|
||||
`Failed:Error`, retryable. A 409 on the write-once upload means an earlier attempt
|
||||
archived the version: the provenance sidecar is healed from the archived bytes (hash,
|
||||
size, ProductVersion) if missing — this is also how a manual skill backfill turns a null
|
||||
row into `Archived` on the next authoritative sighting.
|
||||
|
||||
## Manual path (skill)
|
||||
|
||||
`/ime-decompile` downloads from the blob (`ime-blob.sh download`), and
|
||||
`fetch-ime-decompile.py` names the decompile folder after the **actual** FileVersion of
|
||||
the extracted binaries — so the decompile repo can never be poisoned, only the blob folder.
|
||||
When the blob's version mismatches, look for `msiDownloadUrl` in *other* sessions of the
|
||||
version (the first-seen session often lacks the enrichment) and fetch from that host.
|
||||
`ime-blob.sh upload … --overwrite` replaces a blob known to hold the wrong build; merge the
|
||||
row's `MsiSha256` / `MsiBytes` / `MsiSourceUrl` afterwards.
|
||||
|
||||
# Examples
|
||||
|
||||
* First sighting of 1.105.103.0 with no URL: primary → 1.105.101.0 (mismatch), secondary →
|
||||
1.104.102.0 (mismatch), hotfix → 1.105.103.0 (match) → archived from hotfix, provenance
|
||||
lists all three.
|
||||
* Hotfix host down during the walk, others mismatching → `Failed:Download`, queue retries.
|
||||
* Version 1.101.111.0 (predates the archiver, row status null): a session whose CSP row
|
||||
matches by ProductVersion arrives → status `Queued`, job runs; if no host serves it any
|
||||
more → `Failed:VersionMismatch`, next try no sooner than 24 h later.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiver.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Ime/MsiProductVersionReader.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Ime/ImeMsiArchiveQueueWorker.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs` (ime_agent_version block)
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs` (`RecordImeVersionAsync`, `UpdateImeVersionArchiveInfoAsync`)
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ImeMsiInstallSourceProbe.cs` (event enrichment)
|
||||
* `.claude/commands/scripts/ime-blob.sh`, `.claude/commands/scripts/fetch-ime-decompile.py`
|
||||
* Tests: `ImeMsiArchiverTests`, `MsiProductVersionReaderTests` (synthetic packages + local sweep over `ime-files/`)
|
||||
@@ -1,100 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: IME Pattern Health — the pattern-drift loop
|
||||
description: How a Microsoft change to IME log wording becomes visible before it breaks timelines — the agent's session-end pattern histogram, the global per-IME-version statistics, the data-driven drift alarm, and the operator page/MCP tool that turn the alarm into a repair workflow.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/Ime
|
||||
tags: [backend, agent, ime, log-patterns, drift, ops-events, metrics, mcp]
|
||||
timestamp: 2026-08-30
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
The IME log-pattern pack is the product's core signal source, and Microsoft changes IME log
|
||||
wording without notice. Until now a changed line was invisible until sessions looked wrong:
|
||||
the agent kept matching nothing, the client log said nothing, and the only alarm was the
|
||||
registry/log reconciliation for app states. Tracker problems (skipped lines, regex timeouts)
|
||||
were a client-log Warning nobody sees without pulling diagnostics.
|
||||
|
||||
# Design
|
||||
|
||||
Three layers, one loop:
|
||||
|
||||
## 1. Tracker health in `agent_metrics_snapshot`
|
||||
|
||||
`ImeLogTracker` keeps cumulative counters (`ImeTrackerHealth`: lines read, entries matched,
|
||||
oversized lines, regex timeouts, per-line budget breaks, held tails, unanchored patterns,
|
||||
files tailed, backlog bytes = Σ file length − bookmark). `AgentSelfMetricsCollector` reads them
|
||||
through a read-only probe (`ImeLogHost.GetTrackerHealth`) and adds `ime_*` keys to the periodic
|
||||
snapshot next to the spool queue. Fleet expectation for every skip counter: **0**. The session
|
||||
page's "IME Tracker" section shows queue depth and skips.
|
||||
|
||||
## 2. `ime_tracker_degraded` (Warning, one-shot, immediate upload)
|
||||
|
||||
The first pass that skipped work raises `ImeLogTracker.OnTrackerDegraded`; the adapter emits
|
||||
the event with the counters, file and first skipped pattern. Persisted flag (`ImeTrackerState`)
|
||||
so a restart cannot re-emit. Listed in the MCP `health_events` catalog; informational for the
|
||||
idle/stall clocks (`SignalActivityClassifier`).
|
||||
|
||||
## 3. `ime_pattern_hits` → `ImePatternStats` → `ImePatternDriftSuspected`
|
||||
|
||||
* **Agent.** At termination (same slot as `app_tracking_summary`, so WhiteGlove Part 2 reports
|
||||
the whole session via the persisted histogram) the handler emits `ime_pattern_hits`:
|
||||
`hits` = every ENABLED pattern ID → match count, zeros included, plus the health counters and
|
||||
`imeVersion` (IME's own "Agent version is:" line). Only sessions with a terminal run report —
|
||||
crashes/kills are outside the denominator by construction. Severity is Debug: the event is an
|
||||
operator/backend signal, hidden from the default session timeline; the session page's
|
||||
"IME Tracker" card and the ingest fold read it regardless of severity.
|
||||
* **Ingest.** `EventIngestProcessor` hands the event to `ImePatternHealthService` fire-and-forget:
|
||||
IDs are filtered against `BuiltInImeLogPatterns.BuiltInPatternIds` (a device may only claim an
|
||||
ID; tenant custom IDs never reach the global table), the version must pass
|
||||
`ImeMsiArchiver.IsPlausibleVersion` (event value first, session row as fallback), then one
|
||||
partition read + one transactional batch upsert into `ImePatternStats`
|
||||
(PK = version, RK = patternId: `Sessions`, `SessionsWithHit`, `Hits`, `LastHitAt`,
|
||||
`DriftFlaggedAt`). Permanent table (`TableLifecycleBucketTests` KeptByDesign) — the baseline
|
||||
needs the established versions.
|
||||
* **Drift statistic (`ImePatternDriftEvaluator`, pure).** No hard-coded must-hit list.
|
||||
Baseline = the version with the most reporting sessions (≥ 100, never the candidate).
|
||||
A pattern is *expected* when its baseline hit rate (`SessionsWithHit / Sessions`) is ≥ 0.8.
|
||||
Drift is suspected when the candidate has ≥ 25 reporting sessions and an expected pattern
|
||||
matched in none of them. Conditional patterns (platform/remediation scripts, WinGet, errors)
|
||||
sit below 0.8 on the baseline and cannot alarm. One alarm per version × pattern:
|
||||
`TryMarkImePatternDriftFlaggedAsync` (ETag-conditional) then
|
||||
`OpsEventService.RecordImePatternDriftSuspectedAsync` (Warning, Agent, `System.Ingest`;
|
||||
dual-registered in the web `OPS_EVENT_TYPES`). The stats snapshot used for evaluation is
|
||||
cached 10 minutes and patched with each batch, so a version crossing the threshold is judged
|
||||
promptly.
|
||||
* **Read side.** `GET metrics/ime-pattern-health` (`GlobalReadOrAdmin`) returns baseline,
|
||||
thresholds, versions (with `ImeVersionHistory` first/last seen and fleet session count),
|
||||
patterns (catalog + retired IDs still in the statistics), cells and open alerts. Web:
|
||||
Global Admin → Metrics → IME Pattern Health (matrix, legend: drift / silent / low / few).
|
||||
MCP: `get_ime_pattern_health` (GA + Global Reader).
|
||||
|
||||
# Workflow on an alert
|
||||
|
||||
1. `ImePatternDriftSuspected` names version + pattern; the matrix shows the cell.
|
||||
2. `search_sessions imeAgentVersion=<version>` → `get_session_diagnostics` on a session with a
|
||||
package (upload mode must not be Off).
|
||||
3. `/ime-pattern-validate` against the real log; `/ime-decompile` for the version's sources.
|
||||
4. Fix the pattern under `rules/ime-log-patterns/` (anchored, linear —
|
||||
[IME Log Tracker](../agent/ime-log-tracker-matching-budget.md)); Backend-Deploy delivers it
|
||||
through the config channel without an agent release.
|
||||
|
||||
# Limits
|
||||
|
||||
* Denominator = sessions that reached a terminal run; a version that only crashes never gets
|
||||
a histogram and therefore never alarms here (it alarms elsewhere).
|
||||
* The baseline is fleet-wide: a pattern that only some tenants can produce stays "conditional"
|
||||
even if it is 100 % within those tenants.
|
||||
* Thresholds are constants (`ImePatternDriftEvaluator`), tuned for the current fleet size
|
||||
(~25k sessions/month); revisit when versions roll out faster than 25 sessions per day.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeTrackerHealth.cs`, `ImeLogTracker.cs` (`GetHealthSnapshot`, `OnTrackerDegraded`), `ImeTrackerStatePersistence.cs`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs` (`EmitImePatternHits`)
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/AgentSelfMetricsCollector.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Ime/{ImePatternHealthService,ImePatternDriftEvaluator}.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.ImePatternStats.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetImePatternHealthFunction.cs`
|
||||
* `src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionImePatternHealth.tsx`, `imePatternHealthLogic.ts`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts` (`get_ime_pattern_health`)
|
||||
@@ -1,64 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Lifecycle Manifests & Session Scope Resolution
|
||||
description: The enforced completeness nets added by the fragility-audit P5/P6 remediation — table lifecycle buckets, the SessionsIndex field manifest, the single session-scope resolver with its SessionTenantLookup point-read, and the server-side dashboard free-text search.
|
||||
resource: backend
|
||||
tags: [lifecycle, offboarding, sessionsindex, manifest, scope-resolution, search, performance]
|
||||
timestamp: 2026-08-13
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Four mechanisms replace hand-synchronized copies with a declared source plus tests that fail on drift.
|
||||
|
||||
## 1. Table lifecycle buckets
|
||||
|
||||
Every table in `Constants.TableNames` must belong to at least one deletion lifecycle:
|
||||
|
||||
| Bucket | Mechanism | Source of truth |
|
||||
|---|---|---|
|
||||
| Tenant-offboarding wipe | `TenantOffboardingHandler` bucket arrays (`TenantPartitionTables`, `CompositePartitionTables`, `DiscriminatorTables`, `PropertyOnlyTables`, `RowKeyTables`, `ArchivedRuleTables`) + the one-off `TenantConfiguration` final delete | handler arrays (read via reflection by the test) |
|
||||
| Per-session cascade | `DeletionManifestBuilder.BuildAsync` step list (incl. the Final tombstone → Sessions + SessionsIndex) | a real manifest built against an empty-inventory fake |
|
||||
| Kept by design | reviewed exception list with per-table reasons | `TableLifecycleBucketTests.KeptByDesign` |
|
||||
|
||||
`TableLifecycleBucketTests` (Functions.Tests) asserts: constants set-equal `TableNames.All`, no table unclassified, kept-by-design disjoint from every wipe path, bucket entries are real constants, offboarding buckets disjoint except documented multi-membership (`BootstrapSessions`).
|
||||
|
||||
**Not covered mechanically:** whether a bucket's key shape matches the table's writer. That was the UsageMetrics bug (PK=date, RK=tenantId sat in the exact-PK bucket; the wipe matched 0 rows) — SafeWipe **Variant D** (`WipeByRowKeyAsync`, `RowKey eq '{tenantId}'`) now covers RowKey-anchored tables. Read the writer's `new TableEntity(pk, rk)` before classifying a new table.
|
||||
|
||||
## 2. SessionsIndex field manifest
|
||||
|
||||
`SessionIndexFieldManifest` (Functions/Services) declares the full-mirror column set: `AlwaysProjected` (32, written with defaults on every rebuild), `ConditionallyProjected` (18, written only when present), `PrimaryOnly` (4 — ServerActions + deletion-CAS fields that never touch the index). `SessionIndexFieldManifestTests` pins `BuildSessionIndexEntity` against it bidirectionally (full-row fixture built from a test-local sample table, NOT from the manifest — deriving the fixture from the manifest would make the check circular), pins that every manifest field influences `MapToSessionSummary`, and pins the rebuild's `TableUpdateMode.Replace` (a merge-mode upsert could never clear a conditional column the primary blanked). `MergeSessionIndexAsync` logs a warning for merged keys outside the manifest (fail-soft drift guard).
|
||||
|
||||
## 3. Session scope resolution (one implementation)
|
||||
|
||||
`RequestContextExtensions.ResolveSessionScopeAsync(ctx, sessionRepo, sessionId, requireGlobalAdmin)` is the single implementation of the former 15-site copy-paste fallback ("which tenant owns this session?"):
|
||||
|
||||
- Non-global callers always keep the middleware-validated `TargetTenantId`.
|
||||
- Global-scope callers resolve via `ISessionRepository.ResolveSessionTenantIdAsync` — a point-read on **SessionTenantLookup** (PK=sessionId, RK="tenant") with a legacy cross-partition SessionsIndex scan fallback that self-heals the lookup row, so the scan is paid at most once per pre-table session.
|
||||
- `requireGlobalAdmin: true` on WRITE paths gates the cross-tenant reach on `IsGlobalAdmin` so a read-only Global Reader can never steer a write into a foreign tenant.
|
||||
- Unknown session → `TargetTenantId`, leaving each endpoint's own not-found semantics (404 / empty 200 / 400) in charge.
|
||||
- Delegated (MSP) callers deliberately get NO fallback — extending it to `AllowedTenantIds` is a pending user decision.
|
||||
|
||||
The lookup row is **claimed at registration, first-writer-wins** (`ClaimSessionTenantLookupAsync`, create-only `AddEntity` BEFORE any Sessions/SessionsIndex write): on 409 the existing owner is read back — same tenant ⇒ normal re-registration, different tenant ⇒ `SessionTenantConflictException` ⇒ `409 Conflict` with `ErrorCode=session_owner_mismatch` (owning tenant not disclosed) plus a `SessionTenantConflict` ops event, and nothing is written. The claim is deliberately NOT fail-soft. Before this, the row was an unconditional last-writer-wins upsert after the session write, so any tenant's device (self-service onboarding) that knew a victim session GUID (portal deep links, Teams/Slack notifications, progress-portal URLs) could re-point the mapping at its own tenant and have every no-`tenantId` Global Admin/Reader read — and GA writes such as the `globaladmin` annotation lane — silently served from a forged session. Session ids are 122-bit random, so a genuine cross-tenant collision does not occur; a conflict is an attack or an agent bug and fails loudly. The legacy self-heal write is Add-only too (an existing claim is the authority), and `scripts/Migration/Backfill-SessionTenantLookup.ps1` closes the pre-table window once (idempotent, Add-only) so no legacy session id is claimable. The row is deleted by the cascade one step before the tombstone (no tombstone is kept — a freed id gives a foreign tenant nothing a fresh GUID would not) and is rebuildable from Sessions (not in the critical-backup set). The former lazy triggers (`events.Count == 0` → scan) are gone: empty-but-owned results no longer pay a table scan, and endpoints resolve upfront.
|
||||
|
||||
`requireGlobalAdmin: true` additionally corroborates the resolved tenant against its Sessions row (one point-read): a mapping whose session does not exist in the resolved tenant keeps `TargetTenantId`, so a stale or poisoned row can never steer a GA write — silently, without any operator confirmation step (a "confirm tenant GUID" dialog was rejected as unusable). Read paths keep the single point-read; their own session read is the corroboration (404).
|
||||
|
||||
Related enforcement added with the migration: `RecomputeTriggerGate` — `?reanalyze=true` and `?rescan=true` on the MemberRead session routes are actions (delete + rewrite stored results) and now 403 for Viewer / Global Reader / cross-tenant tenant roles (Global Admin or own-tenant Admin/Operator only).
|
||||
|
||||
## 4. Server-side dashboard search
|
||||
|
||||
`q=` on `/api/search/sessions` + `/api/global/search/sessions`: case-insensitive substring over exactly the fields the dashboard's client-side filter searches (device name, serial, manufacturer, model, status, sessionId, geo, agent version, OS fields — NOT the client-derived date/duration/blocked tokens; a server-only field would produce ghost results the client filters back out). Azure Tables has no substring operator, so the scan path backfills whole Azure pages until it has `pageSize` matches (max 10 pages per request, continuation at a page boundary — gap-free). `q` is bound into the pagination fingerprint. The web dashboard's `searchAll()` (useDashboardSessions) follows nextLink up to 10 requests / 50 matches and dedupe-merges results into the loaded list (`mergeSessionsById`), replacing the former `loadAll()` walk that downloaded the entire session history to search it client-side. Delegated callers in cross-tenant mode keep the legacy full sweep (the global search route has no delegated tier).
|
||||
|
||||
# Examples
|
||||
|
||||
Adding a new table: classify it in one lifecycle path before `TableLifecycleBucketTests` lets it ship — offboarding bucket (check the writer's PK/RK shape first), deletion-manifest step, or `KeptByDesign` with a reason.
|
||||
|
||||
Adding a mirrored SessionsIndex field: manifest entry + `BuildSessionIndexEntity` write + `MapToSessionSummary` read + a sample value in the test — any missing step is red.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Backend/AutopilotMonitor.Functions.Tests/TableLifecycleBucketTests.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Services/SessionIndexFieldManifest.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Helpers/RequestContext.cs` (`ResolveSessionScopeAsync`)
|
||||
- `src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeService.cs` (Variant D)
|
||||
- `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.AgentApi.cs` (`MatchesFreeText`, backfill loop)
|
||||
@@ -1,107 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: OOBE Bootstrap go-Code Flow — Always-200 Script Endpoint
|
||||
description: The full lifecycle of the bootstrap short-code — created in the portal, typed as `irm go.autopilotmonitor.com/CODE | iex` at Shift+F10, served by GetBootstrapScriptFunction as an always-HTTP-200 PowerShell script — including the two-line injection defense and the rate-limit story behind Front Door.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Functions/Bootstrap/GetBootstrapScriptFunction.cs
|
||||
tags:
|
||||
- backend
|
||||
- bootstrap
|
||||
- security
|
||||
- oobe
|
||||
timestamp: 2026-07-28T00:00:00+02:00
|
||||
---
|
||||
|
||||
# OOBE Bootstrap go-Code Flow
|
||||
|
||||
Bootstrap Sessions let a technician install the agent **before** Intune
|
||||
enrollment: an admin creates a short-lived session in the portal (Settings →
|
||||
Bootstrap Sessions), hands the technician a short URL, and the technician types
|
||||
`irm https://go.autopilotmonitor.com/CODE | iex` into a Shift+F10 console
|
||||
during OOBE. The URL is deliberately short because it is typed by hand on a
|
||||
device that has no clipboard.
|
||||
|
||||
# Lifecycle
|
||||
|
||||
1. **Create** — `POST /api/bootstrap/sessions` (`BootstrapManagerOrGA`).
|
||||
`BootstrapSessionService.CreateAsync` mints an unambiguous-charset short
|
||||
code + a GUID token, validity clamped 1–168 h. The response's `bootstrapUrl`
|
||||
is built from `Constants.BootstrapGoBaseUrl` — the single producer of the
|
||||
absolute URL (pinned by `CreateBootstrapSessionUrlShapeTests`).
|
||||
2. **Serve** — `GET /api/bootstrap/go/{code}` (`GetBootstrapScriptFunction`,
|
||||
`PublicAnonymous`). Customers reach it through the Front Door custom domain
|
||||
`go.autopilotmonitor.com`, whose `/*` route rewrites onto the API path.
|
||||
Validation runs in-process (`ValidateCodeAsync` + the tenant's
|
||||
`BootstrapTokenEnabled` gate) — no HTTP hop to the validate endpoint.
|
||||
3. **Install** — the generated script downloads the agent ZIP from
|
||||
`Constants.AgentDownloadBaseUrl`, extracts, and runs
|
||||
`--install --no-auth --bootstrap-token … --tenant-id …`. The agent monitors
|
||||
from OOBE start, switches to certificate auth after Intune enrollment, and
|
||||
self-destructs on completion.
|
||||
|
||||
The legacy Next.js `/go/[code]` route (SSR on the SWA) stays alive only for
|
||||
URLs issued before the migration and is removed with the static-export PR —
|
||||
see the transition rule in [URL Registry](../url-registry.md).
|
||||
|
||||
# Always-200 contract
|
||||
|
||||
Every response — success, invalid format, unknown/expired code, disabled
|
||||
feature, rate limit, internal error — is **HTTP 200 `text/plain`**. The
|
||||
consumer is `irm | iex`: a non-2xx status would make `irm` throw an opaque
|
||||
exception on the OOBE console, while a 200 error script surfaces the actual
|
||||
message via `Write-Host 'ERROR: …'`. Consequences:
|
||||
|
||||
- Failures are invisible in status-code metrics; the compensating signal is the
|
||||
`LogWarning` per rejection (code + client IP).
|
||||
- Error messages are capped at 200 chars **before** quote-escaping and
|
||||
single-quote-doubled — a hostile upstream message cannot break out of the
|
||||
`Write-Host` literal.
|
||||
|
||||
A disabled tenant returns the **same generic message** as an unknown code — no
|
||||
enumeration oracle for whether a code exists.
|
||||
|
||||
# Injection defense (two lines)
|
||||
|
||||
The script runs as SYSTEM during OOBE, so interpolation is treated as a
|
||||
security boundary:
|
||||
|
||||
1. **`BootstrapScriptValueValidator`** — tenantId/token must be exact canonical
|
||||
GUIDs, the download URL must be https on an allow-listed host
|
||||
(`AgentDownloadBaseUrl`/`AgentBlobBaseUrl` hosts) with the strict
|
||||
`/agent/….zip` path shape, expiry must be future and ≤ 14 days. No accepted
|
||||
value can contain `$`, quotes, backticks, spaces, or newlines. Kept even
|
||||
though values are produced in-process — defense-in-depth against tampered
|
||||
table data.
|
||||
2. **Single-quoted PS literals** — every interpolated value sits inside
|
||||
single-quoted PowerShell strings, which do not expand `$()`, `$var`, or
|
||||
backticks.
|
||||
|
||||
The template is a C# raw string with `__TOKEN__` placeholders substituted via
|
||||
`Replace` (never `$`-interpolation — the script is full of PS `$vars`).
|
||||
`OobeBootstrapScriptGeneratorTests` pins byte parity with the original
|
||||
TypeScript template through a golden file, so the legacy route and the backend
|
||||
serve identical scripts during the transition.
|
||||
|
||||
# Rate limiting behind Front Door
|
||||
|
||||
The endpoint self-limits per client IP (`bootstrap-script:{ip}`, 20/min via
|
||||
`ClientIpExtractor.GetRateLimitClientIp`) — anonymous routes are skipped by
|
||||
`UserRateLimitMiddleware` and must bring their own limiter. Because traffic
|
||||
arrives through Front Door, the rightmost X-Forwarded-For hop resolves to AFD
|
||||
egress IPs and buckets would collapse (the shared-bucket 429 incident class);
|
||||
the AFD-aware extraction therefore honors `X-Azure-ClientIP` only when
|
||||
`X-Azure-FDID` matches the `FrontDoorProfileId` app setting (fail-closed to
|
||||
the trusted-hop path otherwise). Verified post-deploy 2026-07-29: real client
|
||||
IPs are keyed, not AFD egress.
|
||||
|
||||
Front Door route caching is DISABLED for this route: the body inlines the
|
||||
bearer token, so a cache hit would replay one requester's token to another.
|
||||
`NoStoreCacheMiddleware` covers the `/api/bootstrap/go/` prefix.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Backend/AutopilotMonitor.Functions/Functions/Bootstrap/GetBootstrapScriptFunction.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Functions/Bootstrap/OobeBootstrapScriptGenerator.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Security/BootstrapScriptValueValidator.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions.Tests/OobeBootstrapScriptGeneratorTests.cs` — golden parity
|
||||
- `src/Web/autopilot-monitor-web/app/go/[code]/route.ts` — legacy route (transition only)
|
||||
- [URL Registry](../url-registry.md) — go-URL registry entries + transition rule
|
||||
@@ -1,134 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Ops-Events Query Surface
|
||||
description: The server-side filter surface of GET /api/global/ops-events (category, eventType, severity/minSeverity, tenant drill, date window), why every filter is folded into the continuation fingerprint, why minSeverity expands to an OR-set, and why the paged cross-category read has to fan out over the complete category vocabulary.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Functions/Admin/GetOpsEventsFunction.cs
|
||||
tags:
|
||||
- backend
|
||||
- table-storage
|
||||
- ops-events
|
||||
- pagination
|
||||
- mcp
|
||||
timestamp: 2026-09-01T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
`OpsEvents` is the platform operator's log: consent flows, maintenance runs, security blocks,
|
||||
tenant offboards, agent emergency breaks, SLA and Azure Monitor alerts. It is partitioned by
|
||||
**category** (`PartitionKey`), with a reverse-tick `RowKey` for newest-first ordering.
|
||||
|
||||
Until 2026-09-01 the read endpoint could only narrow by category and date window. Every other
|
||||
question — *"did any `AgentEmergencyBreak` fire yesterday?"*, *"show me everything at Error or
|
||||
worse"* — had to be answered by pulling the whole category and filtering it on the client. For the
|
||||
MCP tool that client is a language model, so an event-type question cost a category-sized response
|
||||
in model context.
|
||||
|
||||
# Schema
|
||||
|
||||
`GET /api/global/ops-events` (GA + Global Reader; `TenantScoping.None`, so a delegated admin cannot
|
||||
reach it at all):
|
||||
|
||||
| Param | Where it is applied | Notes |
|
||||
| --- | --- | --- |
|
||||
| `category` | `PartitionKey eq` — indexed | One partition; omit to fan out over all of them |
|
||||
| `eventType` | `EventType eq` — server-side scan | **Exact and case-sensitive**; Table Storage cannot fold case |
|
||||
| `severity` | `Severity eq` — server-side scan | Normalised onto the canonical vocabulary before use |
|
||||
| `minSeverity` | OR-set over `Severity` — server-side scan | Threshold: that level and everything above |
|
||||
| `dateFrom` / `dateTo` | RowKey range — indexed | Both absent ⇒ backend resolves the last 30 days |
|
||||
| `tenantId` | **in-memory**, post-fetch | Drill, not an isolation boundary — see below |
|
||||
| `pageSize` / `continuation` | — | `pageSize` opts into pagination; absent ⇒ full window, no cap |
|
||||
|
||||
The filter surface lives in two shared pieces so no path can honour a different one:
|
||||
`OpsEventFilterRequest` (query → `OpsEventQueryFilters` + pagination extras) and
|
||||
`TableOpsEventRepository.AppendFieldFilters` (filters → OData clauses), which **both** filter
|
||||
builders call — the single-category/unpaged builder and the per-partition fan-out builder that adds
|
||||
the `RowKey gt` bound. This mirrors `AuditLogFilterRequest` / `AuditLogQueryFilters` on the audit
|
||||
endpoints one-for-one.
|
||||
|
||||
## Why `minSeverity` is an OR-set, not a range
|
||||
|
||||
Table Storage compares strings lexicographically, and the severity names sort
|
||||
`Critical < Error < Info < Warning` — nothing to do with severity order. A `Severity ge 'Warning'`
|
||||
clause would return `Warning` only, silently dropping `Error` and `Critical`, which is precisely the
|
||||
alert-worthy half. `minSeverity` therefore expands to
|
||||
`(Severity eq 'Warning' or Severity eq 'Error' or Severity eq 'Critical')`. The ladder itself lives
|
||||
in `OpsEventSeverity.Rank` and is shared with `OpsAlertDispatchService`, so *"Warning and above"* in
|
||||
a read means exactly what *"Warning and above"* meant when the alert rules decided to fire.
|
||||
|
||||
`minSeverity=Info` emits no clause at all — it is the floor, so the clause would only cost query
|
||||
length.
|
||||
|
||||
## Why severity is normalised but eventType is not
|
||||
|
||||
Both columns are matched with `eq`, which is case-sensitive. Severity has a closed vocabulary of
|
||||
four values, so `?severity=warning` is normalised to `Warning`, and anything outside the vocabulary
|
||||
is a **400** — an unnormalised value would return an empty page that reads exactly like *"no such
|
||||
events happened"*.
|
||||
|
||||
Event types have a declared vocabulary too (`OpsEventTypes`, reflected into the shared manifest and
|
||||
served to the model as `get_resource(name="ops_event_types")`), but the filter deliberately does
|
||||
**not** validate against it: stored rows outlive the vocabulary, so a type that was retired after a
|
||||
refactor must stay searchable in the historical window. The value is passed verbatim and a typo
|
||||
legitimately returns zero rows — the catalog is there so nobody has to guess.
|
||||
|
||||
## Continuation fingerprint
|
||||
|
||||
Every filter value is appended to the pagination *extras*, which are both folded into the
|
||||
continuation-token fingerprint and echoed on `nextLink`. Consequences:
|
||||
|
||||
* a token minted for one filter cannot page a different one (it is rejected with a 400),
|
||||
* the follow-up request is self-contained — the resolved date window and the filters travel on the
|
||||
link, so no page re-resolves "now",
|
||||
* extras are appended **after** the pre-existing `category` / `tenantId` discriminators and skipped
|
||||
when empty, so tokens minted before the field filters existed still fingerprint identically.
|
||||
|
||||
Because severity values are normalised *before* fingerprinting, `?severity=error` and
|
||||
`?severity=Error` share one token instead of minting two that each reject the other.
|
||||
|
||||
## The tenant drill stays in memory
|
||||
|
||||
`?tenantId=` is filtered post-fetch (`GetOpsEventsFunction.FilterByTenant`) because OpsEvents is
|
||||
partitioned by category, and — deliberately — because the in-memory comparison is
|
||||
`OrdinalIgnoreCase` while an OData `eq` would be case-sensitive: stored tenant casing is not
|
||||
guaranteed, so pushing this one down would lose rows. It is drill *correctness*, not isolation:
|
||||
every caller that reaches this route already has full cross-tenant scope. Pages therefore can report
|
||||
fewer items than `pageSize` when the drill is narrow.
|
||||
|
||||
## The category vocabulary must be complete
|
||||
|
||||
The paged cross-category read does not issue one cross-partition query (Azure would page it by
|
||||
`PartitionKey asc`, so the first page would come entirely from the alphabetically first category).
|
||||
It fans out per partition over a fixed list and merge-sorts. That list is therefore load-bearing: a
|
||||
category that is written but not listed is **invisible to every paged reader** while the unpaged
|
||||
full-table path still returns it. `Platform` (Azure Monitor alerts relayed through the ops alert
|
||||
webhook) was missing that way from its introduction until 2026-09-01.
|
||||
|
||||
The list is now owned by `OpsEventCategory.All`, and `OpsEventCategoryCoverageTests` reflects over
|
||||
the constants to fail the build when a new category is not added to it.
|
||||
|
||||
# Examples
|
||||
|
||||
```
|
||||
# One event type, last 24h, cross-category
|
||||
GET /api/global/ops-events?eventType=AgentEmergencyBreak&dateFrom=2026-08-31T12:00:00Z&pageSize=200
|
||||
|
||||
# Everything alert-worthy in the maintenance partition this month
|
||||
GET /api/global/ops-events?category=Maintenance&minSeverity=Error&dateFrom=2026-08-01T00:00:00Z
|
||||
```
|
||||
|
||||
MCP `get_ops_events` exposes the same surface plus a `days` shorthand, which it resolves to a
|
||||
concrete `dateFrom` **client-side** — a backend-side "last N days" would re-resolve "now" on every
|
||||
page and blow the token fingerprint. An explicit `dateFrom`/`dateTo` always wins over the shorthand.
|
||||
|
||||
# Citations
|
||||
|
||||
* `/src/Backend/AutopilotMonitor.Functions/Functions/Admin/GetOpsEventsFunction.cs`
|
||||
* `/src/Backend/AutopilotMonitor.Functions/Functions/Admin/OpsEventFilterRequest.cs`
|
||||
* `/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableOpsEventRepository.cs`
|
||||
* `/src/Shared/AutopilotMonitor.Shared/DataAccess/IOpsEventRepository.cs`
|
||||
* `/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs` — the declared event-type vocabulary
|
||||
* `/src/Backend/AutopilotMonitor.Functions.Tests/OpsEventFieldFilterTests.cs`
|
||||
* `/src/Backend/AutopilotMonitor.Functions.Tests/OpsEventCategoryCoverageTests.cs`
|
||||
* `/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts`
|
||||
* [Business Timestamps](business-timestamps.md) — why the date window filters on the RowKey
|
||||
@@ -1,151 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Ops Notification Channels
|
||||
description: How platform alerts are routed to named channels instead of three fixed provider slots — the shared NotificationChannel model on both the tenant and platform side, one NotificationChannelDispatcher that routes each channel to its transport, the legacy-slot synthesis that keeps pre-migration dispatch behaviour identical, the empty-means-all rule binding, why Telegram is Global-Admin-only by construction, and why an ops event's structured payload only reaches a destination when a rule opts in.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Services/OpsAlertDispatchService.cs
|
||||
tags:
|
||||
- backend
|
||||
- notifications
|
||||
- ops-events
|
||||
- configuration
|
||||
- security
|
||||
timestamp: 2026-09-01T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
Platform (ops) alerting had no routing. An `OpsAlertRule` was
|
||||
`{ EventType, MinSeverity, Enabled }` with no destination, and `OpsAlertDispatchService`
|
||||
broadcast every matching rule to all three hard-wired slots in `AdminConfiguration`
|
||||
(`OpsAlertTelegramChatId`, `OpsAlertTeamsWebhookUrl`, `OpsAlertSlackWebhookUrl`). Three
|
||||
consequences followed:
|
||||
|
||||
1. **No per-event destinations.** Binding a conversion signal to a sales channel meant every
|
||||
other rule reached that channel too.
|
||||
2. **Only the first matching rule was evaluated** (`FirstOrDefault`), so two rules on the same
|
||||
event type could not coexist.
|
||||
3. **The payload never left the backend.** `OpsEventService.WriteAsync` serialized a `details`
|
||||
object into the `OpsEvents` table but passed only `(category, eventType, severity, message,
|
||||
tenantId)` to the dispatch — a receiving system saw a tenant GUID and a sentence.
|
||||
|
||||
The tenant side already had the right model: `NotificationChannel` (a named list in
|
||||
`TenantConfiguration.NotificationChannelsJson`) with analyze rules binding channels by id
|
||||
(`RuleState.NotifyChannelIdsJson`). The platform side needed that model, not a second one.
|
||||
|
||||
# Schema
|
||||
|
||||
## One model, two scopes
|
||||
|
||||
`NotificationChannel` is unchanged and now backs both lists:
|
||||
|
||||
| Scope | Storage | Bound by |
|
||||
| --- | --- | --- |
|
||||
| Tenant | `TenantConfiguration.NotificationChannelsJson` | per-channel `NotifyOn*` toggles + analyze rules (`NotifyChannelIdsJson`) |
|
||||
| Platform | `AdminConfiguration.OpsNotificationChannelsJson` | `OpsAlertRule.NotifyChannelIds` |
|
||||
|
||||
The scopes stay separate on purpose — platform events resolve against platform channels, and a
|
||||
tenant's channel list can never receive an ops alert. What is shared is the *model* and the
|
||||
*dispatch path*, so a new provider is added once.
|
||||
|
||||
## One dispatch path
|
||||
|
||||
`NotificationChannelDispatcher` is the channel-level send API. It routes by provider:
|
||||
`WebhookProviderType.Telegram` goes to `TelegramNotificationService`; everything else is a
|
||||
rendered webhook POST through `WebhookNotificationService`. Every caller that sends to
|
||||
configured channels — enrollment (`EventIngestProcessor`), session start
|
||||
(`RegisterSessionFunction`), hardware rejection (`ReportDistressFunction`), SLA
|
||||
(`SlaBreachEvaluationService`), analyze rules (`AnalyzeOnEnrollmentEndHandler`), ops alerts
|
||||
(`OpsAlertDispatchService`) and both test endpoints — goes through it.
|
||||
|
||||
## Telegram is Global-Admin-only by construction
|
||||
|
||||
Telegram (`WebhookProviderType.Telegram = 40`) is the odd provider: it is not a webhook. The
|
||||
channel's `Url` carries the destination **chat ID**, and the bot token is platform-owned
|
||||
(PreviewConfig `WebhookUrl`). A tenant admin configuring one would be sending through *our*
|
||||
bot, which is why the gate is a server-side rule, not a hidden dropdown option:
|
||||
|
||||
* `TenantConfigValidation.ValidateTelegramChannelGate` refuses a non-GA caller who **adds** a
|
||||
Telegram channel or changes an existing one's destination or enabled state. It lives in
|
||||
`ValidateModel`, so it covers both write paths — the full-model PUT *and* the MCP field-patch
|
||||
flow (`TenantConfigPatchService`). A check placed in `UpdateTenantConfigurationFunction` alone
|
||||
would not.
|
||||
* An **unchanged** GA-created channel passes, so a tenant admin can still save unrelated config.
|
||||
* Telegram destinations skip `SsrfGuard.ValidateWebhookUrlFormat` (a chat ID is not a URL and
|
||||
would always fail it) and are format-checked as a numeric id or `@username` instead.
|
||||
|
||||
## Migration without a data job
|
||||
|
||||
`AdminConfiguration.GetOpsNotificationChannels()` returns the stored list, or — when it is empty —
|
||||
synthesizes one channel per *configured* legacy slot with stable ids (`legacy-telegram`,
|
||||
`legacy-teams`, `legacy-slack`) and the legacy enabled flags. `GetAdminConfigurationFunction`
|
||||
projects the same synthesis into the read model, so the editor shows exactly what receives
|
||||
alerts today and the first save materializes it. The legacy fields stay as the migration source
|
||||
and are no longer written by the Alerts section.
|
||||
|
||||
## Rule binding: empty means all
|
||||
|
||||
`OpsAlertRule.NotifyChannelIds` null or empty = every enabled channel. That is what every rule
|
||||
written before routing existed carries, so behaviour on the deploy day is identical. Dispatch
|
||||
resolves **all** matching rules (`Where`, not `FirstOrDefault`) and unions their channels;
|
||||
ids that no longer resolve are dropped rather than falling back to broadcast — a rule whose only
|
||||
channel was deleted must reach nothing, not everything. The web editor mirrors the same rule in
|
||||
`app/admin/components/opsChannelRouting.ts`.
|
||||
|
||||
## Payload delivery is opt-in per rule
|
||||
|
||||
`WriteAsync` hands the serialized `details` to `DispatchAsync`, but a channel only receives it when
|
||||
a rule sets `OpsAlertRule.IncludePayload`. **Default false**: an ops alert is a "something happened,
|
||||
go look" signal, most payloads are operational noise in a chat, and some carry data the baseline
|
||||
never does (tenant domain, administrator contact address). Widening what leaves the platform stays
|
||||
a deliberate per-rule decision.
|
||||
|
||||
`ResolveTargets` therefore returns two groups. Channels reached only by plain rules get an alert
|
||||
built with `detailsJson: null`; channels reached by an opted-in rule get the enriched one. A channel
|
||||
in both groups appears in the payload group only — one message per channel, and an unrelated
|
||||
sibling rule cannot cancel an explicit opt-in.
|
||||
|
||||
For the enriched alert, `BuildAlert`:
|
||||
|
||||
* flattens top-level **scalars** into `NotificationFact`s — capped at 12 facts and 256 characters,
|
||||
nulls and nested values skipped — so card formats and the plain-text Telegram message carry the
|
||||
same information,
|
||||
* passes the raw JSON through `NotificationAlert.DataJson`, which only `GenericJsonRenderer`
|
||||
emits, as a real JSON object under `data`. Additive within `schemaVersion` 1.0.
|
||||
|
||||
`NotificationAlert.EventType` is set either way: it is the same value the `Event` fact already
|
||||
carries, so it adds no information — only a stable key for a generic consumer to branch on.
|
||||
|
||||
# Examples
|
||||
|
||||
A trial-conversion signal reaching only a sales webhook:
|
||||
|
||||
1. Admin → Alerts → add a channel `Sales` (Generic JSON, the internal endpoint).
|
||||
2. Enable the `TenantTrialStarted` rule at `Info`, select **only** `Sales`, and switch its
|
||||
**Payload** toggle on (off by default, so every other rule keeps the baseline alert).
|
||||
3. `PlanManagementFunction` fires `TenantTrialStarted` from both plan write paths — the
|
||||
self-service `POST config/{tenantId}/trial` and a GA grant via `PATCH config/{tenantId}/plan`
|
||||
(distinguished by `selfService`). Ending a trial is a downgrade and stays with
|
||||
`TenantPlanDowngraded`.
|
||||
4. The webhook receives `eventType`, the rendered facts, and `data` with `domainName`,
|
||||
`contactEmail`, `trialStartedUtc`, `trialExpiresUtc`, `grantedBy`, `selfService`. The operator
|
||||
push channel receives nothing, because it is not bound to that rule.
|
||||
|
||||
# Gotchas
|
||||
|
||||
* **A new ops event type must be dual-registered** in `OPS_EVENT_TYPES`
|
||||
(`app/admin/components/OpsAlertRulesSection.tsx`) or it cannot be selected. `OpsEventTypeDualRegisterTests`
|
||||
enforces it.
|
||||
* **`OpsNotificationChannelsJson` is secret-bearing** and is on the deny-list in
|
||||
`AdminConfiguration.RedactedCopyForReader()`. `NotificationChannel.RedactList` /
|
||||
`RestoreRedactedList` are shared with the tenant list — do not add a second copy.
|
||||
* **Details are capped at 4096 characters** in the `OpsEvents` table (`TableOpsEventRepository`)
|
||||
before they ever reach a channel.
|
||||
|
||||
# Citations
|
||||
|
||||
* `/src/Backend/AutopilotMonitor.Functions/Services/OpsAlertDispatchService.cs`
|
||||
* `/src/Backend/AutopilotMonitor.Functions/Services/Notifications/NotificationChannelDispatcher.cs`
|
||||
* `/src/Shared/AutopilotMonitor.Shared/Models/Notifications/NotificationChannel.cs`
|
||||
* `/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs`
|
||||
* `/src/Backend/AutopilotMonitor.Functions/Helpers/TenantConfigValidation.cs`
|
||||
* `/src/Backend/AutopilotMonitor.Functions/Functions/Config/PlanManagementFunction.cs`
|
||||
@@ -1,93 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Plan Downgrade Policy & the Retention Grace Period
|
||||
description: What happens when a tenant loses Pro (explicit downgrade or trial expiry) — every entitlement gates read-time and reversibly, except retention, which gets a 30-day grace before the Community cap starts hard-deleting; the anchor model, the ops events, and the deliberate non-grace of everything else.
|
||||
resource: backend
|
||||
tags: [plan, edition, downgrade, retention, grace-period, trial, entitlements]
|
||||
timestamp: 2026-08-22
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
## The downgrade model: gate, don't destroy
|
||||
|
||||
Edition is resolved at READ time (`FeatureEntitlementCatalog.ResolveEdition`): Pro ⇔
|
||||
permanent tier `pro`/legacy `enterprise` OR an active trial. A plan mutation is a single
|
||||
field write (`PATCH config/{tenantId}/plan`, GA-only) — there is no downgrade event
|
||||
handler, no cascade. On Pro → Community every entitlement simply gates on the next read
|
||||
(≤ ~5 min config cache):
|
||||
|
||||
* **Unrestricted Mode** — effective value flips false; the GA on-request flag and the
|
||||
tenant toggle survive untouched (inert). Re-upgrade restores it verbatim.
|
||||
* **Delegated (MSP) admin** — scope suppressed at the home-tenant gate
|
||||
(`DelegatedAdminService.ApplyHomeTenantGateAsync`); grant rows / tenant groups /
|
||||
assignments are never touched. Resurrects on re-upgrade.
|
||||
* **OOBE bootstrap** — endpoints 403/404 unless the additive GA per-tenant flag
|
||||
(`BootstrapTokenEnabled`) is set; `BootstrapSessions` rows are kept.
|
||||
* **Rate limits / MCP quota** — pure computed-at-read floors/plan names.
|
||||
|
||||
**Retention is the one exception**: the retention sweep (12-hourly
|
||||
`SessionDeletionMaintenance` → `SessionRetentionFanoutService`) hard-deletes sessions
|
||||
older than `GetEffectiveRetentionDays` = `min(stored DataRetentionDays, edition cap)`.
|
||||
Without protection, a downgrade (or trial expiry) with stored retention 365 would start
|
||||
irreversibly cascade-deleting the 90–365-day band on the next tick.
|
||||
|
||||
## The retention grace period (30 days)
|
||||
|
||||
`FeatureEntitlementCatalog.RetentionDowngradeGraceDays = 30`. For 30 days after losing
|
||||
Pro, `GetEffectiveRetentionDays` keeps clamping to the **Pro** cap (365); only after the
|
||||
grace does the Community cap (90) bite. The stored `DataRetentionDays` is never rewritten.
|
||||
`days <= 0` (GA-only "infinite") short-circuits before any cap — those tenants are never
|
||||
swept, grace or not.
|
||||
|
||||
**Anchor** — "when was Pro lost" (`TenantEntitlementService.GetRetentionGraceEndUtc`,
|
||||
returns null while effectively Pro; grace end = latest anchor + 30d):
|
||||
|
||||
* **Explicit downgrade**: `TenantConfiguration.ProDowngradedUtc` (backend-only field, no
|
||||
agent ConfigVersion impact), maintained exclusively by the plan endpoint's pure mutation
|
||||
core `PlanManagementFunction.ApplyPlanChanges`. Stamped when the EFFECTIVE edition
|
||||
(before vs. after, same `nowUtc`) transitions Pro → Community — this covers both a
|
||||
planTier downgrade and a GA explicitly ending a trial (`trialExpiresUtc: null`, where
|
||||
the trial timestamp disappears and could not anchor). Cleared on any transition back to
|
||||
effectively Pro (PATCH pro, trial grant, self-service trial start), so a re-upgrade also
|
||||
resets the grace clock.
|
||||
* **Trial expiry**: no write happens — the stored `TrialExpiresUtc` itself is the anchor
|
||||
(read-time, like edition resolution). A planTier downgrade under a still-active trial
|
||||
deliberately does NOT stamp: the edition is still Pro, and the later trial expiry
|
||||
anchors the grace then.
|
||||
|
||||
The field is on the patch-endpoint `BaseDeniedFields` and `RevertProtectedFields` lists
|
||||
(a config revert must not time-travel the anchor) and roundtrips the table-serialization
|
||||
contract (Store+Map + roundtrip tests). Legacy rows read null → no anchor → no grace,
|
||||
which is correct: no tenant had been downgraded before the feature existed.
|
||||
|
||||
## Ops visibility (dual-registered event types)
|
||||
|
||||
* `TenantPlanDowngraded` (Warning) — emitted by the PATCH handler on an effective
|
||||
Pro→Community transition; carries `retentionGraceEndsUtc` + stored retention days. The
|
||||
response body also returns `retentionGraceEndsUtc`.
|
||||
* `TenantRetentionGraceExpiring` / `TenantRetentionGraceEnded` (Warning) — emitted by the
|
||||
daily `TrialExpirySweepFunction` (03:30 UTC) with the same stateless window mechanics as
|
||||
the trial events (≤3-day heads-up re-fired daily; ended once via the 24h look-back), and
|
||||
ONLY when data is actually at risk: stored `DataRetentionDays` above the Community cap
|
||||
(0 = infinite ⇒ never). Re-upgraded tenants go silent automatically (grace resolves null).
|
||||
|
||||
All three are registered in the web `OPS_EVENT_TYPES` catalog (OpsAlertRulesSection.tsx)
|
||||
so Telegram alert rules can route them.
|
||||
|
||||
## Deliberate non-goals
|
||||
|
||||
* No grace for Unrestricted/MSP/bootstrap/limits — reversible gates need none.
|
||||
* The retention WRITE cap (`TenantConfigValidation`) does not honor the grace: during the
|
||||
grace a tenant admin cannot SET a value above the Community cap. The grace protects
|
||||
existing data; new writes follow the new plan.
|
||||
* The feature-flags endpoint keeps publishing the plain edition cap
|
||||
(`retentionCapDays`) — it feeds the settings input maximum, which is the write cap.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TenantEntitlementService.cs` — grace + effective retention
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Config/PlanManagementFunction.cs` — `ApplyPlanChanges` anchor lifecycle
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/TrialExpirySweepFunction.cs` — grace event windows
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs` — enforcement point
|
||||
* Tests: `TenantEntitlementServiceTests`, `PlanManagementTransitionTests`, `TrialExpirySweepFunctionTests`, `SessionRetentionFanoutServiceTests`, `TenantConfigTableSerializationTests`
|
||||
@@ -1,112 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Rule Regression Radar (F3)
|
||||
description: How the radar detects analyze rules whose hit rate suddenly rose (7-day window vs 28-day baseline, Wilson-separated), which suppressions keep it quiet, how alert episodes live in the notification tracker, and which surfaces show them.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/RuleRegressionRadar.cs
|
||||
tags:
|
||||
- backend
|
||||
- rules
|
||||
- regression-radar
|
||||
- insights
|
||||
timestamp: 2026-07-27T16:00:00+02:00
|
||||
---
|
||||
|
||||
# Rule Regression Radar (F3)
|
||||
|
||||
Detects "this analyze rule suddenly fires much more often" before helpdesk volume does, and
|
||||
says what the spike correlates with — with honest statistics and honest wording. Analyze
|
||||
rules only in v1: their daily stats are audit-verified session-deduplicated
|
||||
(`FireCount` ≈ distinct sessions with a hit, `SessionsEvaluated` the same-source
|
||||
denominator). Gather rules are excluded until their per-batch stats dedup is fixed —
|
||||
never silently included with bad math.
|
||||
|
||||
# Schema
|
||||
|
||||
## Signal & detection
|
||||
|
||||
Per (tenant, ruleId), daily hit rate = `FireCount ÷ SessionsEvaluated` from the
|
||||
`RuleStats` rows — RATES, not counts (absolute counts spike with every rollout wave).
|
||||
A regression fires only when ALL hold for the trailing 7-day window vs. the prior
|
||||
28-day baseline:
|
||||
|
||||
1. window has ≥ 5 hit sessions AND ≥ 20 evaluated sessions;
|
||||
2. window rate ≥ 2× baseline rate;
|
||||
3. the Wilson 95 % intervals are disjoint in the increase direction
|
||||
(`MetricsMath.WilsonInterval` / `RateIncreaseSeparated` — deterministic, unit-pinned),
|
||||
so small-n noise never alerts.
|
||||
|
||||
## Suppression (false positives are trust damage)
|
||||
|
||||
* Rule entity missing (deleted) — never alerts.
|
||||
* Rule younger than baseline + window (`CreatedAt` within 35 days — grace period for new
|
||||
rules; the entity timestamp is the honest gate, a stats-row count would fail on
|
||||
low-volume tenants whose quiet days simply have no rows).
|
||||
* Rule edited inside the window (`UpdatedAt` ≥ window start) — edits legitimately change
|
||||
hit rates.
|
||||
* Empty baseline denominator (rule disabled throughout the baseline).
|
||||
* Platform kill switch: app setting `RuleRegressionRadarDisabled=true` skips the radar
|
||||
(fail-open — it only notifies, never mutates data).
|
||||
|
||||
## Alert episodes (tracker keyspace)
|
||||
|
||||
An episode lives as `ruleregression|{ruleId}` in the notification-tracker table
|
||||
(the `tpmpss|` pattern): the row IS the dedup (bell + ops event fire exactly once per
|
||||
episode), the badge state and the `regressions[]` payload — window/baseline counts,
|
||||
rates, lift (null for a zero-baseline "new signal" — no finite lift is invented), window
|
||||
dates, and the dimension concentration captured at first fire. Numbers refresh on every
|
||||
radar pass while the episode stays active; `FirstNotifiedAt` never moves. The episode
|
||||
closes (badge clears, dedup re-arms) when the window rate falls back under 1.5× baseline
|
||||
or fires stop entirely; the tracker's 30-day retention sweep also re-arms, so a
|
||||
month-old still-burning regression rings once more by design.
|
||||
|
||||
## Dimension correlation (on fire only)
|
||||
|
||||
Hit sessions (RuleResults scan: tenant partition range + RowKey = ruleId + DetectedAt in
|
||||
window, intersected with the window's sessions) are compared against all window sessions
|
||||
across `osBuild`, `model` (manufacturer + model), `agentVersion` and `imeVersion`. The
|
||||
top value by lift is reported with BOTH shares — wording contract: "X % of affected
|
||||
sessions are on … vs Y % of all sessions — correlated, not necessarily causal". No value
|
||||
with ≥ 5 hit sessions at ≥ 2× lift → "no clear dimension concentration"; a correlation
|
||||
failure also yields no claim. App-version correlation for `ANALYZE-APP-*` rules is a
|
||||
follow-up (needs rule→app matching semantics), tracked with the gather-rule fix.
|
||||
|
||||
## Compute & alert path
|
||||
|
||||
Runs in the daily maintenance pass right after the rule-stats aggregation
|
||||
(`RunRuleRegressionRadarAsync`), anchored on YESTERDAY — whole days only, a partial
|
||||
today would understate the window rate. One 35-day stats read covers all tenants;
|
||||
evaluation is the pure `RuleRegressionRadar` core. On a NEW episode:
|
||||
|
||||
1. `RuleFrequencyRegression` ops event (category Tenant, Warning) with the full numbers
|
||||
— dual-registered in the web `OPS_EVENT_TYPES` catalog for alert-rule routing.
|
||||
2. Tenant bell notification (type `rule_frequency_regression`, Admin audience) with the
|
||||
same numbers and a deep link to the rule's card (`/analyze-rules#rule-card-{ruleId}`).
|
||||
A sessions-list pre-filtered to rule hits needs a rule filter the session search does
|
||||
not have yet — follow-up alongside the gather-rule work.
|
||||
3. v1 channel: bell + ops event only. Provider fan-out (webhook/Telegram) follows once
|
||||
alert quality is proven — a noisy first version burns trust permanently.
|
||||
|
||||
# Surfaces
|
||||
|
||||
* `GET api/metrics/rule-stats` (and the global variant with `tenantId`) carries
|
||||
`regressions[]` — the active episodes. MCP `get_rule_stats` inherits the block.
|
||||
* Analyze-rules page: per-rule 30-day fire-count sparkline (densified from the `trend`
|
||||
rows the endpoint already served) and a red "↑ Regression" badge while an episode is
|
||||
active, with the full numbers in the tooltip.
|
||||
|
||||
# Examples
|
||||
|
||||
A rule firing on 2 % of sessions for a month jumps to 15 % over a week (15/100 vs
|
||||
10/500): lift 7.5×, Wilson-separated → one bell, one ops event, badge until the rate
|
||||
falls back under 3 %. The same jump on 5/20 sessions with a 12 % baseline stays silent —
|
||||
the intervals overlap, so the spike is not statistically real yet. A rule that never
|
||||
fired in the baseline and now hits 15 % alerts as a "new signal" with no lift figure.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/RuleRegressionRadar.cs` — gates, suppression, re-arm, dimension concentration.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs` — Wilson interval + separation primitive.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.RuleRegression.cs` — sweep, episode reconcile, bell/ops wiring.
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableHardwareRejectionNotificationTracker.cs` — episode keyspace.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/RuleRegressionRadarTests.cs` — hand-computed Wilson vectors + every suppression branch.
|
||||
* `tasks/insights-expansion-spec.md` — F3 specification and source-data audit (§0.5).
|
||||
@@ -1,84 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Send-Time Frame Separation (X-Send-Time-Utc)
|
||||
description: One device-clock send timestamp per upload batch splits ReceivedAt − OccurredUtc into pure spool delay and pure device-vs-server clock offset — turning clock diagnostics from median statistics into direct per-request measurement.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs
|
||||
tags: [backend, agent, ingest, timestamps, clock-skew, telemetry, provenance]
|
||||
timestamp: 2026-08-21
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
Events carry two time anchors: `OccurredUtc` (device clock at emission) and `ReceivedAt`
|
||||
(server clock at ingest). Their difference conflates two physically distinct phenomena:
|
||||
|
||||
* **Spool delay** — the event waited offline or in retry backoff before upload. Expected,
|
||||
harmless, minutes to hours.
|
||||
* **Device-vs-server clock offset** — the actual diagnostic signal.
|
||||
|
||||
A uniform spool backlog (events emitted close together, uploaded after an outage) is
|
||||
indistinguishable from a clock offset in that delta; a draining backlog fakes a clock jump
|
||||
ramp. Every consumer had to filter this statistically (spread caps, plateau guards,
|
||||
persistence tests) — and the first field clock-jump measurement failed exactly on this
|
||||
ambiguity.
|
||||
|
||||
# Schema
|
||||
|
||||
The agent stamps one header per upload attempt (`BackendTelemetryUploader`):
|
||||
|
||||
```
|
||||
X-Send-Time-Utc: <DateTime.UtcNow, ISO-8601 round-trip>
|
||||
```
|
||||
|
||||
Per **attempt**, not per batch content: a retry after backoff is a new send moment. The
|
||||
ingest function parses it (`ParseSendTimeHeader`: round-trip parse, UTC-normalized, values
|
||||
before year 2000 rejected as garbage — deliberately **no upper clamp**, a future-dated
|
||||
send time IS the clock-error measurement), and `StampServerFields` writes it as `SentAt`
|
||||
onto every event of the request — server-stamped, request-level, never trusted per-event
|
||||
from the wire. Stored as the nullable `SentAt` column on Events (absent for pre-header
|
||||
agents), mapped back everywhere events are read, exposed as `sentAt` in the `fields=`
|
||||
projection and verbatim on the raw endpoints.
|
||||
|
||||
The separation is then exact:
|
||||
|
||||
```
|
||||
SentAt − OccurredUtc = pure spool delay (both device frame; offset cancels)
|
||||
ReceivedAt − SentAt = network latency + clock offset (the only frame crossing)
|
||||
```
|
||||
|
||||
Network latency is seconds; anything beyond is device clock error, per request, no
|
||||
statistics required.
|
||||
|
||||
# Consumers
|
||||
|
||||
* **`clock_skew` analyze condition** (ANALYZE-DEV-008): batches whose events carry
|
||||
`SentAt` measure the batch offset directly as `SentAt − ReceivedAt` — spool-immune by
|
||||
construction, so the spread cap and the IME/clamped exclusions are unnecessary there
|
||||
(no event timestamps are read at all; IME-only sessions become measurable). Batches
|
||||
without `SentAt` keep the legacy per-event median + spread-cap path. Downstream logic
|
||||
(stability check, plateau step detection, end-state persistence) is shared; evidence
|
||||
carries `sentAtBatchCount`.
|
||||
* **`CmTraceSkewTripwire` deliberately unchanged**: `median(Δ_IME) − median(Δ_other)`
|
||||
cancels the device clock offset as common mode — send time adds nothing to that
|
||||
differential measurement.
|
||||
|
||||
# Constraints
|
||||
|
||||
* `SentAt` comes from the same device wall clock as the event timestamps — a clock jump
|
||||
between emission and send lands in the spool-delay term. That is the intended place for
|
||||
it to become visible, not a defect.
|
||||
* Signals and DecisionTransitions do not carry `SentAt` (no consumer); the header is
|
||||
request-level, so extending them later is a storage-only change.
|
||||
* Fleet coverage grows with agent rollout; all analyses must treat `SentAt == null`
|
||||
(legacy rows, old agents) as "legacy path", never as zero.
|
||||
|
||||
# Citations
|
||||
|
||||
* `BackendTelemetryUploader.UploadBatchAsync` — header stamping per attempt.
|
||||
* `IngestTelemetryFunction.ParseSendTimeHeader` + `IngestTelemetryFunctionTests` — parse
|
||||
contract incl. the no-upper-clamp decision.
|
||||
* `EventIngestProcessor.StampServerFields` + `IngestCriticalPathTests` — server-side
|
||||
stamping, forged per-event values overridden.
|
||||
* `RuleEngine.ConditionEvaluators.EvaluateClockSkewCondition` + `RuleEngineClockSkewTests`
|
||||
— the two measurement paths (the backlog-alias test pair pins why SentAt exists).
|
||||
* `tasks/todo.md` P14 (2026-08-21) — decision trail incl. why the tripwire stays as is.
|
||||
@@ -1,102 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ServerActions & On-Demand Diagnostics Collection
|
||||
description: The queue-on-session, deliver-on-ingest ServerAction channel (terminate_session / rotate_config / request_diagnostics), the operator-tier authorization split, the portal's Collect Logs flow including the admin quick-config path, and why the offboarding cascade sweeps the hosted diagnostics prefix.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Functions/Sessions/QueueSessionActionFunction.cs
|
||||
tags:
|
||||
- backend
|
||||
- agent
|
||||
- web
|
||||
- server-actions
|
||||
- diagnostics
|
||||
- authorization
|
||||
timestamp: 2026-07-24T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
**Channel.** A `ServerAction` (`Type`, `Reason`, `RuleId`, `Params`, `QueuedAt`) is queued onto
|
||||
`Sessions.PendingActionsJson` by server-side writers — the admin endpoint
|
||||
`POST /api/sessions/{sessionId}/actions`, the RuleEngine, maintenance — and delivered
|
||||
piggy-backed on the agent's next ingest response. No polling channel exists; delivery latency
|
||||
equals the agent's telemetry cadence (seconds to ~1 minute during enrollment). Delivery is
|
||||
at-least-once: the agent's `ServerActionDispatcher` dedups on `Type|RuleId|QueuedAt` and
|
||||
processes a batch sequentially in queue order. Unknown types are logged and skipped, so new
|
||||
types roll out server-first. Every action emits `server_action_received` and then
|
||||
`server_action_executed` or `server_action_failed` into the session timeline.
|
||||
|
||||
**Authorization (two layers).** The route sits on `EndpointPolicy.TenantAdminOrOperator`
|
||||
(tenant Admin, tenant Operator, or Global Admin — no Viewer, no Global Reader: it is a write
|
||||
tier). Because the route serves multiple action kinds, `QueueSessionActionFunction`
|
||||
re-gates per type via `IsTypeAllowedForCaller`: Operators may queue only
|
||||
`request_diagnostics`; `terminate_session` and `rotate_config` require
|
||||
`RequestContext.IsTenantAdmin || IsGlobalAdmin` (403 otherwise). Any route on this tier with
|
||||
a multi-kind body MUST follow this same in-function re-gating pattern.
|
||||
|
||||
**On-demand collection (`request_diagnostics`).** The dispatcher calls
|
||||
`DiagnosticsPackageService.CreateAndUploadAsync(enrollmentSucceeded: false, suffix:
|
||||
"server-requested")` — producing `AgentDiagnostics-{sessionId}-{timestamp}-server-requested.zip`.
|
||||
Passing `enrollmentSucceeded: false` is what makes on-demand collection work under
|
||||
`DiagnosticsUploadMode=OnFailure`; only `Off` (or an unconfigured destination) blocks it,
|
||||
surfacing as `server_action_failed`. The `diagnostics_uploaded` event stamps
|
||||
`DiagnosticsBlobName` + `Destination` onto the Sessions row (latest wins — an on-demand name
|
||||
is later replaced by the terminal package), and the SignalR `sessionUpdate` delta carries the
|
||||
field, so the portal's Download button flips live.
|
||||
|
||||
**Portal flow (Collect Logs, session detail header).** The button is always rendered; the
|
||||
disabled states carry tooltips (inactive session, cross-tenant view, missing role, diagnostics
|
||||
not configured) as a deliberate nudge. Members below Admin cannot read the tenant config, so
|
||||
the member-readable feature-flags endpoint exposes `diagnosticsUploadConfigured`
|
||||
(`mode != Off && (customer SAS present || destination Hosted)`). Progress tracking is
|
||||
sequence-based (backend-assigned, monotonic), never timestamp-based — agent clock skew and
|
||||
at-least-once redelivery of older actions cannot produce false matches
|
||||
(`collectLogsLogic.ts`). Cross-tenant views are gated off because `TenantHelper.GetTenantId`
|
||||
resolves the tenant strictly from the caller's JWT.
|
||||
|
||||
**Admin quick-config.** For an unconfigured tenant, a Tenant Admin's click opens a dialog
|
||||
that switches the tenant to `Destination=Hosted`, `Mode=OnFailure` (GET full config → patch
|
||||
exactly those two fields → PUT verbatim), then queues `rotate_config` **before**
|
||||
`request_diagnostics` — the dispatcher's sequential ordering guarantees the agent refetches
|
||||
the now-enabled config (the `RemoteConfigMerger` applies diagnostics fields live, remote wins)
|
||||
before building the package. This works against the already-deployed agent fleet; a
|
||||
"one-time override" variant would not, because the `mode=Off` check is client-side in
|
||||
`CreateAndUploadAsync` and the deployed handler reads no params.
|
||||
|
||||
**Offboarding cascade.** Latest-wins on `DiagnosticsBlobName` means a session can leave more
|
||||
hosted blobs behind than the deletion manifest references. The cascade's Hosted branch
|
||||
therefore sweeps the canonical prefix `{tenantId}/AgentDiagnostics-{sessionId}-`
|
||||
(`HostedDiagnosticsBlobService.DeleteBySessionPrefixAsync`) plus the manifest-named blob as
|
||||
belt-and-braces; a non-GUID legacy `SessionId` degrades to the single-name delete instead of
|
||||
poisoning the cascade. The CustomerSas branch is unchanged (delete only with the `d`
|
||||
permission; the customer's lifecycle rules stay authoritative).
|
||||
|
||||
# Examples
|
||||
|
||||
Queue an on-demand collection (Admin or Operator, own tenant):
|
||||
|
||||
```
|
||||
POST /api/sessions/{sessionId}/actions
|
||||
{ "type": "request_diagnostics", "reason": "On-demand log collection from portal" }
|
||||
→ 202 { "success": true, "queuedAt": "..." }
|
||||
```
|
||||
|
||||
Operator queueing an admin-only type:
|
||||
|
||||
```
|
||||
POST /api/sessions/{sessionId}/actions
|
||||
{ "type": "terminate_session" }
|
||||
→ 403 { "message": "Action type 'terminate_session' requires the Tenant Admin role" }
|
||||
```
|
||||
|
||||
Timeline for a successful round trip: `server_action_received` → `server_action_executed`
|
||||
(with `blobName`) → `diagnostics_uploaded` → Sessions row + SignalR delta carry the new
|
||||
`DiagnosticsBlobName`.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Sessions/QueueSessionActionFunction.cs` — allowed types, per-type caller gate.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs` — `TenantAdminOrOperator` tier definition.
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/ServerActionDispatcher.cs` — dedup, sequential dispatch, telemetry contract.
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs` — mode gates, archive build, suffix.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Diagnostics/DiagnosticsBlobCascadeDeleter.cs` — hosted prefix sweep.
|
||||
* `src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/collectLogsLogic.ts` — button gating matrix, sequence-based progress evaluation.
|
||||
@@ -1,122 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Session Annotations — Role-Separated Human Verdicts
|
||||
description: The SessionAnnotations table (PK=tenantId, RK={sessionId}_{lane}) storing per-session human verdicts + notes in three role lanes (operator / tenantadmin / globaladmin), the per-lane write matrix with own-tenant binding, the platform-internal GA lane, the fired-rule-id snapshot that makes annotations rule-quality labels, and the portal/MCP surfaces.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Functions/Annotations/UpsertSessionAnnotationFunction.cs
|
||||
tags:
|
||||
- backend
|
||||
- web
|
||||
- mcp
|
||||
- annotations
|
||||
- authorization
|
||||
- rule-quality
|
||||
timestamp: 2026-08-11T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
**Table.** `SessionAnnotations`: PartitionKey = `tenantId`, RowKey = `{sessionId}_{lane}` with
|
||||
lane ∈ `operator | tenantadmin | globaladmin` (a closed set — `AnnotationLanes.All`). One row per
|
||||
session + lane, edited in place; per-session reads use the RK range `ge '{sid}_' / lt '{sid}_~'`.
|
||||
Serialization lives only in `TableSessionAnnotationRepository.StoreAnnotation`/`MapAnnotation`
|
||||
(round-trip-pinned; corrupt `RuleIdsJson` degrades to an empty list on read).
|
||||
|
||||
**Row.** `Verdict` (nullable: `root_cause_confirmed | analysis_wrong | different_problem |
|
||||
inconclusive`), `Note` (nullable, ≤ 4096 chars), `AuthorUpn`/`AuthorDisplayName` (last editor),
|
||||
`CreatedByUpn`/`CreatedAtUtc` (first writer, immutable across edits), `UpdatedAtUtc`, and
|
||||
`RuleIdsJson` — a **snapshot of the rule ids that had fired for the session at write time**
|
||||
(from `IRuleRepository.GetRuleResultsAsync`, fail-soft to empty). The snapshot is what turns an
|
||||
annotation into a rule-quality label without a join: filtering annotations by `ruleId` + `verdict`
|
||||
yields confirmed vs false-positive counts per rule even after results are re-analyzed or deleted.
|
||||
|
||||
**Verdict + note are independently optional**; a PUT with both null **deletes** the lane row
|
||||
(there is no DELETE route). Author identity is always stamped server-side from the JWT
|
||||
(`TenantHelper.GetUserDisplayName` + `RequestContext.UserPrincipalName`) — a body-supplied
|
||||
author never wins (same anti-spoof contract as the gather-rule PUT-upserts).
|
||||
|
||||
# Authorization
|
||||
|
||||
Routes (all in `EndpointAccessPolicyCatalog`):
|
||||
|
||||
| Route | Policy | Notes |
|
||||
| --- | --- | --- |
|
||||
| `GET sessions/{sessionId}/annotations` | `MemberRead` + QueryParam | handler filters the GA lane |
|
||||
| `PUT sessions/{sessionId}/annotations/{lane}` | `TenantAdminOrOperator` + QueryParam | re-gated per lane in-function |
|
||||
| `GET global/session-annotations` | `GlobalReadOrAdmin` + QueryParam | evaluation stream; delegated ("MSP") readers are rescue-admitted on a managed `?tenantId=` only |
|
||||
|
||||
**Per-lane write matrix** (`UpsertSessionAnnotationFunction.IsLaneWritableByCaller`, an
|
||||
`internal static` pure function like `QueueSessionActionFunction.IsTypeAllowedForCaller`):
|
||||
`operator` → tenant Operator or tenant Admin; `tenantadmin` → tenant Admin; `globaladmin` →
|
||||
Global Admin only. **Own-tenant binding:** the session's tenant is resolved *before* the gate
|
||||
(GA cross-tenant fallback via `FindSessionTenantIdAsync`), and tenant-role inputs are zeroed
|
||||
when the resolved tenant differs from the caller's JWT tenant — so a GA who is also an admin of
|
||||
their home tenant can never write another tenant's operator/tenantadmin lanes. Cross-tenant, a
|
||||
GA writes exactly the `globaladmin` lane (the platform labeling flow).
|
||||
|
||||
**GA lane is platform-internal.** `GetSessionAnnotationsFunction.FilterLanesForCaller` drops the
|
||||
`globaladmin` lane for every caller without `HasGlobalScope` (tenant members and delegated
|
||||
admins). Both list endpoints enforce the same rule server-side via the repository's
|
||||
`excludeGlobalAdminLane` OData clause — on the global route this matters exactly for the
|
||||
delegated-rescue path, which reaches it without global scope. Tenant lanes are readable by all
|
||||
members of that tenant.
|
||||
|
||||
**Audit.** Non-GA writes/clears log `LogAuditEntryAsync(..., "SessionAnnotation",
|
||||
"{sessionId}/{lane}", ...)`; GA writes are skipped (platform-internal labeling must not surface
|
||||
in the tenant-visible audit log — same convention as GA session-report submissions).
|
||||
|
||||
# Lifecycle
|
||||
|
||||
* **Per-session cascade delete**: three exact-RK steps (one per lane) in `DeletionManifestBuilder`
|
||||
(orders 17–19; inventory 20/21, tombstone 22). `ComputePreflightCounts` sums repeated-table
|
||||
steps so the `sessionAnnotations` count covers all lanes.
|
||||
* **Tenant offboarding**: `SessionAnnotations` is in the PK=tenantId wipe list — deliberately
|
||||
wiped, unlike `Feedback` (annotations are customer session data, not product feedback).
|
||||
* **Backup**: member of `Constants.CriticalBackupTables.All` — hand-labeled data is the most
|
||||
expensive data on the platform to lose.
|
||||
* Being in `Constants.TableNames.All` auto-creates the table at startup and exposes it to the
|
||||
raw `list_tables`/`query_table` tools.
|
||||
|
||||
# Surfaces
|
||||
|
||||
* **Portal**: `SessionAnnotationsCard` on the session detail page (`#section-annotations`, after
|
||||
Analysis; registered in BOTH the section div and the `sessionSections` sidebar registry). The
|
||||
card mirrors the matrix via the pure module `sessionAnnotationLogic.ts` (vitest-pinned) and
|
||||
uses the explicit-Save pattern; the backend re-gates every save. Collapsed by default with a
|
||||
count + verdict-pill summary header.
|
||||
* **Portal overview page** `/annotations` (Monitoring sidebar group — deliberately NO new
|
||||
"Insights" group; Fleet Health & co. are insights too and a second analytics group would
|
||||
dilute both): tenant-scoped list backed by `GET sessions/annotations/list`
|
||||
(`ListTenantSessionAnnotationsFunction`, MemberRead + QueryParam). The path sits under the
|
||||
`/api/sessions` client-cert exclusion prefix (rename-over-exclusion convention; the Azure
|
||||
exclusion list is portal-only config) and uses THREE segments because a two-segment literal
|
||||
would be eaten by the `sessions/{sessionId}` route template. The tenant is always the
|
||||
middleware-validated `TargetTenantId`, and for callers without global scope the repository
|
||||
adds a server-side `Lane ne 'globaladmin'` OData clause — hidden rows never consume page
|
||||
budget, and an explicit lane=globaladmin filter self-contradicts to an empty page. Rows
|
||||
deep-link to `sessions?id=…#section-annotations`. The page carries the standard tenant
|
||||
switcher (`useAggregatedAdminScope` + `TenantScopeSelector`): a GA/Reader defaults to the
|
||||
"All tenants" aggregate (the portal twin of MCP's `list_session_annotations`) with per-tenant
|
||||
drill-down, a delegated ("MSP") admin gets the managed subset; cross-tenant scopes route to
|
||||
`GET global/session-annotations`, a tenant column appears, and deep links carry the row's
|
||||
`tenantId`.
|
||||
* **MCP read**: `get_session_summary` carries an `annotations` key (4th parallel leg, fail-soft
|
||||
null); `list_session_annotations` (`ga`-gated) is the evaluation stream with `tenantId / lane /
|
||||
verdict / ruleId / dateFrom / dateTo` filters and nextLink pagination. The `ruleId` filter is
|
||||
applied server-side but client-of-Azure (substring on the JSON column), so the backend
|
||||
back-fills short pages by looping the Azure continuation (bounded rounds) — filtered-out rows
|
||||
never consume page budget.
|
||||
* **MCP write**: `annotate_session` (`strictGa`-gated, MUTATING) always PUTs the `globaladmin`
|
||||
lane — no lane argument by design. This is the frictionless labeling path after a session-debug.
|
||||
* **Session reports**: `SessionReportService.SubmitReportAsync` snapshots ALL lanes into
|
||||
`annotations.json` inside the report ZIP (`annotationCount` in report-metadata.json; fail-soft).
|
||||
Safe because report blobs are readable only through the GA-only session-reports routes, and
|
||||
valuable because the ZIP outlives the session's retention/delete.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/SessionAnnotation.cs` — model, lanes, verdicts.
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionAnnotationRepository.cs` — Store/Map + query back-fill.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Annotations/` — the three endpoints.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Pagination/SessionAnnotationsPagination.cs` — filter-bound continuation tokens.
|
||||
* `src/Web/autopilot-monitor-web/app/sessions/components/sessionAnnotationLogic.ts` — portal matrix mirror.
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts` — `list_session_annotations`, `annotate_session`.
|
||||
@@ -1,157 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Session Owner Binding — Session-Level Authorization for Agent Writes
|
||||
description: Why tenant-level agent authentication (any device certificate, or a bootstrap token) was enough to rewrite every session in the tenant, how each Sessions row is now bound to the device identity that created it, the rule matrix that keeps every legitimate lifecycle working, the shadow-then-enforce rollout, and why enforcement needs the agent to rotate its session id rather than count an auth failure.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Security/SessionOwnershipPolicy.cs
|
||||
tags:
|
||||
- backend
|
||||
- security
|
||||
- authorization
|
||||
- sessions
|
||||
- agent
|
||||
- multi-tenant
|
||||
timestamp: 2026-08-29T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
Agent endpoints authenticate at **tenant** granularity: a chain-valid Intune MDM certificate whose
|
||||
tenant stamp matches the request (see [client-cert-tenant-binding.md](client-cert-tenant-binding.md)),
|
||||
or a bootstrap token minted for the tenant. They authorize writes at **session** granularity using a
|
||||
session id the caller supplies — `IngestTelemetryFunction` takes it from the `PartitionKey` of the
|
||||
first item, `RegisterSessionFunction` from the body, `ReportAgentErrorFunction` from the body. Until
|
||||
2026-08-29 nothing on the Sessions row recorded which device created it, so nothing could be compared.
|
||||
|
||||
The consequence: one enrolled device — or anyone who saw a 6-character bootstrap short code — could,
|
||||
for every other session of the tenant, forge `enrollment_complete` / failure events (flipping the live
|
||||
status, firing webhooks and SLA logic), pollute the timeline, re-register the session with different
|
||||
device metadata (which also redirects the Progress-Portal serial-knowledge proof), and — worst — send a
|
||||
Signal-only batch that **fetches and clears the victim's pending ServerActions**, silently stealing
|
||||
admin commands such as diagnostics collection. Session ids are GUIDs, but they travel in portal URLs
|
||||
and tenant SignalR messages; they were never meant to be a capability.
|
||||
|
||||
# Mechanism
|
||||
|
||||
## Owner on the row
|
||||
|
||||
`SessionOwner` (Shared) is stamped by the backend from the validated caller identity — never from
|
||||
agent input — and lives in six primary-only Sessions columns (`OwnerKind`, `OwnerThumbprint`,
|
||||
`OwnerDeviceId`, `OwnerBootstrapCode`, `OwnerSerial`, `OwnerBoundAt`). They are listed in
|
||||
`SessionIndexFieldManifest.PrimaryOnly`: never mirrored to `SessionsIndex`, never on `SessionSummary`,
|
||||
so no certificate identity travels to clients.
|
||||
|
||||
| Auth path | Identity that binds | Proven by |
|
||||
|---|---|---|
|
||||
| Certificate | `OwnerThumbprint` + `OwnerDeviceId` (Intune device id from the certificate CN, lower-case GUID) | TLS handshake; the CN survives a certificate re-issue |
|
||||
| Bootstrap token | `OwnerBootstrapCode` + `OwnerSerial` | Token possession + the serial the caller announced (header, unverified) |
|
||||
|
||||
`SecurityValidationResult.IntuneDeviceId` is new: the CN was already extracted for the Cloud PC
|
||||
validator, now it is surfaced on the result for every certificate-authenticated request.
|
||||
|
||||
## Rule matrix
|
||||
|
||||
`SessionOwnershipPolicy.Evaluate(existingRow, validation, now)` is pure. It returns an outcome, the
|
||||
owner to stamp (if any) and whether the caller's serial equals the row's serial.
|
||||
|
||||
| Caller | Row owner | Outcome | Stamps | Would reject |
|
||||
|---|---|---|---|---|
|
||||
| any | no row | `Fresh` | caller | no |
|
||||
| any | no owner columns (legacy), serial equal | `ClaimLegacy` | caller | no |
|
||||
| any | no owner columns, serial differs | `LegacySerialMismatch` | — | **yes** |
|
||||
| any | — (validation carried neither thumbprint nor code) | `CallerUnidentified` | — | no |
|
||||
| Cert | Cert, same thumbprint | `Match` | — | no |
|
||||
| Cert | Cert, other thumbprint, same device id | `RebindCertRotation` | caller | no |
|
||||
| Cert | Cert, other thumbprint and device id | `MismatchCert` (+ `serialMatch`) | — | **yes** |
|
||||
| Cert | Bootstrap, same serial | `RebindBootstrapHandoff` | caller | no |
|
||||
| Cert | Bootstrap, other serial | `MismatchBootstrapOwned` | — | **yes** |
|
||||
| Bootstrap | Bootstrap, same code + serial | `Match` | — | no |
|
||||
| Bootstrap | Bootstrap, otherwise | `MismatchBootstrap` | — | **yes** |
|
||||
| Bootstrap | Cert | `DowngradeToBootstrap` | — | **yes** |
|
||||
|
||||
The tolerated rows are the legitimate lifecycle: agent restart and reboot (same certificate),
|
||||
WhiteGlove Part 2 (same certificate), the install→runtime handoff where a bootstrap-registered session
|
||||
continues under the MDM certificate once it exists, a re-issued certificate of the same Intune device,
|
||||
and rows that predate the binding.
|
||||
|
||||
`MismatchCert` with `serialMatch=true` is the one shape that is *not* an attack: an Intune
|
||||
re-enrollment **without a wipe**. `session.id` survives in `%ProgramData%\AutopilotMonitor`, but the
|
||||
device gets a new Intune device id and therefore a new certificate identity. Under enforcement this
|
||||
would be refused — which is why enforcement needs the agent-side rotation below.
|
||||
|
||||
## Where it runs
|
||||
|
||||
`SessionOwnerBindingObserver` is the single side-effect carrier; the three call sites hand it the
|
||||
row they already hold:
|
||||
|
||||
* `RegisterSessionFunction.ProcessRegisterAsync` — the cascade-delete guard now returns the row
|
||||
(`EnsureWritableAndGetRowAsync`); the decision's owner goes into `StoreSessionAsync(registration,
|
||||
ownerToStamp)`, whose Replace preserves an existing owner when nothing new is stamped (including a
|
||||
Merge-stamped owner that landed after the initial read — the CAS re-read selects the owner columns).
|
||||
The bootstrap wrapper inherits this.
|
||||
* `IngestTelemetryFunction.Run` — evaluated against `guardSessionRow` (zero extra reads) before
|
||||
`PersistItemsAsync`, so the Signal-only path that serves pending ServerActions is covered. Legacy
|
||||
claims and rebinds are written with `UpdateSessionOwnerAsync` (Merge, fail-soft).
|
||||
* `ReportAgentErrorFunction` and its bootstrap wrapper — one point-read, observe only.
|
||||
|
||||
## Shadow telemetry (stage 1)
|
||||
|
||||
Same three carriers as the cert-tenant binding:
|
||||
|
||||
* **Denominator** — `requests | extend o = tostring(customDimensions.SessionOwnerBinding) | where
|
||||
isnotempty(o) | summarize count() by o` (request-row dimension via `RequestTelemetryMiddleware`;
|
||||
worker `LogInformation` never reaches App Insights).
|
||||
* **Numerator** — `traces | where message startswith "AgentSessionOwnerBinding"` — one Warning per
|
||||
non-Match/non-Fresh request with `outcome`, `wouldReject`, `callerKind`, `ownerKind`, `serialMatch`,
|
||||
`endpoint`, `ver`.
|
||||
* **Operator signal** — ops event `SessionOwnerMismatch` (Security / Warning, dual-registered in the
|
||||
admin alert-rule catalog) for would-reject outcomes, throttled to one per session+outcome per hour.
|
||||
`serialMatch=true` counts re-enrollments without wipe; `serialMatch=false` is a foreign device.
|
||||
|
||||
Nothing is refused in stage 1. `SessionOwnershipPolicy` deliberately has no `Rejects` member
|
||||
(pinned by `SessionOwnershipPolicyTests.Stage1_has_no_Rejects_rule`).
|
||||
|
||||
## Enforcement contract (stage 2, not yet built)
|
||||
|
||||
* Backend: `Rejects(outcome)` = the would-reject set; register/ingest/error answer 403 with
|
||||
`RegisterSessionResponse.ErrorCode = Constants.AgentErrorCodes.SessionOwnerMismatch`
|
||||
(`"session_owner_mismatch"`). The code and field already exist so the agent could ship first.
|
||||
* Agent (shipped dormant with this change): `BackendApiClient` reads `errorCode` off a JSON 401/403
|
||||
body into `BackendAuthException.ErrorCode`; `SessionRegistrationHelper` reacts to exactly that code
|
||||
by calling `rotateSession` once and re-registering immediately — **without** feeding
|
||||
`AuthFailureTracker` (five consecutive 403s would otherwise soft-shutdown a perfectly authorized
|
||||
agent). `BackendSessionRegistration.RotateSession` rotates `session.id` (keeping
|
||||
`whiteglove.complete`), re-targets `agentConfig.SessionId` and the `EmergencyReporter`, and drops
|
||||
`spool.jsonl` + `upload-cursor.json` because their lines carry the refused partition key. The
|
||||
`agent_started` event then reports `sessionRotated=true` + `previousSessionId`.
|
||||
* Registration is the first session-scoped call of every agent start and everything after Phase 6
|
||||
reads `agentConfig.SessionId` live, so a mismatch on ingest can only mean an agent older than the
|
||||
rotation logic or a genuinely foreign caller.
|
||||
|
||||
Switch on only after: the shadow distribution has run for weeks, every `wouldReject=true` is
|
||||
explained, and the fleet's `X-Agent-Version` is at or above the rotation-capable release. Then add
|
||||
`Rejects`, delete the stage-1 pin test, update the trust pages (isolation model) and bump their
|
||||
"Last reviewed" date.
|
||||
|
||||
# Residual risk (accepted)
|
||||
|
||||
* The serial is a caller-supplied header on both paths. A certificate holder who also knows a victim's
|
||||
serial can still claim a **legacy** row, or a **bootstrap-owned** row during its handoff window.
|
||||
* One bootstrap token serves many devices; two devices on the same short code can still write into
|
||||
each other's session until the certificate handoff closes it.
|
||||
* `SupersedeOrphanedPredecessorsAsync` is keyed on the announced serial and can still resolve foreign
|
||||
open sessions to `Incomplete`. Follow-up: restrict to rows without owner or with the same
|
||||
`OwnerSerial`, and require a minimum age.
|
||||
|
||||
None of these are closable without a signature scheme over the request body, which is deliberately
|
||||
not built (it would duplicate the TLS proof).
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/SessionOwnershipPolicy.cs` — matrix, `WouldRejectUnderEnforcement`, row (de)serialization
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/SessionOwnerBindingObserver.cs` — carriers, throttle, stamping
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/Enrollment/SessionOwner.cs` — column names and kinds
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs` — `StoreSessionAsync(…, ownerToStamp)`, `UpdateSessionOwnerAsync`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs` — rotation branch
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendSessionRegistration.cs` — `RotateSession`
|
||||
* Tests: `SessionOwnershipPolicyTests`, `SessionOwnerBindingObserverTests`, `StoreSessionReregistrationPreserveTests`, agent `SessionRegistrationHelperTests`, `BackendSessionRegistrationTests`, `SessionIdPersistenceTests`, `BackendApiClientAuthClassificationTests`
|
||||
* Grep marker: `SESSION-OWNER-BINDING`
|
||||
@@ -1,140 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Silent-Session Classification (Sweep, Max-Lifetime, Retro-Reconcile)
|
||||
description: How a session whose agent went silent gets its honest status — the two-stage maintenance sweep (2h Stalled marker, 5h timeout classification), the EnrollmentTimeoutClassifier rule order incl. the WhiteGlove Part-2 and self-deploying gates, which ingest paths share the classifier, and the admin retro-reconcile modes.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs
|
||||
tags:
|
||||
- backend
|
||||
- maintenance
|
||||
- session-status
|
||||
- classification
|
||||
- whiteglove
|
||||
- self-deploying
|
||||
timestamp: 2026-08-27T20:00:00+02:00
|
||||
---
|
||||
|
||||
# Silent-Session Classification
|
||||
|
||||
A session only ever becomes terminal through evidence. When the agent stops reporting
|
||||
(power-off, reboot without relaunch, Wi-Fi never re-associating, max-lifetime watchdog),
|
||||
the backend must decide the status from what it last saw — it never hard-fails a session
|
||||
for being silent. One pure function holds that decision,
|
||||
`EnrollmentTimeoutClassifier.ClassifyTimedOutSession`, and every silent-session path calls it
|
||||
so a session gets the same verdict regardless of which path noticed the silence first.
|
||||
|
||||
# Schema
|
||||
|
||||
## Inputs (`EspProvisioningRollup` + session flags)
|
||||
|
||||
`ExtractRollup(events)` distills the event stream into facts: Device/Account Setup
|
||||
subcategory counts (incl. the 30s fallback-confirmed messages), `HasExplicitFailure`,
|
||||
`HasEnrollmentComplete` vs. `HasTerminalComplete` (the Part-1 `whiteglove_complete`),
|
||||
`HasAgentEmergencyBreak`, `DesktopArrived`, `HelloResolved`, `HelloPolicyDisabled`,
|
||||
`SkipUserEsp`, RealmJoin detection/resolution, `HasAppInstallFailure` (any
|
||||
`app_install_failed`), and `HasAgentMaxLifetimeTimeout` (either max-lifetime shape:
|
||||
`enrollment_failed`/`agent_timeout` or `agent_shutting_down`/`max_lifetime` — the agent is
|
||||
provably gone). Session-row flags are passed alongside:
|
||||
`IsPreProvisioned` + `ResumedAt` (WhiteGlove Part 2) and `IsSelfDeployingProfile`
|
||||
(registry-confirmed `CloudAssignedOobeConfig 0x20|0x40`, set at registration).
|
||||
|
||||
## Rule order (first match wins)
|
||||
|
||||
| # | Condition | Verdict |
|
||||
| --- | --- | --- |
|
||||
| 1 | `HasExplicitFailure` | Failed |
|
||||
| 1b | `IsWhiteGloveAwaitingUser`: pre-provisioned + resumed + Device Setup done + **no** user evidence since resume + no explicit failure/enrollment_complete | AwaitingUser within grace; Succeeded (honest WG reason) past grace |
|
||||
| 1c | `IsSelfDeployingProvisioned`: self-deploying profile + Device Setup done + no explicit failure | **Succeeded** (reconcile) — no grace, no AwaitingUser: the profile has no user phase |
|
||||
| 2 | Account Setup all-succeeded, or `enrollment_complete`, or `whiteglove_complete` on a **non-resumed** session | Succeeded |
|
||||
| 3 | `HasAgentEmergencyBreak` | Incomplete immediately (agent self-destructed, nothing can arrive) |
|
||||
| 4 | `DesktopArrived` + (Hello terminal, or Hello disabled + SkipUserEsp) | Succeeded (user provably finished) |
|
||||
| 5 | Device Setup done + within grace + **no** max-lifetime watchdog observed | AwaitingUser |
|
||||
| 5a | Device Setup done + past grace (or watchdog observed) + `DesktopArrived` + no `app_install_failed` | **Succeeded** — completed (assumed), `r5_assumed` |
|
||||
| 5b | Device Setup done, otherwise past grace (or watchdog observed) | Incomplete |
|
||||
| 6 | otherwise | Incomplete |
|
||||
|
||||
Grace = `ResolveGraceHours(SessionGraceHours, AbsoluteMaxSessionHours)` (~51h default),
|
||||
measured from `ResumedAt ?? StartedAt`. Every Succeeded-reconcile reason carries the
|
||||
silence-transparency clause (`AppendReconcileTiming`: last agent contact, silence, verdict time).
|
||||
|
||||
## Why rule 1c exists (kiosk tenant audit 2026-08-23)
|
||||
|
||||
On a self-deploying profile Device ESP all-succeeded IS the enrollment's end; the agent's
|
||||
own SelfDeploying terminal only adds a 5-min confirmation window
|
||||
(`DecisionEngine.SelfDeploying.s_deviceOnlyEspDetectionWindow`). Tenant `aebdce78` (school,
|
||||
Wi-Fi ThinkBooks, a language-change app forcing a reboot right after Device ESP) lost that
|
||||
window on 836 of ~970 sessions in one week: the agent died 0–4 min after
|
||||
`DeviceSetup fallback_confirmed`, the reboot relaunch never registered (30s registration
|
||||
budget on a Wi-Fi link still associating), and rule 5 labeled finished kiosks
|
||||
"awaiting user / Account Setup 0/5" → Incomplete 51h later. Nobody signs in on these devices
|
||||
(prepared during the holidays, boxed, in service) — AwaitingUser was factually wrong and
|
||||
Incomplete turned a working rollout into a red wall. Other self-deploying tenants completed
|
||||
normally in the same window, so this is a classification gap, not a profile-wide agent bug.
|
||||
Agent-side companion fix: `SessionRegistrationHelper` now waits up to 15s for a network link
|
||||
and retries 6× (~62s backoff) instead of 5× (~30s). Measured 2026-08-27 (builds
|
||||
2.0.1428/1429, both carrying the fix): the reconcile share did not move, and the backend saw
|
||||
no registration attempt from any relaunch — no success, no 4xx, zero `register:superseded`
|
||||
in the tenant — so the relaunch never reaches the network at all (device powered off after
|
||||
ESP, or the relaunch never runs). The sweep reconcile is the permanent completion path for
|
||||
this profile, not a stopgap.
|
||||
|
||||
## Callers (all pass both gate inputs)
|
||||
|
||||
| Path | File | Notes |
|
||||
| --- | --- | --- |
|
||||
| Sweep stage 1 (2h agent-silent marker) | `MaintenanceService.MarkStalledSessionsAsTimedOutAsync` | Default: InProgress → Stalled. Gates run BEFORE the Stalled write: `TryMarkWhiteGloveAwaitingUserAsync` (AwaitingUser) and `TryReconcileSelfDeployingAsync` (Succeeded). Only flagged sessions pay the event read. |
|
||||
| Sweep stage 2 (SessionTimeoutHours, 5h default) | same | InProgress/Stalled/AwaitingUser past the window and past the 2h silence cutoff → full classification; AwaitingUser within grace is skipped without an event read. |
|
||||
| Agent max-lifetime (`enrollment_failed`/max_lifetime, `agent_shutting_down`/max_lifetime) | `EventIngestProcessor.ApplyMaxLifetimeVerdictAsync` | Watchdog = "agent stopped waiting", not a verdict; classified from evidence. Pending sessions exempt. |
|
||||
| Late-telemetry reconcile | `EventIngestProcessor.TryLateTelemetryReconcileAsync` | Heal-only: applies a Succeeded verdict to Incomplete/AwaitingUser rows when straggler events arrive. |
|
||||
| Agent stall probe (`session_stalled`) | `EventIngestProcessor.TryMapStallToWhiteGloveAwaitingUserAsync` | Only the WG gate (1b); a stall probe on a self-deploying device with Device Setup done means the agent is alive and something else blocks — stays Stalled. |
|
||||
| Admin retro-reconcile | `LegacyReclassificationService` | Modes `legacy_timeouts`, `pending_orphans`, `self_deploying_silent` (Incomplete/AwaitingUser/Stalled rows with the profile flag, heal-only to Succeeded, admin-marked rows skipped). `POST /api/maintenance/reclassify-legacy?mode=…&dryRun=false&tenantId=…&maxSessions=…`, dry-run by default. |
|
||||
|
||||
## Why rule 5a exists (calibration read 2026-08-27)
|
||||
|
||||
The verdict-calibration matrix showed r5-Incomplete devices re-enrolling at 2.6 % within
|
||||
7 days — the succeeded background (`agent:complete_soft` sits at 2.8 %), not the ~29 %
|
||||
failure level — consistently across tenants, and half the sampled sessions were healthy
|
||||
desktops with a clean app record whose agent died before the Hello terminal (some with the
|
||||
agent's own "AccountSetup treated as complete" synthesis as the literally last event). Past
|
||||
the full grace, "the user phase is still running" is no longer a possible explanation, so
|
||||
desktop + zero `app_install_failed` + no explicit failure now reconciles to Succeeded
|
||||
("completed (assumed)") instead of a red Incomplete. Sessions without a desktop (user never
|
||||
began sign-in: battery-drained, powered off, max-lifetime) and sessions with any failed app
|
||||
stay Incomplete — "completed" must not overclaim. Rule 4 remains the IMMEDIATE reconcile
|
||||
(desktop + Hello proof); 5a only fires where waiting is over. Desktop arrival alone is still
|
||||
rejected as a live completion signal.
|
||||
|
||||
## Closed gap: max-lifetime AwaitingUser (2026-08-27)
|
||||
|
||||
After max-lifetime the agent writes `enrollment-complete.marker` and self-destructs, so an
|
||||
AwaitingUser verdict from that path could never heal — the calibration correction stream
|
||||
confirmed it: all five observed `maxlife:r5_awaiting` episodes expired to Incomplete
|
||||
unhealed, while live-agent `sweep:r5_awaiting` parks upgraded 7 of 26 to Succeeded. Rule 5
|
||||
therefore skips the grace once `HasAgentMaxLifetimeTimeout` is in the stream and decides the
|
||||
terminal fork (5a/5b) immediately; the max-lifetime ingest appends its trigger event to the
|
||||
read stream so the fact is never missed, and late straggler telemetry still heals a terminal
|
||||
verdict via `TryLateTelemetryReconcileAsync`. WhiteGlove rule 1b deliberately keeps its
|
||||
park — sealed/boxed devices do heal (the observed `maxlife:r1b_awaiting` episode resolved to
|
||||
Succeeded).
|
||||
|
||||
# Examples
|
||||
|
||||
* Session `195593e2` (tenant aebdce78): `DeviceSetup 4/4` + `fallback_confirmed`, `AccountSetup 0/5`
|
||||
(IME false positive), Hello disabled, SkipUser=True, silent after 13:01 → rule 1c →
|
||||
Succeeded "self-deploying profile — Device Setup fully provisioned … agent went silent
|
||||
before confirming completion. Agent last reported 2026-08-21 13:01 UTC …".
|
||||
* Same event shape on a user-driven profile → rule 5 → AwaitingUser (within grace) /
|
||||
Incomplete (past grace). Guarded by
|
||||
`Classify_same_stream_without_self_deploying_flag_keeps_user_phase_verdicts`.
|
||||
* fairstone WG Part 2 reseal-reboot, technician powers off at logon screen → rule 1b →
|
||||
AwaitingUser; user unboxes days later → ingest completion heals to Succeeded.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs` — rollup + rule order, `IsWhiteGloveAwaitingUser`, `IsSelfDeployingProvisioned`, reason helpers.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs` — `MarkStalledSessionsAsTimedOutAsync`, `TryMarkWhiteGloveAwaitingUserAsync`, `TryReconcileSelfDeployingAsync`.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs` — max-lifetime verdict, late-telemetry reconcile, stall mapping.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/LegacyReclassificationService.cs`, `Functions/Admin/ReclassifyLegacySessionsFunction.cs` — retro modes.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs` — `GetAgentSilentSessionsAsync`, `GetStalledSessionsAsync`, `GetSelfDeployingSilentSessionsAsync`.
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs` — link wait + 6-attempt registration budget.
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.SelfDeploying.cs` — the agent-side 5-min SelfDeploying terminal window.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs` — rule coverage incl. the 2026-08-23 self-deploying block.
|
||||
@@ -1,67 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Table Schema Sentinel (startup table initialization)
|
||||
description: Why a cold start no longer issues one CreateTableIfNotExists per registered table - a hash over TableNames.All stored in AdminConfiguration gates the full pass, the SessionsIndex backfill only runs after a full pass and behind a conditional-insert claim, and daily maintenance repairs out-of-band deletions.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs
|
||||
tags:
|
||||
- backend
|
||||
- table-storage
|
||||
- startup
|
||||
- maintenance
|
||||
timestamp: 2026-08-23
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
## Problem
|
||||
|
||||
`TableInitializerService` (hosted service) called `InitializeTablesAsync` on every process start, which issued `CreateTableIfNotExistsAsync` for every entry of `Constants.TableNames.All` (66 at the time of writing, DoP 8). Each call is a Create POST whose 409 is swallowed - on Flex Consumption with frequent cold starts that was ~9 serial round-trip waves before the first request. It was followed by a SessionsIndex emptiness query whose *error* path returned "empty", so a transient storage error triggered a full Sessions scan with serial upserts - on every scaled-out instance at once.
|
||||
|
||||
## Sentinel
|
||||
|
||||
* Row `AdminConfiguration` / PK `SchemaSentinel` / RK `tables`, property `TableSchemaHash` (+ `TableCount`, `UpdatedUtc`).
|
||||
* Value = `ComputeTableSchemaHash(TableNames.All)`: SHA-256 over the ordinal-sorted, newline-joined table names. Derived, never hand-maintained: adding a table to `TableNames.All` changes the hash by construction (`TableLifecycleBucketTests` already forces every constant into `All`).
|
||||
* `InitializeTablesAsync` (returns `bool fullPassRan`):
|
||||
1. Point-read the sentinel. Match -> mark initialized, **zero** table calls, return `false`.
|
||||
2. Missing / stale / unreadable (fresh storage has no AdminConfiguration table yet) -> `EnsureAllTablesAsync` (the old full pass), then upsert the sentinel **only if no table failed**, return `true`. A failed pass leaves `_tablesInitialized = false`, so the next call retries.
|
||||
* Scale-out: two instances that both see a stale sentinel both run the idempotent pass - harmless.
|
||||
|
||||
## SessionsIndex backfill
|
||||
|
||||
* Only evaluated when the full pass ran (`fullPassRan == true`): an empty index can only exist on fresh storage, which is exactly the case without a sentinel.
|
||||
* `IsSessionIndexEmptyAsync` now returns `false` on error (skip), not `true`.
|
||||
* `TryClaimSessionIndexBackfillAsync`: conditional `AddEntityAsync` on `SchemaSentinel` / `sessionIndexBackfill` - only the winner scans; 409 or any error -> skip. The manual maintenance backfill (`RunManualAsync`) remains the safety net.
|
||||
|
||||
## Known gap and repair
|
||||
|
||||
A table deleted out-of-band (cleanup skill, portal, tests) is not noticed at startup anymore. `IStorageInitializer.EnsureAllAsync` runs as the first step of the daily timer maintenance (`MaintenanceService.RunAllAsync`), recreating it within 24h. Storage helpers are fail-soft in the meantime.
|
||||
|
||||
## Measuring it
|
||||
|
||||
`StartupTelemetryService` (hosted service) emits two App Insights metrics on `IHostApplicationLifetime.ApplicationStarted` - i.e. after every hosted service finished `StartAsync`:
|
||||
|
||||
* `BackendStartupMs` - process start (`Process.StartTime`) to host ready; includes runtime, DI and all hosted services.
|
||||
* `BackendTableInitMs` - the `InitializeTablesAsync` slice alone (`TableStorageService.LastInitialization`).
|
||||
|
||||
Dimensions: `version`, `commit`, `tableInitFullPass`, `instance`. Metrics bypass adaptive sampling and the worker log level, so they arrive even though Information traces do not. The MCP server logs the same idea as one `{"type":"boot","readyMs":…}` stderr line per start (ContainerAppConsoleLogs_CL, see `.claude/commands/mcp-telemetry.md`).
|
||||
|
||||
```kusto
|
||||
customMetrics
|
||||
| where name in ("BackendStartupMs", "BackendTableInitMs")
|
||||
| summarize p50 = percentile(value, 50), p95 = percentile(value, 95), n = count()
|
||||
by name, version = tostring(customDimensions.version), fullPass = tostring(customDimensions.tableInitFullPass)
|
||||
```
|
||||
|
||||
# Examples
|
||||
|
||||
* Deploy that adds a table: first cold start after the deploy runs the full pass once (log: "Table schema sentinel missing or stale"), every later start logs "Table schema sentinel matches (...)".
|
||||
* Fresh storage account: sentinel read throws TableNotFound -> full pass -> index empty -> one instance claims and backfills.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs` - sentinel, `InitializeTablesAsync`, `EnsureAllTablesAsync`, `TryClaimSessionIndexBackfillAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableInitializerService.cs` - startup gating
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs` - daily `EnsureAllTablesAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/StartupTelemetryService.cs` - `BackendStartupMs` / `BackendTableInitMs`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/index.ts` - `{"type":"boot"}` line
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/TableSchemaSentinelTests.cs`, `StartupTelemetryServiceTests.cs`
|
||||
@@ -1,89 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Telemetry Ingest Shaping (App Insights / Log Analytics)
|
||||
description: What the backend deliberately keeps out of Application Insights and why - one canonical request record (worker copy, per-item sampling bypass; host HTTP duplicate dropped by a workspace transformation), expected storage outcomes (404/412/409) and successful SignalR REST calls filtered in the worker, and the query rules that follow.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Telemetry/StorageDependencyFilterProcessor.cs
|
||||
tags:
|
||||
- backend
|
||||
- telemetry
|
||||
- application-insights
|
||||
- cost
|
||||
timestamp: 2026-08-23
|
||||
---
|
||||
|
||||
# Telemetry Ingest Shaping
|
||||
|
||||
The backend is storage-I/O heavy and runs on the Functions isolated worker, which emits telemetry from TWO processes (host and worker). Left alone, Log Analytics ingestion is dominated by rows nobody reads. This document records what is removed, where, and what that means for queries.
|
||||
|
||||
# Schema
|
||||
|
||||
## Request telemetry — one canonical record
|
||||
|
||||
| Source | Emitted by | Content | Fate |
|
||||
|---|---|---|---|
|
||||
| Host HTTP request (`Properties.Source` empty, `Url` set) | Functions host process | Name, duration, status — no tenant/user/correlation | **Dropped at ingestion** by the workspace transformation on `AppRequests` (`infra/appinsights-workspace-transforms.bicep`) |
|
||||
| Host non-HTTP invocation (`Url` empty) | Functions host process | Timer / queue executions | Kept (the worker middleware never sees them) |
|
||||
| Worker request (`Properties.Source == 'WorkerMiddleware'`) | `RequestTelemetryMiddleware` | TenantId, UserId, UserRole, CorrelationId, ClientSource, McpToolName, HttpPath | **Kept, unsampled** — the canonical record |
|
||||
|
||||
Why two mechanisms:
|
||||
|
||||
* The host request item cannot be suppressed from worker code or from `host.json` (`Host.Results` controls log traces, not the request item; verified 2026-06-09). The only lever is a Log Analytics workspace transformation: `source | where tostring(Properties.Source) == 'WorkerMiddleware' or isempty(Url)`.
|
||||
* `host.json` `samplingSettings.excludedTypes: "Request;Event"` only applies to the host process. The worker's `AddApplicationInsightsTelemetryWorkerService` runs its own adaptive sampling; before 2026-08-23 worker request rows carried `ItemCount` 2–5 on weekdays (≈30 % of individual requests missing, `count()` undercounting by the same share — `sum(ItemCount)` matched the host copy to 1.00 per day). Re-configuring the worker sampling chain did not take effect in the isolated worker (see the L4 note in `Program.cs`). The per-item opt-out does: the middleware sets `SamplingPercentage = 100` on its own `RequestTelemetry`, and the SDK sampling processor passes through every item whose sampling percentage is already set.
|
||||
|
||||
## Dependency telemetry — keep only signal
|
||||
|
||||
`StorageDependencyFilterProcessor` (worker pipeline) drops:
|
||||
|
||||
* successful Azure Storage calls (Table/Queue/Blob) in both shapes — the HTTP shape (`Azure table`, target `*.table.core.windows.net`) and the Azure SDK ActivitySource shape (`InProc | Microsoft.Tables`, `InProc | Microsoft.Storage`);
|
||||
* storage calls with an **expected outcome**: `404` (point-read miss — EventTypeIndex, BlockedDevices lookups), `412` (ETag precondition), `409` (idempotent insert conflict). HTTP-shape rows carry the code in `ResultCode`; InProc rows only in `Properties["Error"]` (`Status: 404 (`). Before this filter these were 100 % of the remaining "failed" storage rows (~540 MB/week, two rows per call);
|
||||
* successful Azure SignalR Service REST calls the worker makes itself (`*.service.signalr.net`).
|
||||
|
||||
The bulk of SignalR REST traffic (group add/remove per connection, ~470 MB/week) is emitted by the **host** process for the SignalR output binding (`SDKVersion rdddsc:*`) and never passes the worker pipeline — live-verified after the 2026-08-23 deploy (worker rows: 0, host rows: ~560/h). Those are removed by the second data flow of the same workspace transformation: `AppDependencies | where not(Success == true and Target endswith '.service.signalr.net')`.
|
||||
|
||||
Everything else passes: storage 429/5xx/auth/timeouts, failed SignalR calls, all HTTP/Graph/NVD/MSRC dependencies, the worker's own `InProc Invoke` span, and every non-dependency item.
|
||||
|
||||
## Measured baseline (7 days before the change, `_BilledSize`, `autopilotmonitor-api-eu`)
|
||||
|
||||
| Table | Rows | MB | After |
|
||||
|---|---|---|---|
|
||||
| AppDependencies | 976k | 1248 | ≈ 240 (SignalR + expected-outcome rows gone) |
|
||||
| AppRequests host copy | 274k | 323 | ≈ 0 (only timer/queue rows) |
|
||||
| AppRequests worker copy | 189k | 208 | ≈ 300 (now 100 % of requests) |
|
||||
| AppTraces | 142k | 137 | unchanged |
|
||||
| AppEvents | 6k | 4 | unchanged — not worth touching |
|
||||
|
||||
Net ≈ −1.3 GB/week with MORE usable data (every request now has tenant context).
|
||||
|
||||
# Examples
|
||||
|
||||
Counting requests (exact since 2026-08-23; for older data replace `count()` with `sum(itemCount)`):
|
||||
|
||||
```kql
|
||||
requests
|
||||
| where customDimensions.Source == 'WorkerMiddleware'
|
||||
| summarize total=count(), failed=countif(success == false) by name
|
||||
```
|
||||
|
||||
Real storage trouble only:
|
||||
|
||||
```kql
|
||||
dependencies
|
||||
| where type startswith 'InProc | Microsoft.Tables' or target endswith '.table.core.windows.net'
|
||||
| summarize count() by resultCode, tostring(customDimensions.Error)
|
||||
```
|
||||
|
||||
Post-deploy verification of the per-item sampling bypass (must be `ItemCount == 1` only):
|
||||
|
||||
```kql
|
||||
AppRequests
|
||||
| where TimeGenerated > ago(1h) and tostring(Properties.Source) == 'WorkerMiddleware'
|
||||
| summarize rows=count() by ItemCount
|
||||
```
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Middleware/RequestTelemetryMiddleware.cs` — `SamplingPercentage = 100` before `TrackRequest`.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Telemetry/StorageDependencyFilterProcessor.cs` + `StorageDependencyFilterProcessorTests.cs` — the filter contract.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Program.cs` — the failed L4 worker-sampling attempt and why it must not be retried.
|
||||
* `infra/appinsights-workspace-transforms.bicep` — the AppRequests transformation DCR, apply/verify/rollback commands.
|
||||
* `.claude/commands/backend-logs.md`, `session-debug.md` — query conventions.
|
||||
@@ -1,154 +0,0 @@
|
||||
---
|
||||
type: concept
|
||||
title: Tenant Activation & the Once-Only Welcome Mail
|
||||
description: How a signup becomes an activated tenant (manual approve or auto-approve queue), and how the welcome mail is sent exactly once even though activation and the user-provided address race each other.
|
||||
resource: backend
|
||||
tags: [activation, auto-approve, welcome-email, preview-whitelist, race]
|
||||
timestamp: 2026-08-23
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
## Activation
|
||||
|
||||
A tenant is **activated** when the row `PartitionKey={tenantId}, RowKey="approved"` exists
|
||||
in the `PreviewWhitelist` table (legacy name kept on purpose). Everything else in the
|
||||
portal gates on that row via `PreviewWhitelistService.IsApprovedAsync` (5 min positive /
|
||||
30 s negative cache).
|
||||
|
||||
Two activation routes, one shared implementation
|
||||
(`TenantApprovalService.ApproveWithSideEffectsAsync`):
|
||||
|
||||
* **Manual**: Global Admin approves in the admin panel (`POST /api/preview/whitelist/{tenantId}`).
|
||||
* **Auto**: signup unconditionally enqueues a `tenant-auto-approve` envelope; the queue
|
||||
worker (`TenantAutoApproveHandler`) re-checks the `AutoApproveNewTenants` flag
|
||||
(uncached, fail-closed), the suspension gate (fresh config read), then approves.
|
||||
Typical latency: ~1 minute after signup.
|
||||
|
||||
The whitelist add is a **conditional INSERT** (`AddEntityAsync`, 409 = lost) — storage
|
||||
arbitrates concurrent activations; exactly one caller runs the side effects
|
||||
(TenantAdmin auto-promote of the signup UPN, welcome mail, `TenantAutoApproved` ops event).
|
||||
|
||||
## The welcome-mail race
|
||||
|
||||
The welcome mail goes to the **notification email** (`RowKey="notification-email"` in the
|
||||
same table) — the only reliable address: it is typed by the user on the activation-pending
|
||||
page (`PUT /api/preview/notification-email`). The signup admin UPN may have no mailbox and
|
||||
`ContactEmail` is merely seeded from the notification email.
|
||||
|
||||
The PUT is `AuthenticatedUserWithRole`, not a member tier, because the signup user is still
|
||||
roleless until approval auto-promotes them to Admin. The function re-gates on the resolved
|
||||
role (`PreviewWhitelistFunction.MayWriteNotificationEmailAsync`): a member role or platform
|
||||
scope always may write; a roleless caller may write only while the tenant has **no enabled
|
||||
member** at all. Once any member exists the window is closed for good — the address feeds the
|
||||
welcome and farewell mails and is shown to operators as the tenant contact, so an ordinary
|
||||
employee with a JWT but no product role must not be able to redirect or clear it (403).
|
||||
|
||||
That makes activation and address entry concurrent writers. With auto-approve, activation
|
||||
usually wins (observed in production: approve at T+64 s, address saved at T+69 s → the
|
||||
approval path found no address and the mail was silently skipped).
|
||||
|
||||
**Arbitration — write-then-read on both sides plus a send-once marker:**
|
||||
|
||||
* The approval path writes the `approved` row, then reads the address.
|
||||
* The save path (`SavePreviewNotificationEmail`) writes the address row, then checks
|
||||
approval **fresh** (`IsApprovedFreshAsync` — the 30 s negative cache would hide the
|
||||
activation for exactly this window).
|
||||
* Since both do write-then-read, at least one sees both halves and calls
|
||||
`TenantApprovalService.TrySendWelcomeEmailAsync`.
|
||||
* There, a conditional insert of `RowKey="welcome-email-sent"` decides who actually
|
||||
sends: at most one wins. The marker is consumed strictly **after** the address check —
|
||||
consuming it without an address would permanently suppress the mail.
|
||||
|
||||
Marker lifecycle: revoke (`RevokeAsync`) deletes it (re-approve = fresh activation =
|
||||
fresh mail); the offboarding cascade wipes the whole `PreviewWhitelist` partition anyway.
|
||||
The Global-Admin resend endpoint (`POST /api/preview/send-welcome-email/{tenantId}`)
|
||||
always sends (explicit intent) and consumes the marker best-effort so the automatic
|
||||
paths never duplicate afterwards.
|
||||
|
||||
## Finding the tenant behind an address
|
||||
|
||||
The provider's delivery log names only the recipient — never the tenant it belonged to. The
|
||||
reverse lookup is therefore part of the tenant console: its search matches `tenantId`,
|
||||
`domainName`, `ContactEmail` **and** the notification address. No other address is searchable
|
||||
(an admin UPN never receives platform mail and would only widen the PII surface of the list).
|
||||
|
||||
The addresses live in a different table than the tenant configs the list is built from, so the
|
||||
console loads them as a map from `GET /api/preview/notification-emails` (GA + read-only Global
|
||||
Reader, `excludeDelegated`) — one cross-partition scan of `RowKey eq 'notification-email'`
|
||||
instead of a point read per tenant. Keys are lowercased on both sides of the join: the address
|
||||
rows are written from the JWT `tid` (lowercase) while a config PartitionKey's casing is not
|
||||
guaranteed. Stored addresses keep the casing the user typed, so matching is case-insensitive.
|
||||
|
||||
The map is **best-effort** in the portal: it powers only the search-by-address, so a failed read
|
||||
degrades that one capability instead of taking the tenant list down. The addresses are not
|
||||
rendered in the list — they are a search key, and stay visible in the tenant editor.
|
||||
|
||||
## Mail transport
|
||||
|
||||
Both transactional mails (welcome, post-offboarding farewell) go through the
|
||||
provider-neutral `EmailService` (`IEmailService` for the welcome path,
|
||||
`IOffboardFarewellEmailSender` for the farewell path). The provider is an implementation
|
||||
detail confined to one private transport method plus the `Email:*` configuration
|
||||
section — a provider swap must not rename classes, DI registrations or callers.
|
||||
|
||||
* Current provider: **Mailchimp Transactional (Mandrill)**, `POST {Email:Endpoint}`
|
||||
(`messages/send`), API key in the JSON body, typed `HttpClient` with the shared
|
||||
`ResiliencePolicies.Notification` retry policy (15 s timeout).
|
||||
* Settings (App Settings use `Email__Key`): `Email:ApiKey` (required — empty means every
|
||||
send is a logged no-op, never an error), `Email:Endpoint`, `Email:FromAddress`
|
||||
(default `noreply@autopilotmonitor.com`), `Email:FromName` (default `Autopilot Monitor`).
|
||||
* Result handling: per-recipient status `sent`/`queued`/`scheduled` = success;
|
||||
`rejected`/`invalid` (with `reject_reason`), non-2xx or malformed bodies = warning.
|
||||
Never throws — both callers rely on that fail-soft contract.
|
||||
* `track_opens`/`track_clicks` are explicitly `false` and `auto_text` is `true`: the provider
|
||||
receives only the recipient address and the tenant domain, which is exactly the claim on
|
||||
the privacy page and the trust data-flows page. Changing tracking = changing those pages.
|
||||
|
||||
## Template overrides (operator-editable)
|
||||
|
||||
Global Admins can replace either HTML body without a deployment (portal: Settings →
|
||||
Email Templates). `EmailTemplateService` resolves the body for every send: the stored
|
||||
override wins, otherwise the built-in `EmailTemplates.*`. Overrides live in the
|
||||
`PreviewConfig` table (partition `EmailTemplates`, row key `welcome` | `farewell`,
|
||||
properties `Html`, `UpdatedBy`, `UpdatedUtc`), cached 5 minutes, invalidated on save/reset.
|
||||
|
||||
* The only placeholder is `{{domainName}}`; an empty domain renders as "your organization",
|
||||
matching the built-ins. Subjects are not editable (pinned constants).
|
||||
* The domain is HTML-encoded at the point of substitution (`EmailTemplateService.Render` and the
|
||||
built-in template builders) — the mail goes out under the product's signed sender to a
|
||||
tenant-chosen recipient, so no writer of `DomainName` may ever inject markup into it.
|
||||
`DomainName` itself is tenant identity: seeded once at first login from the UPN domain
|
||||
(`AuthFunction`, only when it passes `TenantConfigValidation.IsValidDomainName` — a strict
|
||||
dotted host name), preserved from the stored row on the full-model config PUT for every
|
||||
caller, and "never writable" in the field-patch flow.
|
||||
* Limit 30 000 characters (Table string property cap is 32K UTF-16 chars).
|
||||
* The send path is fail-soft: a storage error falls back to the built-in body. The admin
|
||||
read path is not — a broken store surfaces as an error instead of silently showing the
|
||||
built-in.
|
||||
* Endpoints (all GlobalAdminOnly): `GET|PUT|DELETE /api/global/email-templates/{kind}` and
|
||||
`POST /api/global/email-templates/{kind}/test`. The test send goes to the caller's tenant
|
||||
contact address (activation notification email → `ContactEmail` → caller UPN) with the
|
||||
caller's tenant domain, optionally with an unsaved draft body; provider tag `test`.
|
||||
|
||||
# Examples
|
||||
|
||||
* Auto-approve wins the race: approve at T+64 s finds no address → logs "deferred to the
|
||||
notification-email save path"; user saves address at T+69 s → save path sees approved
|
||||
(fresh read), wins the marker, sends. Response carries `welcomeEmailSent: true`.
|
||||
* Address saved before manual approve (classic shape): approval path finds the address,
|
||||
wins the marker, sends; a later re-save of the same address does not re-send.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TenantApprovalService.cs` — shared activation + `TrySendWelcomeEmailAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Activation/TenantAutoApproveHandler.cs` — queue-side gates
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Rules/PreviewWhitelistFunction.cs` — save-path trigger, GA resend
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs` — conditional inserts (approved / welcome-email-sent)
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/TenantApprovalServiceTests.cs` — ordering + dedup pins
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EmailService.cs` — provider-neutral transport (Mandrill wire format, result handling, tracking off)
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/EmailServiceTests.cs` — wire-format, fail-soft and gate-ordering pins
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EmailTemplateService.cs` — override resolution, placeholder rendering, cache
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Admin/EmailTemplatesFunction.cs` — GA endpoints (read / override / reset / test send)
|
||||
* `src/Web/autopilot-monitor-web/app/admin/settings/sections/SectionEmailTemplates.tsx` — portal editor, preview frame, test send
|
||||
* `src/Web/autopilot-monitor-web/app/admin/components/tenantSearch.ts` — the console's match rules (searchable fields, quoted exact mode, lowercased join)
|
||||
@@ -1,144 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Enrollment Time Attribution (F1)
|
||||
description: How a terminal session's wall clock is partitioned into named segments with an explicit unattributed remainder, how ESP-blocking apps and reboots are attributed, and where breakdowns and fleet aggregates are computed, stored and served.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs
|
||||
tags:
|
||||
- backend
|
||||
- metrics
|
||||
- time-attribution
|
||||
- insights
|
||||
timestamp: 2026-07-26T20:00:00+02:00
|
||||
---
|
||||
|
||||
# Enrollment Time Attribution (F1)
|
||||
|
||||
Answers "where did the 47 minutes go?" per session and per fleet. Every number is
|
||||
traceable to observed events; a missing signal produces "unknown" or an explicit
|
||||
unattributed remainder — never a guess, and the partition is never normalized to 100 %.
|
||||
|
||||
# Schema
|
||||
|
||||
## Invariant
|
||||
|
||||
For every computed breakdown:
|
||||
|
||||
```
|
||||
sum(span seconds) + UnattributedSeconds == WallClockSeconds == session DurationSeconds
|
||||
```
|
||||
|
||||
`DurationSeconds` is the authoritative wall clock — for WhiteGlove it is part 1 + part 2
|
||||
WITHOUT the pause, and re-terminal stamps move `CompletedAt`, so `CompletedAt − StartedAt`
|
||||
is never used (it diverges in ~25 % of terminal sessions). Span seconds are floored;
|
||||
sub-second dust lands in the unattributed remainder.
|
||||
|
||||
## Observation windows
|
||||
|
||||
Windows are end-anchored on authoritative timestamps and constructed so their lengths sum
|
||||
exactly to `DurationSeconds`:
|
||||
|
||||
* Non-WhiteGlove: one window `[CompletedAt − DurationSeconds, CompletedAt]`.
|
||||
* WhiteGlove (ONE session row, two windows): part 2 = `[ResumedAt, CompletedAt]`; part 1 =
|
||||
`[part1End − (DurationSeconds − part2), part1End]` where `part1End` is anchored on the
|
||||
`whiteglove_part1_complete` event. The pause between the windows is never attributed.
|
||||
Fallback anchors set the `WhiteGloveAnchorsIncomplete` quality flag.
|
||||
|
||||
## Segments
|
||||
|
||||
Spans are sliced at phase-declaration events (only declaration events carry
|
||||
`Phase != Unknown`) plus `desktop_arrived`, in canonical Sequence order; anchors whose
|
||||
timestamp runs backward are dropped (`ClockSkewDropped`).
|
||||
|
||||
| EnrollmentPhase | Segment key |
|
||||
|---|---|
|
||||
| Start / DevicePreparation / DeviceSetup | `device_prep` |
|
||||
| AppsDevice | `esp_apps` |
|
||||
| AccountSetup / FinalizingSetup | `identity_hello` |
|
||||
| AppsUser | `user_esp` |
|
||||
| Complete, `desktop_arrived` event | `desktop_handoff` |
|
||||
| Failed | attribution ends — tail stays unattributed |
|
||||
|
||||
## Quality flags (AttributionVersion 2)
|
||||
|
||||
Duration-critical flags (`ClockSkewDropped`, `PartialObservation`,
|
||||
`WhiteGloveAnchorsIncomplete`, `PriorEnrollmentResidue`) exclude a breakdown from fleet
|
||||
segment statistics with a disclosed count; the blocking-set flags (`BlockingSetUnknown`,
|
||||
`BlockingSetTruncated`) only limit per-app evidence and never gate fleet stats.
|
||||
|
||||
`PriorEnrollmentResidue` (v2) fires on `historic_ime_replay_detected`: the IME log on disk
|
||||
predates the enrollment, so the device was re-enrolled without a wipe — pre-installed apps
|
||||
complete as instant detections and pull the phase anchors ahead of the real ESP page
|
||||
(session f475e697: AccountSetup anchor 6.5 min early, 457 s misattributed to
|
||||
`identity_hello`). `registry_app_baseline` successes are deliberately NOT a trigger — they
|
||||
are by-design normal for Windows Device Preparation (DPP Batch-1 apps install before the
|
||||
agent exists) and would starve that class's aggregates.
|
||||
|
||||
## ESP-blocking apps (positive evidence only)
|
||||
|
||||
Per-app intervals use EVENT timestamps (first started/download event → LAST terminal
|
||||
event, covering IME retries) — never the agent payload timing, which freezes at the first
|
||||
terminal transition. Blocking membership joins the app id against the session's latest
|
||||
list-carrying `esp_config_detected` emission: listed ⇒ blocking; absent ⇒ unknown, never
|
||||
false (`BlockingSetUnknown` / `BlockingSetTruncated` flags). Critical-path occupancy =
|
||||
overlap-merged union of blocking intervals clipped to the `esp_apps` spans;
|
||||
`EspAppsOccupancySeconds` is null (unknown) when no lists were observed.
|
||||
|
||||
The fleet what-if bound per app X is `max(0, cpEnd − cpEndWithoutX)` (latest interval end
|
||||
with vs. without X; the idle gap before X counts — the ESP also waits for X to start).
|
||||
It is an upper bound by construction; every consumer says "up to", never "you will save".
|
||||
|
||||
## Reboots
|
||||
|
||||
`system_reboot_detected` is detection-time-stamped by the NEXT agent run; the real boot
|
||||
moment is its `lastBootUtc` payload. Reboot spans are therefore the event-stream gap
|
||||
bracketing `lastBootUtc`, deduped per boot, clipped to the observation windows, and carried
|
||||
as a cross-cutting annotation (they overlap the segment they started in — never a slice of
|
||||
the partition).
|
||||
|
||||
## Persistence & compute
|
||||
|
||||
* `SessionTimeBreakdowns` (PK = TenantId, RK = SessionId): spans, reboot spans, blocking
|
||||
intervals (top 20 + uncapped count), quality flags, `AttributionVersion`. Written once at
|
||||
the session-terminal single-writer seam (`TableSessionRepository.UpdateSessionStatusAsync`,
|
||||
Succeeded/Failed only — Incomplete deliberately has no duration). Deleted with its session
|
||||
(deletion-manifest step) and on tenant offboarding.
|
||||
* Self-healing 30-day maintenance sweep (2h tick): backfills missing breakdowns, folds
|
||||
late-terminating sessions into their StartedAt-date aggregates, and enforces version
|
||||
purity — a stale-version row whose events aged out counts as missing, never mixed.
|
||||
Reads the tick's **shared window scan** (`MaintenanceService.LoadSweepWindowSessionsAsync`,
|
||||
one projected 35-day cross-tenant drain via `MaintenanceSweepSessionProjection` that
|
||||
also feeds the device-journey and verdict-calibration sweeps) — the StartedAt-only
|
||||
filter is a full-table read in Table Storage, so it is done once per tick, not once per
|
||||
sweep; `MaintenanceSweepProjectionEquivalenceTests` pins the column set.
|
||||
* `TimeAttributionAggregates` (PK = TenantId or `global`, RK = `{yyyy-MM-dd}|{class}` and
|
||||
`rolling30|{class}`): per enrollment class (`user_driven`, `whiteglove`, `self_deploying`,
|
||||
`device_preparation` — never mixed) median/p75/p90 per segment over the fixed six-segment
|
||||
stack (incl. `unattributed`; a session without a segment contributes 0), plus top blocking
|
||||
apps (≥5 sessions per app row). Clean/flagged/missing counts are part of every row; rows
|
||||
exist below the ≥20 UI gate on purpose (the UI needs the n for "insufficient data (n=…)").
|
||||
Daily rows retain 180 days; `rolling30` rows are refreshed whole (a median of per-day
|
||||
medians is not the range median — the rolling rows are the honest range statistics).
|
||||
|
||||
## Surfaces
|
||||
|
||||
* `GET api/sessions/{sessionId}/time-attribution` (MemberRead + tenant query param) —
|
||||
breakdown row or `breakdown: null` (pre-feature / non-terminal / Incomplete = no lane).
|
||||
* `GET api/metrics/time-attribution` (MemberRead) / `GET api/global/metrics/time-attribution`
|
||||
(GlobalReadOrAdmin, optional `tenantId`) — `{ windowDays, classes (rolling), daily }`.
|
||||
* Web: attribution lane under the session-detail phase timeline; "Time attribution" section
|
||||
on Fleet Health. MCP: `get_time_attribution` (sessionId mode / fleet mode).
|
||||
|
||||
# Examples
|
||||
|
||||
A 30-minute user-driven session: `device_prep` 5m → `esp_apps` 10m (occupancy 4m by one
|
||||
blocking app) → `identity_hello` 8m → `user_esp` 5m → `desktop_handoff` 2m, unattributed 0.
|
||||
A WhiteGlove session with a 2-day pause reports wall clock = part1 + part2 only; the resume
|
||||
gap before the first part-2 phase declaration is unattributed, not guessed.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs` — calculator + semantics (AttributionVersion 2).
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.TimeAttribution.cs` — persistence.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs` — sweep + aggregation.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/TimeAttributionCalculatorTests.cs` — golden fixtures pinning the invariant.
|
||||
* `tasks/insights-expansion-spec.md` — F1 specification and source-data audit (§0.5).
|
||||
@@ -1,176 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Verdict Calibration (classifier thermometer)
|
||||
description: How every session status write declares its VerdictPath, how overriding a verdict preserves it as PriorStatus/PriorVerdictPath, how the daily calibration aggregate counts paths with the re-enrollment proxy and the correction stream, and how the drift radar (share / silence / evidence-gap) turns that into operator-only ops events.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs
|
||||
tags:
|
||||
- backend
|
||||
- sessions
|
||||
- classifier
|
||||
- calibration
|
||||
- regression-radar
|
||||
timestamp: 2026-08-27T20:00:00+02:00
|
||||
---
|
||||
|
||||
# Verdict Calibration (classifier thermometer)
|
||||
|
||||
A session's terminal status is written by ~20 code paths: the agent's own terminal events,
|
||||
the silence classifier (`EnrollmentTimeoutClassifier`, rules 1–6) reached via the
|
||||
maintenance sweep / the agent's max-lifetime shutdown / late-telemetry heal / retro
|
||||
reclassification, the analyze-rule engine, admin marks and session superseding. Verdict
|
||||
calibration makes every one of those paths **countable**, tracks how often a path's verdict
|
||||
was later **overridden**, attaches the delayed **re-enrollment proxy**, and raises an
|
||||
operator-only alarm when a path's share **drifts**. It is deterministic, fact-based tuning
|
||||
input for the rules — explicitly not a learned model, and never a customer-facing feature.
|
||||
|
||||
# Schema
|
||||
|
||||
## VerdictPath on the session row
|
||||
|
||||
`SessionSummary.VerdictPath` (`VerdictPaths` vocabulary, `origin:detail`, append-only) is
|
||||
stamped by **every** status write — `UpdateSessionStatusAsync` takes it as a required
|
||||
parameter, `StoreSessionAsync` stamps `register:new` / `register:whiteglove_resume` and
|
||||
preserves an existing path through the re-registration Replace, `SetSessionPreProvisionedAsync`
|
||||
requires it whenever it writes a status. Unlike `FailureSource` (persisted only on `Failed`),
|
||||
the path survives for Incomplete, AwaitingUser, Stalled and Pending.
|
||||
|
||||
| Origin | Paths |
|
||||
| --- | --- |
|
||||
| `agent` | `complete`, `complete_soft` (Continue-Anyway), `failed`, `esp_failure_fallback`, `gather_complete`, `whiteglove_pending`, `whiteglove_resumed`, `stall_probe`, `stall_heal` |
|
||||
| `ingest` | `wg_awaiting` (stall probe mapped to WhiteGlove AwaitingUser) |
|
||||
| `sweep` | `stalled`, `wg_awaiting`, `sd_reconcile`, `r{rule}` (stage-2 classifier) |
|
||||
| `maxlife` / `late` / `retro` | `r{rule}` (classifier via max-lifetime shutdown / late-telemetry heal / retro reclassification); `retro:sd_reconcile`, `retro:superseded` |
|
||||
| `register` | `new`, `whiteglove_resume`, `superseded` |
|
||||
| `rule` | `rule:{ruleId}` (analyze rule with MarkSessionAsFailed) |
|
||||
| `manual` | `failed`, `succeeded` |
|
||||
| `legacy` | read-side derivation only (`legacy:{rule}` when the rule is recoverable from the reason literal, `legacy:unknown` otherwise) — never written |
|
||||
|
||||
Classifier rule ids (`ClassifierRules`): `r1`, `r1b_awaiting`, `r1b_succeeded`, `r1c`, `r2`,
|
||||
`r3`, `r4`, `r5_awaiting`, `r5_incomplete`, `r6`. `ClassifyTimedOutSession` returns
|
||||
`(Status, Reason, Rule)` so each `return` names its rule.
|
||||
|
||||
## PriorStatus / PriorVerdictPath — the correction stream
|
||||
|
||||
When a write replaces a prior **verdict** (existing status ∈ Succeeded/Failed/Incomplete/
|
||||
AwaitingUser with a stamped path, incoming status different), the storage seam preserves the
|
||||
pair (`TableStorageService.ComputePriorVerdict`, both the ETag path and the force path). A
|
||||
stall marker healing to InProgress, a same-status refresh, or an unstamped legacy row never
|
||||
sets it. Admin marks, late agent completions and retro reclassifications all flow through it.
|
||||
|
||||
## Read-side derivation for pre-instrumentation rows
|
||||
|
||||
`VerdictPathDerivation.Derive(session)` returns the stamped path verbatim, otherwise derives
|
||||
from `AdminMarkedAction`, `FailureSource`, `ReconcileReason` and `FailureReason` using the
|
||||
writers' fixed literals (classifier rule prefixes, the `max-lifetime watchdog` / `Retro-
|
||||
reclassified` suffixes, `Rule: `, `Superseded by session`, `Late completion report received`).
|
||||
Unambiguous rows get the real path (`agent:complete`, `manual:failed`, `rule:…`,
|
||||
`maxlife:r6`), ambiguous-but-rule-known rows `legacy:{rule}`, the rest `legacy:unknown`. No
|
||||
mass backfill: the aggregate counts derived rows with `DerivedCount` so the matrix shows
|
||||
history while flagging its weaker evidence. Tests pin the literals against the classifier
|
||||
output so a wording change fails loudly.
|
||||
|
||||
## Daily aggregate — `VerdictCalibrationAggregates`
|
||||
|
||||
PK = `TenantId` | `"global"`, RK = `yyyy-MM-dd` (session **StartedAt** date, like every daily
|
||||
aggregate). `VerdictCalibrationDailyAggregate { Version, SessionCount, TerminalSessionCount,
|
||||
Buckets[], ComputedAt }` with one bucket per (VerdictPath, Status):
|
||||
|
||||
| Field | Meaning |
|
||||
| --- | --- |
|
||||
| `Count` / `DerivedCount` | sessions currently on the path; of those, derived read-side |
|
||||
| `Eligible7d` | terminal sessions whose end (`CompletedAt ?? LastEventAt ?? StartedAt`) lies ≥ 7 d before compute time |
|
||||
| `ReEnrolled7d` | of the eligible: the device's `DeviceHistories` chain shows the next terminal session starting < 7 d after this one's end |
|
||||
| `OverriddenByAdmin` / `OverriddenByLateCompletion` / `OverriddenOther` | sessions whose **PriorVerdictPath** is this bucket — attributed to the path that was overridden, so a bucket can carry overrides with `Count = 0` |
|
||||
|
||||
`MaintenanceService.SweepVerdictCalibrationAsync` (2-hourly, after the device-journey sweep
|
||||
so the chains are fresh; manual maintenance parity; kill switch
|
||||
`VerdictCalibrationSweepDisabled`) slices the tick's shared projected window scan
|
||||
(`LoadSweepWindowSessionsAsync` — one 35-day cross-tenant drain feeds the time-attribution,
|
||||
device-journey and calibration sweeps plus the radar's tenant discovery; the StartedAt-only
|
||||
filter is a full-table read, so never per sweep), recomputes the rolling 30-day window whole (Replace),
|
||||
deletes stale date rows it did not regenerate, and the unbounded-tables cleanup applies the
|
||||
180-day retention. Sessions inside a deletion cascade are skipped; the table is regenerable
|
||||
and is wiped with the tenant on offboarding.
|
||||
|
||||
## Read surfaces
|
||||
|
||||
* `GET global/metrics/verdict-calibration?days=&tenantId=` (GlobalReadOrAdmin,
|
||||
TenantScoping.QueryParam; `VerdictCalibrationResponse.Build`): per path — count, share of
|
||||
all window sessions, derived count, eligible/re-enrolled with `reEnrollRatePct` **null below
|
||||
20 eligible**, the three override counters, a today-anchored 7d-vs-28d share trend
|
||||
(`window7`, `baseline28`, `lift` null without baseline) — plus totals, the trend basis and
|
||||
the active `alerts[]`. Empty but well-formed before the first sweep.
|
||||
* MCP `get_verdict_calibration` (platform scope: GA + Global Reader). Same endpoint, LLM-shaped in
|
||||
`verdict-calibration-shape.ts`: per-row `window7.sessions` / `baseline28.sessions` are dropped (the
|
||||
denominators live once in `trend`), `reEnrollRatePct` / `lift` are always present (`null` = withheld),
|
||||
and `minSharePct` / `top` trim the one-session tail — rows carrying overrides are never trimmed and
|
||||
everything dropped is reported in `omitted`. The wire shape the admin page consumes is untouched.
|
||||
* Portal: Global Admin → Metrics → Verdict Calibration (`/admin/metrics/verdict-calibration`):
|
||||
matrix grouped by origin, status pill, `derived n` gray pill, trend glyph (arrow only from
|
||||
5 window hits and lift ≥ 2 / ≤ 0.5), re-enrollment rate or `— (n=…)`, overrides
|
||||
`admin / late / other`, `↑ Drift` pill on rows with an active episode, and the active
|
||||
episodes list.
|
||||
|
||||
## Drift radar — `VerdictCalibrationRadar`
|
||||
|
||||
Same statistics as the [rule regression radar](rule-regression-radar.md) (7-day window vs the
|
||||
prior 28 days, ≥ 20 window sessions, lift ≥ 2, `MetricsMath.RateIncreaseSeparated`), with
|
||||
stricter floors since the first-month tuning (2026-08-27): **≥ 10 window hits** (verdict
|
||||
shares on single-digit counts are noise — a 3-hit episode survived as such), and a per-path
|
||||
share regression additionally needs **≥ 5 baseline hits** — a path with no established
|
||||
baseline is new (or renamed) vocabulary, not a regression. Anchored on yesterday, over every
|
||||
partition with rows (each tenant and `global`):
|
||||
|
||||
| Kind | Signal | Re-arm |
|
||||
| --- | --- | --- |
|
||||
| `share_regression` | one **backend-decided** path's share of all sessions doubled (`sweep`/`maxlife`/`late`/`retro`/`rule`/`manual`/`ingest`; `agent:*` and `register:*` mirror customer workflow mix — first prod pass fired on `agent:whiteglove_pending` rollout weeks — and `legacy:*` is derived, all excluded per-path) | share < 1.5× baseline, the path stops occurring, the path is no longer eligible, or the current numbers fall below the fire floors |
|
||||
| `silence_share_regression` | the `sweep:*` + `maxlife:*` share doubled — the backend had to decide more often because the agent went silent (a liveness signal, not a classifier one). Rule-shaped `legacy:*` paths count into BOTH sums (window and baseline): they were the same backend decisions before the 2026-08-23 instrumentation, and without that continuity the group had a near-zero baseline by construction and fired a lift-124 rollout artifact | share < 1.5× baseline or hits stop |
|
||||
| `evidence_gap` | absolute: `r6` (pure fallthrough) decides ≥ 20 % of the window's classifier verdicts (stamped rule paths plus rule-shaped `legacy:*`, same continuity), ≥ 20 verdicts | < 15 % or < 20 verdicts |
|
||||
|
||||
Episodes are tracker rows `verdictcalibration|{kind}|{path}|{status}` (register = `AddEntity`,
|
||||
409 = already burning, fail-closed; refresh carries `FirstNotifiedAt`; 30-day tracker
|
||||
retention re-arms long burners). A new episode fires **one** `VerdictCalibrationDrift` ops
|
||||
event (category Maintenance, Warning; dual-registered in the web `OPS_EVENT_TYPES`) — there
|
||||
is deliberately **no tenant bell**. Per-path findings on a tenant partition get the on-fire
|
||||
dimension correlation (`RuleRegressionRadar.ComputeDimensionConcentration` over the window's
|
||||
sessions whose derived path matches), reported with the shared wording contract "correlated,
|
||||
not necessarily causal"; group kinds and the global partition carry no dimension claim.
|
||||
Kill switch `VerdictCalibrationRadarDisabled`.
|
||||
|
||||
# Examples
|
||||
|
||||
* **Reading the thermometer (prod, 2026-08-23 hand analysis that motivated this):**
|
||||
re-enrollment within 7 days after Succeeded 8.9 % (lab background), after Failed 24.3 %,
|
||||
after Incomplete 10.6 % — Incomplete behaves like Succeeded, so rules 5/6 are probably too
|
||||
cautious; the matrix now shows this per rule instead of per status.
|
||||
* **First real calibration read (2026-08-27, four days instrumented):** the per-rule and
|
||||
per-tenant split resolved that hand analysis — `legacy:r5_incomplete` re-enrolled at 2.6 %
|
||||
(the `agent:complete_soft` background), spread across tenants, while `legacy:r6`'s 29 %
|
||||
was concentrated in one tenant that disciplinedly redoes failed enrollments (48.5 % there,
|
||||
6.7 % elsewhere — so r6 stays untouched, it correctly flags that tenant). Consequences
|
||||
shipped the same day: rule 5a "completed (assumed)" (see
|
||||
[Silent-Session Classification](silent-session-classification.md)), the max-lifetime grace
|
||||
skip, and the radar floors + legacy group continuity above — the first production month
|
||||
had produced nine standing alerts, every one an instrumentation-rollout artifact.
|
||||
* **Correction stream:** a session swept to `sweep:r5_incomplete`, later completed by the
|
||||
agent → row `agent:complete` with `PriorVerdictPath = sweep:r5_incomplete`; the aggregate
|
||||
increments `OverriddenByLateCompletion` on the `sweep:r5_incomplete / Incomplete` bucket.
|
||||
Four admin marks since launch were invisible before — now each one counts against the path
|
||||
it corrected.
|
||||
* **Evidence gap:** a tenant whose agent build stopped emitting the Device Setup rollup line
|
||||
drives its sweep verdicts into `r6`; the absolute gate fires at 20 % and names the
|
||||
tenant — the fix is agent-side, the classifier is only the messenger.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/VerdictPaths.cs` — vocabulary + `ClassifierRules`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs` — `ComputePriorVerdict`, `ApplyVerdictPath`, re-registration preserve
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/VerdictPathDerivation.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs` — sweep + `BuildVerdictCalibrationAggregates`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetVerdictCalibrationFunction.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/VerdictCalibrationRadar.cs`, `Services/MaintenanceService.VerdictCalibrationRadar.cs`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts` — `get_verdict_calibration`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/verdict-calibration-shape.ts` — LLM-facing response shaping (`minSharePct` / `top`, explicit nulls)
|
||||
* `src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionVerdictCalibration.tsx`
|
||||
* Tests: `VerdictPathTests`, `VerdictCalibrationTests`, `VerdictCalibrationRadarTests`
|
||||
* Related: [Silent-Session Classification](silent-session-classification.md), [Device Journeys](device-journeys.md), [Rule Regression Radar](rule-regression-radar.md)
|
||||
@@ -1,160 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Vulnerability Correlation Sources (NVD, KEV, MSRC, EPSS)
|
||||
description: How the software inventory is turned into CVE findings - CPE mapping, the per-CPE NVD cache with its budget and TTLs, KEV/MSRC/EPSS enrichment, the throttle contract that keeps a refused NVD answer out of the cache, and the act/attend/track priority.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs
|
||||
tags:
|
||||
- backend
|
||||
- vulnerability
|
||||
- nvd
|
||||
- epss
|
||||
- kev
|
||||
- msrc
|
||||
- cache
|
||||
timestamp: 2026-08-29T18:00:00+02:00
|
||||
---
|
||||
|
||||
# Vulnerability Correlation Sources
|
||||
|
||||
The Software Inventory & Vulnerability Analyzer (tenant opt-in) ships the installed-software
|
||||
list with the session; the backend turns it into CVE findings entirely from cached public
|
||||
reference data. Four sources, all trustworthy primaries, no aggregators:
|
||||
|
||||
| Source | What it contributes | Fetch shape | Cache |
|
||||
| --- | --- | --- | --- |
|
||||
| NVD 2.0 (NIST) | CVE records per product: CVSS v3.1 (v3.0 fallback) score, severity, **vector**, description, affected version ranges; CPE dictionary for auto-mapping | `cves/2.0?virtualMatchString={cpe}` per product, `cpes/2.0?keywordSearch=` for unmapped titles; optional API key (`AdminConfiguration.NvdApiKey`) | Azure Table `VulnerabilityCache`, one row per CPE (`cve_{cpe}` / `cache_result`), `CveDataJson` chunked; soft TTL 7 d (serve + background refresh), hard 14 d |
|
||||
| CISA KEV | "Exploited in the wild" flag, due date, required action, ransomware use | Whole catalog JSON | In-memory dictionary, 24 h, blob mirror `kev-catalog.json` |
|
||||
| MSRC | Microsoft-side confirmation: document id/title, release date, Microsoft severity / score / vector | CVRF v3.0 index of the last six Patch Tuesdays | In-memory index, blob mirror `msrc-index.json` |
|
||||
| FIRST EPSS | Probability (0..1) that the CVE is exploited within 30 days + percentile + scoring date | `api.first.org/data/v1/epss?cve=a,b,c` - 100 ids per request, unauthenticated | Stored **on the cached CVE** (no separate table): filled when a CPE row is (re)fetched, re-scored for every row by the nightly sync |
|
||||
|
||||
Nothing about a tenant leaves the backend. The one outbound query that carries content is the
|
||||
EPSS lookup, and its payload is the list of public CVE ids held in the shared, tenant-agnostic
|
||||
CPE cache. Customer-facing wording: `autopilotmonitor-docs/trust/data-flows.md` and the
|
||||
terms page attribution block.
|
||||
|
||||
# Schema
|
||||
|
||||
## Cached CVE (`CachedCve`, serialised into `CveDataJson`)
|
||||
|
||||
```
|
||||
CveId, CvssScore, CvssSeverity, CvssVector?, Description (<=500), PublishedDate,
|
||||
EpssScore?, EpssPercentile?, EpssDate?, AffectedVersions[] {Criteria, VersionStart/End Incl/Excl}
|
||||
```
|
||||
|
||||
`JsonOptions` is `WhenWritingNull` + case-insensitive, so rows written before the vector / EPSS
|
||||
properties existed deserialise with those as `null`, and a row is never rewritten just to add
|
||||
nulls. `CveCacheBudget` bounds the row (24 months, or CVSS >= 7 within 60 months; max 100 CVEs
|
||||
per CPE, 50 ranges per CVE) - which is also why one CPE fill is exactly one EPSS request.
|
||||
|
||||
`ProjectNvdResponse` is the single NVD -> `CachedCve` projection; the live fill and the nightly
|
||||
refresh both call it, so both write the same shape. `BuildCveCacheRow(cpe, cves, cachedAt)` is the
|
||||
single row builder; `cachedAt` is the **NVD** freshness stamp, an EPSS-only rewrite passes the
|
||||
existing value through so it does not look like a fresh NVD fetch to the 7-day TTL.
|
||||
|
||||
## Per-finding output (report `findings[].vulnerabilities[]`)
|
||||
|
||||
```
|
||||
cveId, cvssScore, cvssSeverity, cvssVector?, description, publishedDate, source ("NVD" | "NVD+MSRC"),
|
||||
isKev, kevDueDate, kevRequiredAction, knownRansomware,
|
||||
epssScore?, epssPercentile?, epssDate?,
|
||||
priority ("act" | "attend" | "track"),
|
||||
msrc* (when MSRC confirmed), versionMatchConfidence ("low" on wildcard CPE matches)
|
||||
```
|
||||
|
||||
Finding level: `riskLevel` (unchanged CVSS+KEV bands), `priority` (highest band), `maxEpssScore?`.
|
||||
`scan_summary` additionally carries `act_cves` / `attend_cves`. `MergeReports` (Part 1 + Part 2)
|
||||
recomputes all of these. The `CveIndex` row (one per tenant/CVE/session) carries `EpssScore`
|
||||
(max over occurrences, absent when unscored) and `Priority` (highest), which the fleet summary
|
||||
(`TopCve.epssScore` / `priority`) reads.
|
||||
|
||||
## Priority (`CvePriority.Compute`)
|
||||
|
||||
* `act` - in CISA KEV.
|
||||
* `attend` - EPSS >= 0.1 **or** CVSS >= 9.0.
|
||||
* `track` - everything else.
|
||||
|
||||
Three visible rules instead of a weighted composite: severity (`riskLevel`) keeps saying how bad,
|
||||
priority says how soon. A CVE without an EPSS score is "unknown likelihood" - it can still reach
|
||||
`attend` via CVSS, never via a defaulted 0. The thresholds are constants; changing them is a
|
||||
product decision, not a tuning knob.
|
||||
|
||||
# Throttle contract (NVD)
|
||||
|
||||
The typed `HttpClient` runs under the `ExternalDataApi` Polly policy: 429/5xx/timeouts are retried
|
||||
three times (1/2/4 s + jitter) with a circuit breaker. A 429 that reaches `NvdApiClient` therefore
|
||||
means NVD is refusing us for real. Two things then happen, and one deliberately does not:
|
||||
|
||||
1. `NvdFetchResult<T>` reports `Succeeded = false`, `Throttled = true`. `Data` is an empty object
|
||||
so enumerating callers keep working, but **nothing that persists may treat it as an answer**:
|
||||
the CPE cache fill returns null and writes no row; the negative auto-resolve cache is not
|
||||
touched; the admin auto-resolve reports `nvd_throttled`. Before this contract an empty response
|
||||
was cached as "0 CVEs" for seven days - a product NVD happened to throttle disappeared from
|
||||
every report for a week.
|
||||
2. A process-wide cooldown opens: `Retry-After` when present (delta or date, capped at 120 s),
|
||||
otherwise 30 s. During the cooldown every NVD call fails fast with `Throttled` - no wait, no
|
||||
retry chain. Worst case added latency per cooldown window is one Polly chain (~7.5 s), never
|
||||
a per-item stall; the nightly refresh breaks out of its loop on the first throttled answer and
|
||||
reports the remainder as `cveSkippedThrottled`.
|
||||
3. Not done on purpose: no in-process retry beyond Polly, no VulnCheck-style mirror, no separate
|
||||
"throttled" cache state. Hard-expired cache rows keep serving while NVD is unavailable
|
||||
(better than nothing, logged); a product with no row at all is reported without CVE data for
|
||||
that run and gets a real attempt on the next correlation.
|
||||
|
||||
# Nightly sync (`VulnerabilityDataSync`, 03:00 UTC) and the manual buttons
|
||||
|
||||
1. KEV catalog refresh (blob mirror updated).
|
||||
2. MSRC index refresh (last six Patch Tuesdays).
|
||||
3. NVD refresh of cache rows older than 7 days (sequential, breaks on throttle).
|
||||
4. **EPSS re-score of every `cve_*` row** - all ids batched through FIRST, only rows whose score /
|
||||
percentile / date changed are rewritten (with their original `CachedAt`). A missing id in the
|
||||
response keeps yesterday's score: a failed batch never erases data.
|
||||
|
||||
Steps 3 and 4 live in `VulnerabilityCacheRefreshService` (singleton, one in-flight slot per walk),
|
||||
which the timer and the Global-Admin buttons share, so a manual run writes exactly what the nightly
|
||||
run writes. Four endpoints, one per source, each with its own "Last … Sync" line on the admin page
|
||||
(`GET vulnerability/sync-status`):
|
||||
|
||||
| Button | Endpoint | Shape | Timestamp |
|
||||
| --- | --- | --- | --- |
|
||||
| Sync KEV | `POST vulnerability/sync` | synchronous | `VulnerabilityDataLastSyncUtc` |
|
||||
| Sync MSRC | `POST vulnerability/sync-msrc` | synchronous | `MsrcLastSyncUtc` |
|
||||
| Refresh NVD Cache | `POST vulnerability/sync-nvd` | **202 + background** (minutes, NVD-rate-limited); 409 while running; the page polls `nvdRefreshRunning` / `nvdLastRun` every 10 s | `NvdCacheLastRefreshUtc` - stamped only for a walk that reached every stale row (no throttle cut-off) |
|
||||
| Sync EPSS | `POST vulnerability/sync-epss` | synchronous (seconds); 409 while running | `EpssLastSyncUtc` |
|
||||
|
||||
The timer skips a walk whose slot a manual run currently holds (`cveSkippedBusy` in telemetry)
|
||||
rather than queueing a duplicate. `nvdLastRun` / `epssLastRun` are in-memory on the answering
|
||||
instance (null after a cold start); the persisted timestamps are the durable signal.
|
||||
|
||||
Telemetry event `VulnerabilityDataSyncCompleted` carries `epssRows / epssCves / epssScored /
|
||||
epssRowsRewritten / epssError` next to the KEV / MSRC / CVE counters; the manual EPSS run emits
|
||||
`ManualEpssSyncCompleted`.
|
||||
|
||||
# Examples
|
||||
|
||||
* Chrome 126 finding: `CVE-2024-4947` CVSS 8.8 HIGH, `AV:N/AC:L/PR:N/UI:R`, KEV yes -> `priority =
|
||||
act`, riskLevel `critical` (KEV) - unchanged severity semantics, priority now explicit.
|
||||
* A 5.5 MEDIUM local-privilege CVE with EPSS 0.31 -> `attend`; the same CVE with EPSS 0.004 ->
|
||||
`track`. Same severity band, different order of work.
|
||||
* Nightly sync while NVD throttles: `[VulnSync] NVD throttled - leaving 37 stale entries for the
|
||||
next run`, then `[VulnSync] EPSS refresh: 1412/1450 CVEs scored across 212 rows, 180 rows
|
||||
rewritten` - EPSS is independent of NVD's mood.
|
||||
|
||||
# Not done (recorded)
|
||||
|
||||
* **ExploitDB CSV** as a "public exploit exists" flag (static file, daily, same shape as the KEV
|
||||
mirror) - useful, cheap, deliberately not added: four primary sources are enough for now and
|
||||
each additional outbound dependency is a trust-page line. Revisit if customers ask "is there a
|
||||
public exploit?".
|
||||
* GitHub / Nuclei PoC search, composite 0-100 risk scores, CWE -> ATT&CK mapping - noise for an
|
||||
Intune admin, not adopted.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/NvdApiClient.cs` - `NvdFetchResult`, cooldown, `ComputeCooldown`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/EpssApiClient.cs` - batching, `ParseResponse`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/CvePriority.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs` - `ProjectNvdResponse`, `ApplyEpssScores`, `BuildCveCacheRow`, `StampFindingPriority`, `FetchAndCacheCvesFromNvdAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Vulnerability/VulnerabilityDataSyncFunction.cs` - `RefreshStaleCveCacheAsync`, `RefreshEpssScoresAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.AgentApi.cs` - `BuildCveIndexEntities`, `ScanCveIndexAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/EpssAndNvdThrottleTests.cs`
|
||||
* FIRST EPSS API: https://api.first.org/epss/ - CISA KEV: https://www.cisa.gov/known-exploited-vulnerabilities-catalog
|
||||
@@ -1,192 +0,0 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Drift Guards — SignalR Names, RowKeys, Wire DTOs, Tool Vocabularies, Config Sections, Scoped Routing
|
||||
description: The 2026-08 fragility-audit follow-up contracts that turn formerly hand-synchronized mirrors into enforced ones — SignalR message-name catalog, the inverted-tick RowKey codec, tombstone table tagging, typed wire DTOs with generated TypeScript interfaces, the MCP's generated tool vocabularies with their inline-enum ratchet, per-section config PATCH, and the web's scoped-endpoint builders.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions.Tests/TypedResponseGuardTests.cs
|
||||
tags:
|
||||
- contracts
|
||||
- guard-tests
|
||||
- signalr
|
||||
- web
|
||||
timestamp: 2026-08-31T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Drift Guards
|
||||
|
||||
Follow-up to the 2026-08 fragility audit (see [Lifecycle Manifests & Session Scope
|
||||
Resolution](backend/lifecycle-manifests-and-session-scope.md) for the earlier rounds).
|
||||
Each section below replaces a hand-synchronized mirror with an enforced contract: a
|
||||
change on one side now fails a build, a type check, or a test on the other side.
|
||||
|
||||
# Schema
|
||||
|
||||
## SignalR message names — one catalog, typed on both ends
|
||||
|
||||
`Constants.SignalRMessages` (Shared) is the complete catalog of SignalR target names.
|
||||
Backend senders (output bindings and the imperative `SignalRNotificationService`)
|
||||
reference the constants; the hub name is `SignalRGroupHelper.HubName`. The catalog is
|
||||
exported into `shared-manifests.json` (`signalRMessages` section) by
|
||||
`SharedManifestParityTests`, and the web's `SignalRContext.on/off` type their event
|
||||
name against the generated union (`lib/signalrMessages.ts`) — subscribing to a name
|
||||
the backend never sends is a tsc error, not a silent no-op. `newevents` is the one
|
||||
legacy lowercase name; it is a persisted wire name, do not "fix" the casing.
|
||||
|
||||
## Inverted-tick RowKeys — one codec
|
||||
|
||||
`RowKeyCodec` (Functions/Helpers) is the single encoder/decoder for
|
||||
`rev(t) = MaxTicks - t.Ticks` RowKeys. Every writer routes through it; the persisted
|
||||
per-table shapes (D19 standard, AuditLogs `!`-prefix, UserActivity's legacy D20 width,
|
||||
ConfigurationBackups' 28-char truncation) are pinned by `RowKeyCodecTests` and must
|
||||
never change — a width/prefix change reorders live tables against their existing rows.
|
||||
`BusinessTimestamp` keeps the OData clause builders and delegates decode to the codec.
|
||||
|
||||
## Tombstone rows carry their table
|
||||
|
||||
`DeletionRowDump.Table` names the source table on FINAL (Tombstone) rows — the one
|
||||
step that mixes two tables while `DeletionStep.Table` is null.
|
||||
`DeletionTombstoneTables.Resolve` is the only consumer-side resolution (delete +
|
||||
restore + dry-run); the historical `Contains('_')` RowKey-shape heuristic survives
|
||||
solely as the fallback for manifests written before the field existed and must outlive
|
||||
one full manifest-retention cycle.
|
||||
|
||||
## Wire DTOs — generated interfaces, zero anonymous success bodies
|
||||
|
||||
Every HTTP success body is a flat class in `AutopilotMonitor.Shared.Models`
|
||||
implementing the `IApiResponse` marker (2026-08-31, `feat/typed-api-contract` —
|
||||
migrated from the 44 `OkAsync(new { … })` + 134 raw `WriteAsJsonAsync(new { … })`
|
||||
sites the 2026-08-13 audit froze). Three layers enforce it:
|
||||
|
||||
- **Compile time**: the only `OkAsync/CreatedAsync/JsonAsync` overloads constrain on
|
||||
`IApiResponse` — an anonymous object cannot satisfy the constraint. Error bodies
|
||||
(first key `error`/`message`, or literal `success = false`) stay anonymous by design.
|
||||
- **`TypedResponseGuardTests`** (successor of `OkAsyncBaselineGuardTests`): both
|
||||
per-file baselines are EMPTY — any anonymous success body, through the helpers or
|
||||
raw `WriteAsJsonAsync`, is a straight failure. A reflection fact keeps every
|
||||
`IApiResponse` implementer flat (System.Text.Json serializes derived properties
|
||||
before base ones — key order is wire contract; MCP hands raw JSON to an LLM) and
|
||||
inside the Shared assembly.
|
||||
- **`*WireParityTests`**: each migrated site carries an ordinal old-anonymous-literal
|
||||
vs. new-DTO serialization proof (production `ApiJsonOptions`), including a null case
|
||||
per key that WhenWritingNull omits.
|
||||
- **Object-slot ratchet** (2026-08-31 typisierung follow-up): a reflection walk over
|
||||
the reachable wire-type graph fails on any `object` / collection-of-`object`
|
||||
property outside `ObjectSlotBaseline` (sole entry: `RuleDryRunCondition.Evidence`,
|
||||
heterogeneous by design). `[ProjectedItems]` slots and dictionary VALUES are exempt.
|
||||
The former deliberately-untyped slots (RuleDryRun trace, HealthCheck,
|
||||
GlobalNotificationDto, TenantConfigFieldSchema, maintenance run reports,
|
||||
MetricsSummary items, auth/me, auth/mcp) are concrete Shared types now. The raw
|
||||
table entities left the wire entirely in the same pass: global-admins,
|
||||
tenant-admins and preview-whitelist serve flat rows (`GlobalAdminRow` /
|
||||
`TenantAdminRow` / `PreviewWhitelistTenantEntry`) WITHOUT
|
||||
partitionKey/rowKey/eTag/timestamp — a deliberate wire change pinned by exact-JSON
|
||||
shape tests instead of parity facts.
|
||||
|
||||
The DTO rules: declaration order == wire order; a property is nullable exactly when a
|
||||
site can emit null (the key then vanishes — never add non-null defaults that would
|
||||
invent a key); preformatted strings (`.ToString("o")`, enum `.ToString()`) stay
|
||||
`string`; `fields=` projections stay `IReadOnlyList<object>` tagged
|
||||
`[ProjectedItems(typeof(Item))]`; raw table rows stay verbatim-keyed dictionaries
|
||||
(no `DictionaryKeyPolicy` — dictionary KEYS never run through the camelCase policy,
|
||||
and dictionary VALUES keep explicit nulls, both pinned by tests).
|
||||
|
||||
`SharedManifestParityTests` (schemaVersion 2) reflects the full graph — every
|
||||
`IApiResponse` implementer plus `[WireContract]`-marked payload types, transitively
|
||||
closed by `WireTypeManifestBuilder` — into the `types` section of
|
||||
`shared-manifests.json`, carrying C# `<summary>` texts. The web codegen
|
||||
(`npm run generate:manifests`) emits `utils/wire-types.generated.ts` (one interface
|
||||
per object, one string union per enum, JSDoc from the summaries); the hand-written
|
||||
`types/session.ts` / `types/enrollment.ts` / `types/adminConfig.ts` re-export from
|
||||
it. Freshness is pinned by the vitest suite and the `shared-manifests-in-sync` CI
|
||||
job. Unmappable shapes (non-string dictionary keys, foreign BCL classes) fail the
|
||||
manifest build instead of degrading to `unknown`.
|
||||
|
||||
## Tool vocabularies — the MCP derives them, it no longer retypes them
|
||||
|
||||
Wire TYPES reached the MCP from day one (`src/generated/wire-types.generated.ts`, the
|
||||
same codegen output the web gets). Vocabularies did not: types are erased at runtime and
|
||||
a `z.enum([...])` needs values, so every MCP tool schema hand-typed its own copy of
|
||||
statuses, severities and categories — with nothing to compare against. The copies drifted
|
||||
exactly as you would expect: `search_sessions` offered 5 of 8 session statuses (no
|
||||
`AwaitingUser`, no `Incomplete`), both event readers offered 4 of 6 severities (no `Debug`,
|
||||
no `Trace`), and `get_ops_events` named 6 ops categories while the backend wrote 7. None of
|
||||
that surfaces as an error — the tool just quietly cannot ask for the missing value.
|
||||
|
||||
The vocabularies now ship as VALUES. `MCP_VOCABULARIES` in the codegen maps manifest
|
||||
sections onto `src/generated/wire-vocabularies.generated.ts` (`export const
|
||||
SESSION_STATUSES = [...] as const` plus its union type); the tool schemas do
|
||||
`z.enum(SESSION_STATUSES)` and descriptions interpolate the list instead of spelling it.
|
||||
Freshness is pinned twice: `wire-types-freshness.test.ts` compares the committed file
|
||||
byte-for-byte against a fresh codegen run, and the `shared-manifests-in-sync` CI job
|
||||
diffs it after regenerating.
|
||||
|
||||
`vocabulary-drift.guard.test.ts` is the ratchet that keeps it that way: every inline
|
||||
`z.enum([...])` left in `src/tools/*.ts` must be on a baseline that names why it has no
|
||||
backend owner (MCP-local knobs like `depth: fast|deep`, plus three free-string columns —
|
||||
rule type, connection type, CVE risk — that have no C# constants class yet). A new
|
||||
hand-typed list fails until it is derived or reviewed onto the baseline, and a stale
|
||||
baseline entry fails too.
|
||||
|
||||
Ops event types got the missing owner: they were bare literals at the `OpsEventService`
|
||||
call sites, so nothing could enumerate them and both the portal alert-rule catalog and the
|
||||
MCP had to retype the list. `OpsEventTypes` (Shared) now declares all 77; the write sites
|
||||
use the constants (a raw literal fails `OpsEventTypeDualRegisterTests`), the manifest
|
||||
exports them, and `get_resource(name="ops_event_types")` serves categories + severities +
|
||||
types to the model from the generated copy.
|
||||
|
||||
Backend-internal lists follow the same rule: a list that enumerates a constants class IS
|
||||
the class. `TableOpsEventRepository.AllCategories` is `OpsEventCategory.All` rather than a
|
||||
retyped copy — the copy had gone stale and silently hid every `Platform` (Azure Monitor)
|
||||
event from the paged cross-category read while the unpaged path still showed it.
|
||||
|
||||
## Tenant settings — per-section PATCH
|
||||
|
||||
The web's Settings sections no longer PUT the full ~92-field configuration. Each
|
||||
section's exact write surface lives in `app/settings/sectionFieldMap.ts` (owned fields
|
||||
plus documented `alsoWrites` write-throughs); `saveConfiguration` diffs those fields
|
||||
against the loaded config and PATCHes only the changes through
|
||||
`PATCH config/{tenantId}/fields` — the transactional endpoint (CAS + fail-closed
|
||||
backup + exactly-these-fields verify + auto-rollback) that previously served MCP only.
|
||||
The policy line is TenantAdminOrGA; a tenant admin runs on the stricter
|
||||
`TenantConfigCallerTier.TenantAdmin` whose deny-list turns GA-only fields into
|
||||
explicit 400s. Deploy order: backend before web. `sectionFieldMap.test.ts` pins
|
||||
field↔section ownership (each field exactly one owner, all fields on the model, none
|
||||
server-denied); backups/revert stay GlobalAdminOnly.
|
||||
|
||||
## Scoped endpoint routing — one decision object
|
||||
|
||||
`lib/scopedApi.ts` owns the tenant/global endpoint-pair choice: pages pass their scope
|
||||
hook object (`routeGlobal`, `selectedTenantId`, `effectiveTenantId`) instead of
|
||||
hand-rolling `routeGlobal ? global(..., sel || undefined) : tenant(tid, ...)` with the
|
||||
tenant parameter at a different position per pair. `useGlobalAdminScope` is now a thin
|
||||
projection over `useAggregatedAdminScope` (one hook, aggregated as a mode) via the
|
||||
pure `resolveConcreteScopeView`. `lib/scopedFetch.ts` adds the shared JSON+ok-check
|
||||
fetch layer. `lib/navVisibility.ts` extracts the sidebar's roles × nav-config × guard
|
||||
logic; its matrix test also ratchets the known "sidebar shows it, guard bounces it"
|
||||
mismatches (plain Operator and GlobalReader vs. `/settings` — pending a user decision).
|
||||
|
||||
# Examples
|
||||
|
||||
Renaming a SignalR message end to end: change the constant in
|
||||
`Constants.SignalRMessages` → `AM_WRITE_SHARED_MANIFESTS=1 dotnet test --filter
|
||||
SharedManifestParityTests` → `node scripts/generate-shared-manifest-types.js` → tsc
|
||||
now flags every stale web subscription literal.
|
||||
|
||||
Adding a field to a Settings section: add it to the section in `sectionFieldMap.ts`
|
||||
(tsc validates the name against the manifest), wire the form state into the
|
||||
`updatedConfig` construction in `TenantConfigContext`, and extend the expected-fields
|
||||
list in `sectionFieldMap.test.ts`.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Shared/AutopilotMonitor.Shared/Constants.cs` — SignalRMessages catalog
|
||||
- `src/Web/autopilot-monitor-web/scripts/generate-shared-manifest-types.js` — `MCP_VOCABULARIES`, the vocabulary codegen
|
||||
- `src/McpServer/autopilot-monitor-mcp/src/__tests__/vocabulary-drift.guard.test.ts` — the inline-enum ratchet
|
||||
- `src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs` + `OpsEventTypeDualRegisterTests.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Helpers/RowKeyCodec.cs` + `RowKeyCodecTests.cs`
|
||||
- `src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionManifest.cs` — `DeletionRowDump.Table`, `DeletionTombstoneTables`
|
||||
- `src/Backend/AutopilotMonitor.Functions.Tests/TypedResponseGuardTests.cs` + `WireTypeManifestBuilder.cs` + the `*WireParityTests` files
|
||||
- `src/Shared/AutopilotMonitor.Shared/Models/CommonApiModels.cs` — `IApiResponse`, `[WireContract]`, `[ProjectedItems]`
|
||||
- `src/Web/autopilot-monitor-web/utils/wire-types.generated.ts` (via `scripts/generate-shared-manifest-types.js`)
|
||||
- `src/Web/autopilot-monitor-web/app/settings/sectionFieldMap.ts` + `__tests__/sectionFieldMap.test.ts`
|
||||
- `src/Web/autopilot-monitor-web/lib/scopedApi.ts`, `lib/navVisibility.ts`, `hooks/concreteAdminScopeView.ts`
|
||||
- [Version Contract](versioning.md) — the web `/version.json` stamp + deploy verify added in the same round
|
||||
|
Before Width: | Height: | Size: 131 KiB |
|
Before Width: | Height: | Size: 160 KiB |
|
Before Width: | Height: | Size: 214 KiB |
|
Before Width: | Height: | Size: 192 KiB |