mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 20:53:06 +00:00
Compare commits
59 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0ec481b479 | |||
| 3ec716a3ec | |||
| b0bb7233e4 | |||
| 6758dc59e4 | |||
| ffd70e0383 | |||
| c18154d76d | |||
| f524473305 | |||
| 8fe420b902 | |||
| 2e8c2cdb2f | |||
| e04b75dd53 | |||
| d55b0ff864 | |||
| 123ba0e55e | |||
| c8175ff7b1 | |||
| 0efeb0f734 | |||
| 7b975d46b6 | |||
| 7680bdff7b | |||
| 93094653a2 | |||
| ba7d14b070 | |||
| 7f7330e9e5 | |||
| 82934a3719 | |||
| 372f0c7c98 | |||
| 8057b44e48 | |||
| 7842d121a0 | |||
| 4bc2435ec5 | |||
| 1bf37f6761 | |||
| 88da317bf4 | |||
| 1e6de47433 | |||
| 6de21e821a | |||
| 55c898bb9c | |||
| e637317bdb | |||
| 4eac7d042c | |||
| c7ed3ec27d | |||
| f311e8746b | |||
| e0b6ce83cf | |||
| 7f7b590db4 | |||
| 4c2c447a2e | |||
| 8a794a1b52 | |||
| 6c53dc33e7 | |||
| 801891d57a | |||
| 1f6efca95b | |||
| 9ec8644be2 | |||
| 9acd10e1dc | |||
| b382e5678e | |||
| 2fee50276e | |||
| 8bb02acc4a | |||
| fd7d3973e7 | |||
| fd2b5c2da2 | |||
| a44e37a7c2 | |||
| d8a252562d | |||
| a5d12ba1d0 | |||
| 530c978bf8 | |||
| 09898d7ba0 | |||
| 71eda93e8b | |||
| 1cfc32c9aa | |||
| 11339c9868 | |||
| 540f543834 | |||
| 088c3b0479 | |||
| 3224774752 | |||
| e7ea431ccf |
+1
-1
Submodule .claude/commands updated: af1b8468c7...997df4b5ed
@@ -63,7 +63,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: src/Web/autopilot-monitor-web/package-lock.json
|
||||
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: src/McpServer/autopilot-monitor-mcp/package-lock.json
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
|
||||
- name: Regenerate
|
||||
run: node rules/scripts/combine.js
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
|
||||
- name: Install ajv-cli
|
||||
run: npm install -g ajv-cli ajv-formats
|
||||
|
||||
@@ -21,6 +21,11 @@ jobs:
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.action != 'closed')
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy Job
|
||||
# Dependabot PRs run without repo secrets, so the deploy token is absent.
|
||||
# Surface it as an env var so the deploy step can gate on it — secrets
|
||||
# cannot be referenced directly in an `if:` expression.
|
||||
env:
|
||||
DEPLOY_TOKEN: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
@@ -28,6 +33,9 @@ jobs:
|
||||
fetch-depth: 0 # Full history needed for git-based sitemap lastmod dates
|
||||
- name: Build And Deploy
|
||||
id: builddeploy
|
||||
# Skip (green) instead of failing when the token is missing (Dependabot
|
||||
# PRs); real deploys on main and internal PRs still have it.
|
||||
if: env.DEPLOY_TOKEN != ''
|
||||
uses: Azure/static-web-apps-deploy@v1
|
||||
env:
|
||||
NEXT_PUBLIC_ENTRA_CLIENT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_CLIENT_ID }}
|
||||
@@ -39,9 +47,6 @@ jobs:
|
||||
with:
|
||||
azure_static_web_apps_api_token: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
repo_token: ${{ secrets.GITHUB_TOKEN }} # Used for Github integrations (i.e. PR comments)
|
||||
# Dependabot PRs run without repo secrets, so the deploy token is absent.
|
||||
# Skip (green) instead of failing the check; real deploys on main still have the token.
|
||||
skip_deploy_on_missing_secrets: true
|
||||
action: "upload"
|
||||
###### Repository/Build Configurations - These values can be configured to match your app requirements. ######
|
||||
# For more information regarding Static Web App workflow configurations, please visit: https://aka.ms/swaworkflowconfig
|
||||
@@ -54,9 +59,12 @@ jobs:
|
||||
if: github.event_name == 'pull_request' && github.event.action == 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
name: Close Pull Request Job
|
||||
env:
|
||||
DEPLOY_TOKEN: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
steps:
|
||||
- name: Close Pull Request
|
||||
id: closepullrequest
|
||||
if: env.DEPLOY_TOKEN != ''
|
||||
uses: Azure/static-web-apps-deploy@v1
|
||||
with:
|
||||
azure_static_web_apps_api_token: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ESP Policy-Provider Stall Detection — the no-timeout Setup/Apps wait and the co-management Sidecar gap
|
||||
description: Why a registered EnrollmentStatusTracking CSP policy provider that never sets TrackingPoliciesCreated parks the user ESP at "Apps (Identifying)" indefinitely, the pre-provisioning + ConfigMgr field case, and how the always-on esp_policy_provider_stalled tripwire (15-min wall-clock dwell, StartupEventGate cross-restart dedup) surfaces it.
|
||||
description: Why the ESP's Apps wait is keyed to the Intune-registered "Sidecar" provider by name — a foreign provider (ConfigMgr) with TrackingPoliciesCreated=1 does not satisfy it — plus the classic never-completes stall, and how the always-on two-arm esp_policy_provider_stalled tripwire (reason=provider_incomplete | sidecar_provider_missing, 15-min wall-clock dwell, StartupEventGate cross-restart dedup) surfaces both.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs
|
||||
tags:
|
||||
- agent
|
||||
@@ -9,7 +9,7 @@ tags:
|
||||
- enrollment-status-tracking
|
||||
- co-management
|
||||
- stall-detection
|
||||
timestamp: 2026-07-23T00:00:00+02:00
|
||||
timestamp: 2026-07-24T00:00:00+02:00
|
||||
---
|
||||
|
||||
# ESP Policy-Provider Stall Detection
|
||||
@@ -42,37 +42,59 @@ exactly these registry paths. The same document states that ESP policy providers
|
||||
aware of each other** and that **pre-provisioning is unsupported** with Autopilot into
|
||||
co-management.
|
||||
|
||||
## The field case
|
||||
## The field case (issue #106)
|
||||
|
||||
A tenant pre-provisions devices and installs the ConfigMgr client as a blocking app.
|
||||
Occasionally the user-phase ESP reaches "Apps (Identifying)" and never leaves it — for
|
||||
hours to days. On those devices `Device\Setup\Apps\PolicyProviders` contains **only a
|
||||
`ConfigMgr` key (no `Sidecar`)** and `TrackingPoliciesCreated` is never set. Manually
|
||||
renaming the key to `Sidecar` unblocks the ESP immediately (the IME then finds its
|
||||
provider node and reports tracking-policies-created). The mechanism is exactly the
|
||||
documented no-timeout wait; the unsupported pre-provisioning + co-management combination
|
||||
explains why only some devices are affected. Why the Sidecar registration is missing or
|
||||
displaced on those devices is not publicly documented — irrelevant for detection, since
|
||||
the signature is deterministic.
|
||||
`ConfigMgr` key (no `Sidecar`)** — and, per the customer's follow-up, **`TrackingPoliciesCreated=1`
|
||||
IS set under it**. The provider is "complete" by the CSP value contract, yet the ESP
|
||||
still waits. Manually renaming the key to `Sidecar` — which creates no tracking entries,
|
||||
only satisfies the name — unblocks the ESP instantly.
|
||||
|
||||
The naive CSP-doc reading ("any provider with TrackingPoliciesCreated=1 satisfies the
|
||||
wait") is therefore wrong for this case. What the evidence supports instead: provider
|
||||
nodes are registered **by the Intune service at enrollment**, and its ESP-policy WAP
|
||||
payload always registers `Sidecar` (ConfigMgr joins only via a co-management settings
|
||||
policy — see the payload documented by Niehaus). The ESP waits on the provider it
|
||||
expects **by name**; a foreign provider occupying the slot does not satisfy it,
|
||||
regardless of its completion values. On the affected devices the CM client (installed
|
||||
as a plain Win32 app, no co-management settings policy) evidently displaced the
|
||||
Sidecar node — consistent with Microsoft's warning that providers "aren't currently
|
||||
aware of others" and that pre-provisioning is unsupported with co-management. The exact
|
||||
CloudExperienceHost mechanism is not publicly documented; the registry signature —
|
||||
setupApps providers present, `Sidecar` absent — is deterministic either way.
|
||||
|
||||
## Detection
|
||||
|
||||
`EspPolicyProviderProbe` (fail-soft, static, `ScopedOverride` test seam — structural clone
|
||||
of `EspTrackingInfoProbe`) enumerates all providers in both kinds and scopes.
|
||||
`EspPolicyProviderStallDetector` evaluates a pure transition function on a 60-second tick
|
||||
from the always-on `EspPolicyProviderStallHost`:
|
||||
from the always-on `EspPolicyProviderStallHost` — **two arms**, same 15-minute wall-clock
|
||||
dwell, one shared event type discriminated by `reason`:
|
||||
|
||||
* A provider **continuously incomplete for ≥ 15 minutes** (wall-clock) is stalled. The
|
||||
dwell clock starts at first incomplete observation and resets on completion or
|
||||
disappearance. Missing values parse as incomplete (fail-toward-observation).
|
||||
* **Arm 1, `reason=provider_incomplete`** — a provider **continuously incomplete for
|
||||
≥ 15 minutes**. The dwell clock starts at first incomplete observation and resets on
|
||||
completion or disappearance. Missing values parse as incomplete
|
||||
(fail-toward-observation). Catches e.g. Sidecar itself never reporting.
|
||||
* **Arm 2, `reason=sidecar_provider_missing`** — **≥ 1 `Setup\Apps` provider registered
|
||||
but none named `Sidecar`**, continuously for ≥ 15 minutes. Deliberately ignores
|
||||
provider completeness (the field case had `TrackingPoliciesCreated=1`). Sidecar
|
||||
registering later — the legitimate startup ordering — clears the condition and resets
|
||||
the clock. DevicePreparation-only snapshots don't activate it.
|
||||
* When both arms cross in the same round, ONE event is emitted and
|
||||
`sidecar_provider_missing` wins as `reason` (it explains the hang regardless of the
|
||||
foreign provider's state).
|
||||
* Missing root key / empty provider list = normal early enrollment: clears all dwell
|
||||
state, never fires.
|
||||
* One `esp_policy_provider_stalled` **Warning** per newly stalled provider set
|
||||
(`ImmediateUpload`), one-shot per provider in-process, cross-restart deduped via a
|
||||
`StartupEventGate` fingerprint over the sorted currently-stalled key set — a restart
|
||||
into the same stall stays silent, a new provider joining the stall re-reports.
|
||||
* Payload: `providers[]` (full table: name, scope, kind, raw values, complete flag),
|
||||
`stalledProviders[]` with `stalledForMinutes`, `sidecarRegistered`, `dwellMinutes`.
|
||||
state (both arms), never fires.
|
||||
* One `esp_policy_provider_stalled` **Warning** per newly stalled condition set
|
||||
(`ImmediateUpload`), one-shot per condition in-process, cross-restart deduped via a
|
||||
`StartupEventGate` fingerprint over the sorted currently-active fired conditions
|
||||
(provider keys + a `sidecarMissing` sentinel) — a restart into the same stall stays
|
||||
silent, a new condition joining re-reports.
|
||||
* Payload: `reason`, `providers[]` (full table: name, scope, kind, raw values, complete
|
||||
flag), `stalledProviders[]` with `stalledForMinutes`, `sidecarMissingForMinutes`
|
||||
(arm 2 only), `sidecarRegistered`, `dwellMinutes`.
|
||||
* **Purely observational**: emitted via `InformationalEventPost` (dispatch-guard exempt
|
||||
pass-through) — never touches decision state. No config toggle, no ConfigVersion bump
|
||||
(precedent: `disk_space_low`).
|
||||
@@ -93,33 +115,37 @@ resource), so both the detector and the guardrail change ship with the agent rel
|
||||
|
||||
## Examples
|
||||
|
||||
Customer-case event (abbreviated):
|
||||
Customer-case event (abbreviated) — note the provider is "complete" by CSP values, the
|
||||
hang is carried entirely by `reason=sidecar_provider_missing`:
|
||||
|
||||
```json
|
||||
{
|
||||
"eventType": "esp_policy_provider_stalled",
|
||||
"severity": "Warning",
|
||||
"data": {
|
||||
"reason": "sidecar_provider_missing",
|
||||
"dwellMinutes": 15,
|
||||
"sidecarRegistered": false,
|
||||
"sidecarMissingForMinutes": 15.0,
|
||||
"providers": [
|
||||
{ "name": "ConfigMgr", "scope": "device", "kind": "setupApps",
|
||||
"trackingPoliciesCreated": null, "installationState": null, "complete": false }
|
||||
"trackingPoliciesCreated": 1, "installationState": null, "complete": true }
|
||||
],
|
||||
"stalledProviders": [
|
||||
{ "name": "ConfigMgr", "scope": "device", "kind": "setupApps", "stalledForMinutes": 15.0 }
|
||||
]
|
||||
"stalledProviders": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`sidecarRegistered=false` while `setupApps` providers exist is the co-management
|
||||
signature; `sidecarRegistered=true` with a stalled third-party provider generalizes the
|
||||
same contract violation.
|
||||
`reason=sidecar_provider_missing` is the co-management signature;
|
||||
`reason=provider_incomplete` with `sidecarRegistered=true` covers a stalled Sidecar or
|
||||
third-party provider — the generalized contract violation.
|
||||
|
||||
# Citations
|
||||
|
||||
* EnrollmentStatusTracking CSP — <https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp>
|
||||
* Windows Autopilot into co-management (requirements, provider-conflict warning, troubleshooting registry paths) — <https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment>
|
||||
* Troubleshoot the ESP (models app tracking exclusively via the `Sidecar` subkey) — <https://learn.microsoft.com/en-us/troubleshoot/mem/intune/device-enrollment/understand-troubleshoot-esp>
|
||||
* Niehaus: the Intune ESP-policy WAP payload registering the provider nodes (`Sidecar` always, `ConfigMgr` only with a co-management settings policy) — <https://oofhours.com/2023/08/25/run-an-sccm-task-sequence-during-autopilot/>
|
||||
* Niehaus: ESP tracks arbitrarily named providers when they are properly registered — <https://oofhours.com/2023/09/18/track-anything-using-esp/>
|
||||
* Agent code: `EspPolicyProviderProbe.cs`, `EspPolicyProviderStallDetector.cs`, `EspPolicyProviderStallHost.cs` (SystemSignals / Orchestration.Hosts)
|
||||
* Related: [MDM Reboot Coalescing](mdm-reboot-coalescing.md), [Gather-Rule Guardrails](../rules/gather-rule-guardrails.md)
|
||||
|
||||
@@ -37,6 +37,21 @@ Fail-soft contract like `OobeStateReader`: never throws; failures map to status
|
||||
(`not_uefi` = legacy BIOS via `ERROR_INVALID_FUNCTION`, `variable_not_found`,
|
||||
`privilege_denied`, `error_<code>`).
|
||||
|
||||
This is the repo's only privilege-adjustment code, so it also carries the ownership rules
|
||||
any future one should copy:
|
||||
|
||||
* The process token lives in a `SafeHandle` assigned by the P/Invoke marshaler, so the CLR
|
||||
owns it from the moment `OpenProcessToken` returns — the critical finalizer releases it
|
||||
even if `Dispose` never runs (async exception before the instance reaches its `using`).
|
||||
* The restore obligation is recorded immediately after the successful `AdjustTokenPrivileges`,
|
||||
and only when the privilege was not *already* enabled; nothing that can throw sits between
|
||||
the adjustment and that assignment. `Dispose` is idempotent, swallows a failed restore, and
|
||||
releases the handle in a `finally`.
|
||||
* The privilege is adjusted on the **process** token, so enable → read → restore is a
|
||||
process-wide side effect and runs under a static gate. Without it a concurrent reader would
|
||||
see "already enabled", correctly skip its own restore, and then lose the privilege mid-read
|
||||
when the first reader disposes — a spurious `privilege_denied` on a machine that holds it.
|
||||
|
||||
Fields land on the existing `secureboot_status` event (once per agent start,
|
||||
restart-deduped by the StartupEventGate): `uefiFirmwareReadStatus`, per-cert booleans
|
||||
(`uefiDbHasWindowsUefiCa2023`, `uefiDbHasWindowsProductionPca2011`,
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Device Journeys & First-Time-Right (F2)
|
||||
description: How a device's terminal enrollment attempts are grouped into journeys (device key, junk-serial exclusion, first-success end, 30-day gap), where the DeviceHistories chain and the daily FTR aggregates are computed, stored and pruned, and which surfaces serve them.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/DeviceJourneyCalculator.cs
|
||||
tags:
|
||||
- backend
|
||||
- metrics
|
||||
- device-history
|
||||
- first-time-right
|
||||
- insights
|
||||
timestamp: 2026-07-27T12:00:00+02:00
|
||||
---
|
||||
|
||||
# Device Journeys & First-Time-Right (F2)
|
||||
|
||||
Answers "how many devices enrolled right on the first try?" and "show me every attempt
|
||||
this device made". Session success rate hides wipe-and-retry loops — every retry that
|
||||
eventually succeeds counts as a success; First-Time-Right (FTR) exposes the real pain.
|
||||
All grouping semantics live server-side in `DeviceJourneyCalculator` (JourneyVersion 1);
|
||||
no consumer re-derives them.
|
||||
|
||||
# Schema
|
||||
|
||||
## Device key
|
||||
|
||||
`(TenantId, normalized serial)` — normalization is trim + lower-case. Junk/placeholder
|
||||
serials are excluded entirely (never a chain, never in FTR, disclosed as a per-day
|
||||
exclusion count): `System Serial Number`, `To Be Filled By O.E.M.`, `Default string`,
|
||||
`0`, `None`, `INVALID`, `Unknown` (the agent's WMI-failure sentinel), empty, or shorter
|
||||
than 4 characters. A motherboard swap changes the serial and therefore the device key;
|
||||
re-enrollment under a different tenant is a separate key by design.
|
||||
|
||||
## Journey
|
||||
|
||||
Terminal sessions of a device key ordered by StartedAt. The terminal set is
|
||||
**Succeeded / Failed / Incomplete** — `Pending` (a sealed WhiteGlove part 1),
|
||||
`AwaitingUser`, `Stalled` and `InProgress` are OPEN sessions and never attempts.
|
||||
|
||||
* A journey **ends** with the first terminal success; the next session starts a new
|
||||
journey (redeployment).
|
||||
* A gap of **more than 30 days** since the previous attempt's terminal moment
|
||||
(`CompletedAt`, fallback StartedAt) starts a new journey even without a success
|
||||
(device shelved/repurposed). Constant, not a setting — production gaps: median 4.9 h,
|
||||
92 % < 7 d.
|
||||
* WhiteGlove part 1 + part 2 share ONE session row, so a completed WG enrollment is ONE
|
||||
attempt; a WG device still waiting for its user session leaves the journey open —
|
||||
never a failed attempt.
|
||||
* **Attempt count** = terminal sessions in the journey (`Failed` and `Incomplete` are
|
||||
non-successful attempts).
|
||||
* **FTR rate** = completed journeys with attempt count 1 ÷ all completed journeys.
|
||||
Open and gap-abandoned journeys never enter numerator or denominator.
|
||||
|
||||
Durations shown for attempts are the sessions' authoritative `DurationSeconds` verbatim
|
||||
(WhiteGlove pause excluded; `Incomplete` deliberately has none) — never
|
||||
`CompletedAt − StartedAt`, which is later in ~25 % of terminal sessions.
|
||||
|
||||
## Persistence & compute
|
||||
|
||||
* `DeviceHistories` (PK = TenantId, RK = percent-encoded normalized serial): the chain of
|
||||
the device's terminal session refs `{sessionId, startedAt, completedAt, status,
|
||||
enrollmentType, isPreProvisioned, durationSeconds, adminMarked}` capped at the 20 most
|
||||
recent, plus derived `CurrentJourneyAttempts` (the LAST journey's attempts — the
|
||||
banner's "Attempt N"), `JourneyCount` (chain-scoped) and `JourneyVersion`. Written
|
||||
inline at the session-terminal single-writer seam
|
||||
(`TableSessionRepository.UpdateSessionStatusAsync`, all three terminal statuses,
|
||||
DeletionState-guarded) so the session banner is fresh; healed by the maintenance sweep.
|
||||
Deliberately NOT part of the per-session deletion manifest — the row aggregates many
|
||||
sessions. Wiped on tenant offboarding.
|
||||
* Sweep (`SweepDeviceJourneysAsync`, 30-day rolling window, 2h tick): (1) drops chain
|
||||
refs of deleted sessions **tombstone-driven** — every session deletion (cascade and
|
||||
retention both run through the cascade worker) leaves a `SessionTombstones` marker for
|
||||
~7 days, far longer than the sweep cadence, and even an expired-but-unpruned marker is
|
||||
valid deletion evidence; a chain left empty deletes its row. (2) Merges the window's
|
||||
terminal sessions into their chains (backfill, inline-miss healing, re-terminal
|
||||
reclassifications). (3) Recomputes the daily FTR aggregates idempotently.
|
||||
* `DeviceJourneyAggregates` (PK = TenantId or `global`, RK = `{yyyy-MM-dd}`): per day —
|
||||
completed journeys, first-time-right count, attempt histogram, junk-serial exclusion
|
||||
count, `JourneyVersion`. A journey buckets on the StartedAt date of its completing
|
||||
success session (the platform's StartedAt-date convention; late terminals converge via
|
||||
the sweep). All counts are **additive**, so a window rate is the ratio of summed daily
|
||||
rows — there is no rolling-window row (unlike the median-based time-attribution
|
||||
aggregates, which cannot merge). 180-day retention. Rows exist below the ≥20 UI gate
|
||||
on purpose (the UI needs the n for "insufficient data (n=…)").
|
||||
|
||||
# Surfaces
|
||||
|
||||
* `GET api/metrics/device-history?serialNumber=&sessionId=` (MemberRead + tenant query
|
||||
param — the same route serves the web banner and MCP; a Global Admin passes `tenantId`)
|
||||
— the chain plus, with `sessionId`, that session's server-computed attempt number
|
||||
(live sessions get their would-be position via a virtual non-successful attempt, so the
|
||||
redeploy and gap rules apply). `history: null` = no recorded history.
|
||||
* `GET api/metrics/device-journeys?days=` (MemberRead) /
|
||||
`GET api/global/metrics/device-journeys?days=&tenantId=` (GlobalReadOrAdmin) —
|
||||
`{ windowDays, totals, daily, repeatDevices }`. The days selector is honored because
|
||||
counts are additive. Without `tenantId` the global variant serves the cross-tenant
|
||||
aggregate rows and `repeatDevices: null` — a per-device list would require scanning
|
||||
every tenant's partition.
|
||||
* Repeat devices (violator list, capped 10): devices whose LAST journey took ≥2 attempts
|
||||
and whose newest terminal session falls inside the window, ordered by attempts then
|
||||
recency; the failure reason comes from the newest failed attempt (bounded point-reads).
|
||||
* Web: "Attempt N for this device · View history" banner on session detail (expandable
|
||||
chain); "First-time-right" section on Fleet Health (rate + weekly trend + attempt
|
||||
histogram + repeat devices). MCP: `get_device_history` (serial mode / fleet FTR mode).
|
||||
|
||||
# Examples
|
||||
|
||||
A device fails ESP twice and succeeds on the third try within one week: one journey,
|
||||
three attempts, completed — counts once in the FTR denominator, not in the numerator,
|
||||
and the histogram's bucket `3` gains one. The same device redeployed a month later and
|
||||
succeeding immediately starts journey 2 with attempt count 1 — first-time-right. A
|
||||
device with `serialNumber = "Unknown"` (WMI failure) never gets a chain; its terminal
|
||||
sessions surface only as that day's disclosed exclusion count.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/DeviceJourneyCalculator.cs` — device key, junk list, grouping, attempt numbers (JourneyVersion 1).
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.DeviceHistory.cs` — persistence + inline seam entry.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.DeviceJourneys.cs` — sweep, tombstone pruning, daily aggregation.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetDeviceJourneyFunctions.cs` — read surfaces.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/DeviceJourneyAndFtrTests.cs` — table-driven grouping/aggregation/attempt-number tests.
|
||||
* `tasks/insights-expansion-spec.md` — F2 specification and source-data audit (§0.5).
|
||||
@@ -0,0 +1,112 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Rule Regression Radar (F3)
|
||||
description: How the radar detects analyze rules whose hit rate suddenly rose (7-day window vs 28-day baseline, Wilson-separated), which suppressions keep it quiet, how alert episodes live in the notification tracker, and which surfaces show them.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/RuleRegressionRadar.cs
|
||||
tags:
|
||||
- backend
|
||||
- rules
|
||||
- regression-radar
|
||||
- insights
|
||||
timestamp: 2026-07-27T16:00:00+02:00
|
||||
---
|
||||
|
||||
# Rule Regression Radar (F3)
|
||||
|
||||
Detects "this analyze rule suddenly fires much more often" before helpdesk volume does, and
|
||||
says what the spike correlates with — with honest statistics and honest wording. Analyze
|
||||
rules only in v1: their daily stats are audit-verified session-deduplicated
|
||||
(`FireCount` ≈ distinct sessions with a hit, `SessionsEvaluated` the same-source
|
||||
denominator). Gather rules are excluded until their per-batch stats dedup is fixed —
|
||||
never silently included with bad math.
|
||||
|
||||
# Schema
|
||||
|
||||
## Signal & detection
|
||||
|
||||
Per (tenant, ruleId), daily hit rate = `FireCount ÷ SessionsEvaluated` from the
|
||||
`RuleStats` rows — RATES, not counts (absolute counts spike with every rollout wave).
|
||||
A regression fires only when ALL hold for the trailing 7-day window vs. the prior
|
||||
28-day baseline:
|
||||
|
||||
1. window has ≥ 5 hit sessions AND ≥ 20 evaluated sessions;
|
||||
2. window rate ≥ 2× baseline rate;
|
||||
3. the Wilson 95 % intervals are disjoint in the increase direction
|
||||
(`MetricsMath.WilsonInterval` / `RateIncreaseSeparated` — deterministic, unit-pinned),
|
||||
so small-n noise never alerts.
|
||||
|
||||
## Suppression (false positives are trust damage)
|
||||
|
||||
* Rule entity missing (deleted) — never alerts.
|
||||
* Rule younger than baseline + window (`CreatedAt` within 35 days — grace period for new
|
||||
rules; the entity timestamp is the honest gate, a stats-row count would fail on
|
||||
low-volume tenants whose quiet days simply have no rows).
|
||||
* Rule edited inside the window (`UpdatedAt` ≥ window start) — edits legitimately change
|
||||
hit rates.
|
||||
* Empty baseline denominator (rule disabled throughout the baseline).
|
||||
* Platform kill switch: app setting `RuleRegressionRadarDisabled=true` skips the radar
|
||||
(fail-open — it only notifies, never mutates data).
|
||||
|
||||
## Alert episodes (tracker keyspace)
|
||||
|
||||
An episode lives as `ruleregression|{ruleId}` in the notification-tracker table
|
||||
(the `tpmpss|` pattern): the row IS the dedup (bell + ops event fire exactly once per
|
||||
episode), the badge state and the `regressions[]` payload — window/baseline counts,
|
||||
rates, lift (null for a zero-baseline "new signal" — no finite lift is invented), window
|
||||
dates, and the dimension concentration captured at first fire. Numbers refresh on every
|
||||
radar pass while the episode stays active; `FirstNotifiedAt` never moves. The episode
|
||||
closes (badge clears, dedup re-arms) when the window rate falls back under 1.5× baseline
|
||||
or fires stop entirely; the tracker's 30-day retention sweep also re-arms, so a
|
||||
month-old still-burning regression rings once more by design.
|
||||
|
||||
## Dimension correlation (on fire only)
|
||||
|
||||
Hit sessions (RuleResults scan: tenant partition range + RowKey = ruleId + DetectedAt in
|
||||
window, intersected with the window's sessions) are compared against all window sessions
|
||||
across `osBuild`, `model` (manufacturer + model), `agentVersion` and `imeVersion`. The
|
||||
top value by lift is reported with BOTH shares — wording contract: "X % of affected
|
||||
sessions are on … vs Y % of all sessions — correlated, not necessarily causal". No value
|
||||
with ≥ 5 hit sessions at ≥ 2× lift → "no clear dimension concentration"; a correlation
|
||||
failure also yields no claim. App-version correlation for `ANALYZE-APP-*` rules is a
|
||||
follow-up (needs rule→app matching semantics), tracked with the gather-rule fix.
|
||||
|
||||
## Compute & alert path
|
||||
|
||||
Runs in the daily maintenance pass right after the rule-stats aggregation
|
||||
(`RunRuleRegressionRadarAsync`), anchored on YESTERDAY — whole days only, a partial
|
||||
today would understate the window rate. One 35-day stats read covers all tenants;
|
||||
evaluation is the pure `RuleRegressionRadar` core. On a NEW episode:
|
||||
|
||||
1. `RuleFrequencyRegression` ops event (category Tenant, Warning) with the full numbers
|
||||
— dual-registered in the web `OPS_EVENT_TYPES` catalog for alert-rule routing.
|
||||
2. Tenant bell notification (type `rule_frequency_regression`, Admin audience) with the
|
||||
same numbers and a deep link to the rule's card (`/analyze-rules#rule-card-{ruleId}`).
|
||||
A sessions-list pre-filtered to rule hits needs a rule filter the session search does
|
||||
not have yet — follow-up alongside the gather-rule work.
|
||||
3. v1 channel: bell + ops event only. Provider fan-out (webhook/Telegram) follows once
|
||||
alert quality is proven — a noisy first version burns trust permanently.
|
||||
|
||||
# Surfaces
|
||||
|
||||
* `GET api/metrics/rule-stats` (and the global variant with `tenantId`) carries
|
||||
`regressions[]` — the active episodes. MCP `get_rule_stats` inherits the block.
|
||||
* Analyze-rules page: per-rule 30-day fire-count sparkline (densified from the `trend`
|
||||
rows the endpoint already served) and a red "↑ Regression" badge while an episode is
|
||||
active, with the full numbers in the tooltip.
|
||||
|
||||
# Examples
|
||||
|
||||
A rule firing on 2 % of sessions for a month jumps to 15 % over a week (15/100 vs
|
||||
10/500): lift 7.5×, Wilson-separated → one bell, one ops event, badge until the rate
|
||||
falls back under 3 %. The same jump on 5/20 sessions with a 12 % baseline stays silent —
|
||||
the intervals overlap, so the spike is not statistically real yet. A rule that never
|
||||
fired in the baseline and now hits 15 % alerts as a "new signal" with no lift figure.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/RuleRegressionRadar.cs` — gates, suppression, re-arm, dimension concentration.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs` — Wilson interval + separation primitive.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.RuleRegression.cs` — sweep, episode reconcile, bell/ops wiring.
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableHardwareRejectionNotificationTracker.cs` — episode keyspace.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/RuleRegressionRadarTests.cs` — hand-computed Wilson vectors + every suppression branch.
|
||||
* `tasks/insights-expansion-spec.md` — F3 specification and source-data audit (§0.5).
|
||||
@@ -0,0 +1,102 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ServerActions & On-Demand Diagnostics Collection
|
||||
description: The queue-on-session, deliver-on-ingest ServerAction channel (terminate_session / rotate_config / request_diagnostics), the operator-tier authorization split, the portal's Collect Logs flow including the admin quick-config path, and why the offboarding cascade sweeps the hosted diagnostics prefix.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Functions/Sessions/QueueSessionActionFunction.cs
|
||||
tags:
|
||||
- backend
|
||||
- agent
|
||||
- web
|
||||
- server-actions
|
||||
- diagnostics
|
||||
- authorization
|
||||
timestamp: 2026-07-24T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
**Channel.** A `ServerAction` (`Type`, `Reason`, `RuleId`, `Params`, `QueuedAt`) is queued onto
|
||||
`Sessions.PendingActionsJson` by server-side writers — the admin endpoint
|
||||
`POST /api/sessions/{sessionId}/actions`, the RuleEngine, maintenance — and delivered
|
||||
piggy-backed on the agent's next ingest response. No polling channel exists; delivery latency
|
||||
equals the agent's telemetry cadence (seconds to ~1 minute during enrollment). Delivery is
|
||||
at-least-once: the agent's `ServerActionDispatcher` dedups on `Type|RuleId|QueuedAt` and
|
||||
processes a batch sequentially in queue order. Unknown types are logged and skipped, so new
|
||||
types roll out server-first. Every action emits `server_action_received` and then
|
||||
`server_action_executed` or `server_action_failed` into the session timeline.
|
||||
|
||||
**Authorization (two layers).** The route sits on `EndpointPolicy.TenantAdminOrOperator`
|
||||
(tenant Admin, tenant Operator, or Global Admin — no Viewer, no Global Reader: it is a write
|
||||
tier). Because the route serves multiple action kinds, `QueueSessionActionFunction`
|
||||
re-gates per type via `IsTypeAllowedForCaller`: Operators may queue only
|
||||
`request_diagnostics`; `terminate_session` and `rotate_config` require
|
||||
`RequestContext.IsTenantAdmin || IsGlobalAdmin` (403 otherwise). Any route on this tier with
|
||||
a multi-kind body MUST follow this same in-function re-gating pattern.
|
||||
|
||||
**On-demand collection (`request_diagnostics`).** The dispatcher calls
|
||||
`DiagnosticsPackageService.CreateAndUploadAsync(enrollmentSucceeded: false, suffix:
|
||||
"server-requested")` — producing `AgentDiagnostics-{sessionId}-{timestamp}-server-requested.zip`.
|
||||
Passing `enrollmentSucceeded: false` is what makes on-demand collection work under
|
||||
`DiagnosticsUploadMode=OnFailure`; only `Off` (or an unconfigured destination) blocks it,
|
||||
surfacing as `server_action_failed`. The `diagnostics_uploaded` event stamps
|
||||
`DiagnosticsBlobName` + `Destination` onto the Sessions row (latest wins — an on-demand name
|
||||
is later replaced by the terminal package), and the SignalR `sessionUpdate` delta carries the
|
||||
field, so the portal's Download button flips live.
|
||||
|
||||
**Portal flow (Collect Logs, session detail header).** The button is always rendered; the
|
||||
disabled states carry tooltips (inactive session, cross-tenant view, missing role, diagnostics
|
||||
not configured) as a deliberate nudge. Members below Admin cannot read the tenant config, so
|
||||
the member-readable feature-flags endpoint exposes `diagnosticsUploadConfigured`
|
||||
(`mode != Off && (customer SAS present || destination Hosted)`). Progress tracking is
|
||||
sequence-based (backend-assigned, monotonic), never timestamp-based — agent clock skew and
|
||||
at-least-once redelivery of older actions cannot produce false matches
|
||||
(`collectLogsLogic.ts`). Cross-tenant views are gated off because `TenantHelper.GetTenantId`
|
||||
resolves the tenant strictly from the caller's JWT.
|
||||
|
||||
**Admin quick-config.** For an unconfigured tenant, a Tenant Admin's click opens a dialog
|
||||
that switches the tenant to `Destination=Hosted`, `Mode=OnFailure` (GET full config → patch
|
||||
exactly those two fields → PUT verbatim), then queues `rotate_config` **before**
|
||||
`request_diagnostics` — the dispatcher's sequential ordering guarantees the agent refetches
|
||||
the now-enabled config (the `RemoteConfigMerger` applies diagnostics fields live, remote wins)
|
||||
before building the package. This works against the already-deployed agent fleet; a
|
||||
"one-time override" variant would not, because the `mode=Off` check is client-side in
|
||||
`CreateAndUploadAsync` and the deployed handler reads no params.
|
||||
|
||||
**Offboarding cascade.** Latest-wins on `DiagnosticsBlobName` means a session can leave more
|
||||
hosted blobs behind than the deletion manifest references. The cascade's Hosted branch
|
||||
therefore sweeps the canonical prefix `{tenantId}/AgentDiagnostics-{sessionId}-`
|
||||
(`HostedDiagnosticsBlobService.DeleteBySessionPrefixAsync`) plus the manifest-named blob as
|
||||
belt-and-braces; a non-GUID legacy `SessionId` degrades to the single-name delete instead of
|
||||
poisoning the cascade. The CustomerSas branch is unchanged (delete only with the `d`
|
||||
permission; the customer's lifecycle rules stay authoritative).
|
||||
|
||||
# Examples
|
||||
|
||||
Queue an on-demand collection (Admin or Operator, own tenant):
|
||||
|
||||
```
|
||||
POST /api/sessions/{sessionId}/actions
|
||||
{ "type": "request_diagnostics", "reason": "On-demand log collection from portal" }
|
||||
→ 202 { "success": true, "queuedAt": "..." }
|
||||
```
|
||||
|
||||
Operator queueing an admin-only type:
|
||||
|
||||
```
|
||||
POST /api/sessions/{sessionId}/actions
|
||||
{ "type": "terminate_session" }
|
||||
→ 403 { "message": "Action type 'terminate_session' requires the Tenant Admin role" }
|
||||
```
|
||||
|
||||
Timeline for a successful round trip: `server_action_received` → `server_action_executed`
|
||||
(with `blobName`) → `diagnostics_uploaded` → Sessions row + SignalR delta carry the new
|
||||
`DiagnosticsBlobName`.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Sessions/QueueSessionActionFunction.cs` — allowed types, per-type caller gate.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs` — `TenantAdminOrOperator` tier definition.
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/ServerActionDispatcher.cs` — dedup, sequential dispatch, telemetry contract.
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs` — mode gates, archive build, suffix.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Diagnostics/DiagnosticsBlobCascadeDeleter.cs` — hosted prefix sweep.
|
||||
* `src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/collectLogsLogic.ts` — button gating matrix, sequence-based progress evaluation.
|
||||
@@ -0,0 +1,124 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Enrollment Time Attribution (F1)
|
||||
description: How a terminal session's wall clock is partitioned into named segments with an explicit unattributed remainder, how ESP-blocking apps and reboots are attributed, and where breakdowns and fleet aggregates are computed, stored and served.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs
|
||||
tags:
|
||||
- backend
|
||||
- metrics
|
||||
- time-attribution
|
||||
- insights
|
||||
timestamp: 2026-07-26T20:00:00+02:00
|
||||
---
|
||||
|
||||
# Enrollment Time Attribution (F1)
|
||||
|
||||
Answers "where did the 47 minutes go?" per session and per fleet. Every number is
|
||||
traceable to observed events; a missing signal produces "unknown" or an explicit
|
||||
unattributed remainder — never a guess, and the partition is never normalized to 100 %.
|
||||
|
||||
# Schema
|
||||
|
||||
## Invariant
|
||||
|
||||
For every computed breakdown:
|
||||
|
||||
```
|
||||
sum(span seconds) + UnattributedSeconds == WallClockSeconds == session DurationSeconds
|
||||
```
|
||||
|
||||
`DurationSeconds` is the authoritative wall clock — for WhiteGlove it is part 1 + part 2
|
||||
WITHOUT the pause, and re-terminal stamps move `CompletedAt`, so `CompletedAt − StartedAt`
|
||||
is never used (it diverges in ~25 % of terminal sessions). Span seconds are floored;
|
||||
sub-second dust lands in the unattributed remainder.
|
||||
|
||||
## Observation windows
|
||||
|
||||
Windows are end-anchored on authoritative timestamps and constructed so their lengths sum
|
||||
exactly to `DurationSeconds`:
|
||||
|
||||
* Non-WhiteGlove: one window `[CompletedAt − DurationSeconds, CompletedAt]`.
|
||||
* WhiteGlove (ONE session row, two windows): part 2 = `[ResumedAt, CompletedAt]`; part 1 =
|
||||
`[part1End − (DurationSeconds − part2), part1End]` where `part1End` is anchored on the
|
||||
`whiteglove_part1_complete` event. The pause between the windows is never attributed.
|
||||
Fallback anchors set the `WhiteGloveAnchorsIncomplete` quality flag.
|
||||
|
||||
## Segments
|
||||
|
||||
Spans are sliced at phase-declaration events (only declaration events carry
|
||||
`Phase != Unknown`) plus `desktop_arrived`, in canonical Sequence order; anchors whose
|
||||
timestamp runs backward are dropped (`ClockSkewDropped`).
|
||||
|
||||
| EnrollmentPhase | Segment key |
|
||||
|---|---|
|
||||
| Start / DevicePreparation / DeviceSetup | `device_prep` |
|
||||
| AppsDevice | `esp_apps` |
|
||||
| AccountSetup / FinalizingSetup | `identity_hello` |
|
||||
| AppsUser | `user_esp` |
|
||||
| Complete, `desktop_arrived` event | `desktop_handoff` |
|
||||
| Failed | attribution ends — tail stays unattributed |
|
||||
|
||||
## ESP-blocking apps (positive evidence only)
|
||||
|
||||
Per-app intervals use EVENT timestamps (first started/download event → LAST terminal
|
||||
event, covering IME retries) — never the agent payload timing, which freezes at the first
|
||||
terminal transition. Blocking membership joins the app id against the session's latest
|
||||
list-carrying `esp_config_detected` emission: listed ⇒ blocking; absent ⇒ unknown, never
|
||||
false (`BlockingSetUnknown` / `BlockingSetTruncated` flags). Critical-path occupancy =
|
||||
overlap-merged union of blocking intervals clipped to the `esp_apps` spans;
|
||||
`EspAppsOccupancySeconds` is null (unknown) when no lists were observed.
|
||||
|
||||
The fleet what-if bound per app X is `max(0, cpEnd − cpEndWithoutX)` (latest interval end
|
||||
with vs. without X; the idle gap before X counts — the ESP also waits for X to start).
|
||||
It is an upper bound by construction; every consumer says "up to", never "you will save".
|
||||
|
||||
## Reboots
|
||||
|
||||
`system_reboot_detected` is detection-time-stamped by the NEXT agent run; the real boot
|
||||
moment is its `lastBootUtc` payload. Reboot spans are therefore the event-stream gap
|
||||
bracketing `lastBootUtc`, deduped per boot, clipped to the observation windows, and carried
|
||||
as a cross-cutting annotation (they overlap the segment they started in — never a slice of
|
||||
the partition).
|
||||
|
||||
## Persistence & compute
|
||||
|
||||
* `SessionTimeBreakdowns` (PK = TenantId, RK = SessionId): spans, reboot spans, blocking
|
||||
intervals (top 20 + uncapped count), quality flags, `AttributionVersion`. Written once at
|
||||
the session-terminal single-writer seam (`TableSessionRepository.UpdateSessionStatusAsync`,
|
||||
Succeeded/Failed only — Incomplete deliberately has no duration). Deleted with its session
|
||||
(deletion-manifest step) and on tenant offboarding.
|
||||
* Self-healing 30-day maintenance sweep (2h tick): backfills missing breakdowns, folds
|
||||
late-terminating sessions into their StartedAt-date aggregates, and enforces version
|
||||
purity — a stale-version row whose events aged out counts as missing, never mixed.
|
||||
* `TimeAttributionAggregates` (PK = TenantId or `global`, RK = `{yyyy-MM-dd}|{class}` and
|
||||
`rolling30|{class}`): per enrollment class (`user_driven`, `whiteglove`, `self_deploying`,
|
||||
`device_preparation` — never mixed) median/p75/p90 per segment over the fixed six-segment
|
||||
stack (incl. `unattributed`; a session without a segment contributes 0), plus top blocking
|
||||
apps (≥5 sessions per app row). Clean/flagged/missing counts are part of every row; rows
|
||||
exist below the ≥20 UI gate on purpose (the UI needs the n for "insufficient data (n=…)").
|
||||
Daily rows retain 180 days; `rolling30` rows are refreshed whole (a median of per-day
|
||||
medians is not the range median — the rolling rows are the honest range statistics).
|
||||
|
||||
## Surfaces
|
||||
|
||||
* `GET api/sessions/{sessionId}/time-attribution` (MemberRead + tenant query param) —
|
||||
breakdown row or `breakdown: null` (pre-feature / non-terminal / Incomplete = no lane).
|
||||
* `GET api/metrics/time-attribution` (MemberRead) / `GET api/global/metrics/time-attribution`
|
||||
(GlobalReadOrAdmin, optional `tenantId`) — `{ windowDays, classes (rolling), daily }`.
|
||||
* Web: attribution lane under the session-detail phase timeline; "Time attribution" section
|
||||
on Fleet Health. MCP: `get_time_attribution` (sessionId mode / fleet mode).
|
||||
|
||||
# Examples
|
||||
|
||||
A 30-minute user-driven session: `device_prep` 5m → `esp_apps` 10m (occupancy 4m by one
|
||||
blocking app) → `identity_hello` 8m → `user_esp` 5m → `desktop_handoff` 2m, unattributed 0.
|
||||
A WhiteGlove session with a 2-day pause reports wall clock = part1 + part2 only; the resume
|
||||
gap before the first part-2 phase declaration is unattributed, not guessed.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs` — calculator + semantics (AttributionVersion 1).
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.TimeAttribution.cs` — persistence.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs` — sweep + aggregation.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/TimeAttributionCalculatorTests.cs` — golden fixtures pinning the invariant.
|
||||
* `tasks/insights-expansion-spec.md` — F1 specification and source-data audit (§0.5).
|
||||
+9
-1
@@ -22,7 +22,7 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
* [Agent Endpoint Migration](agent/endpoint-migration.md) - How the backend re-homes agents to a new API base URL via the config channel (backend move or per-tenant region move): live-fetch-only, allowlist-validated on both sides, one hop, kill wins.
|
||||
* [MDM Reboot Coalescing](agent/mdm-reboot-coalescing.md) - Attributing the mid-ESP coalesced reboot ("second sign-in") to the device-assigned policy URIs that forced it: aggregated DM-Enterprise 2800 observations (neutral, flush-time watermark) + the ANALYZE-ESP-005 advisory gated on an actually-observed reboot.
|
||||
* [Secure Boot CA-2023 Detection](agent/secureboot-ca2023-detection.md) - Why ANALYZE-SEC-001 v3 reads the UEFI db/KEK variables directly instead of trusting the SecureBoot\Servicing registry, and how the one-sided uefiCa2023FirmwareConfirmed marker plus a not_exists precondition give absence-tolerant suppression across old and new agents.
|
||||
* [ESP Policy-Provider Stall Detection](agent/esp-policy-provider-stall.md) - The EnrollmentStatusTracking CSP's no-timeout Setup/Apps wait: a registered provider that never sets TrackingPoliciesCreated (co-management "ConfigMgr" without the IME's "Sidecar") parks the user ESP at "Apps (Identifying)" for days; the always-on esp_policy_provider_stalled tripwire and the SOFTWARE\Microsoft\Windows\Autopilot gather-rule allowlist entry that surface it.
|
||||
* [ESP Policy-Provider Stall Detection](agent/esp-policy-provider-stall.md) - The EnrollmentStatusTracking CSP's no-timeout Setup/Apps wait is keyed to the Intune-registered "Sidecar" provider by NAME: co-management's "ConfigMgr" parks the user ESP at "Apps (Identifying)" for days even with TrackingPoliciesCreated=1 (issue #106); the always-on two-arm esp_policy_provider_stalled tripwire (provider_incomplete | sidecar_provider_missing) and the SOFTWARE\Microsoft\Windows\Autopilot gather-rule allowlist entry that surface it.
|
||||
|
||||
# Rules
|
||||
|
||||
@@ -32,11 +32,19 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
# Backend
|
||||
|
||||
* [Business Timestamps](backend/business-timestamps.md) - Why the Azure Tables system Timestamp is never authoritative (migrations reset it), and the OccurredUtc + RowKey-decode compensation for AuditLogs/OpsEvents/Events after the 2026-07-18 migration.
|
||||
* [Device Journeys & First-Time-Right](backend/device-journeys.md) - How a device's terminal enrollment attempts are grouped into journeys (trim+casefold device key with junk-serial exclusion, first-success end, 30-day gap, WhiteGlove = one attempt), the DeviceHistories chain with its tombstone-driven deleted-session pruning, the additive daily FTR aggregates, and the banner/fleet/MCP surfaces.
|
||||
* [Enrollment Time Attribution](backend/time-attribution.md) - How a terminal session's authoritative wall clock (DurationSeconds, WhiteGlove pause excluded) is partitioned into named segments with an explicit unattributed remainder, ESP-blocking apps as positive evidence with "up to" what-if bounds, reboot spans from event gaps, and the breakdown/aggregate tables the sweep maintains.
|
||||
* [Rule Regression Radar](backend/rule-regression-radar.md) - How analyze-rule hit-rate regressions are detected (7-day window ≥2× the 28-day baseline with disjoint Wilson intervals), which suppressions keep the radar quiet (grace period, edit-in-window, kill switch), how alert episodes live in the ruleregression tracker keyspace with re-arm semantics, and the on-fire dimension correlation with its "correlated, not necessarily causal" wording contract.
|
||||
* [ServerActions & On-Demand Diagnostics](backend/server-actions-on-demand-diagnostics.md) - The queue-on-session, deliver-on-ingest action channel, the TenantAdminOrOperator tier with in-function per-type re-gating, the portal's Collect Logs flow (quick-config orders rotate_config before request_diagnostics), and the hosted-prefix sweep in the offboarding cascade.
|
||||
|
||||
# MCP Server
|
||||
|
||||
* [MCP Docs Corpus](mcp/docs-corpus.md) - How the published customer documentation becomes the `search_docs` corpus: CRLF-safe heading chunking, build-time embedding as a third precomputed section, and why a docs edit needs an MCP redeploy.
|
||||
|
||||
# Web Portal
|
||||
|
||||
* [Portal Navigation Prefetch & the SWA Runtime Bottleneck](web/portal-navigation-prefetch.md) - Why every navigation Link sets `prefetch={false}`: the App Router's viewport prefetch fans out one RSC request per visible link against an HTTP/1.1 Static Web Apps runtime, and its latency tail starves the six connections that also serve the document and the route chunks. Includes where browser telemetry actually lives.
|
||||
|
||||
# Security & Identity
|
||||
|
||||
* [MCP OAuth Flow](mcp-oauth-flow.md) - Who authenticates where when connecting an AI client to the MCP server; two identities, three parties plus Entra ID.
|
||||
|
||||
+21
-1
@@ -1,6 +1,22 @@
|
||||
# Log
|
||||
|
||||
## 2026-07-23
|
||||
## 2026-07-27
|
||||
|
||||
* **Creation**: Added `web/portal-navigation-prefetch.md` (and a `Web Portal` section to the index) — root cause of the minute-long portal freezes. `GlobalSidebar` and `Navbar` render the whole navigation on every page, so the App Router's default viewport prefetch issued one `?_rsc=` request per visible link against `portal.autopilotmonitor.com` (~10,400 in 14 days). That origin serves the document, the route chunks and the RSC payloads over **HTTP/1.1**, so the six-connection-per-origin cap applies; when the Static Web Apps managed runtime queued the prefetches (p50 91 ms, p95 896 ms, but max 234 s / 21 samples over 30 s), the in-flight prefetches held every socket and the navigation the user wanted could not get one. Measured on the 2026-07-27 17:25 CEST incident: `/admin/presence?_rsc=` 151 s, three siblings finishing within 340 ms of each other at ~37.5 s, all `200`, while the backend API answered in 1.6–2.2 s and `browserTimings` showed `networkDuration` 16 ms against `processingDuration` 225 s. Every navigation Link now sets `prefetch={false}` — these routes are client components behind `ProtectedRoute` that fetch their own data, so the prefetched RSC payload carried no benefit. The doc also records that browser telemetry lives in a **separate** App Insights resource (`AUTOPILOT_MONITOR_WEB_APPINSIGHTS_ID`) that the MCP `query_backend_logs` tool cannot reach, which is why the freezes left no trace in the backend logs.
|
||||
|
||||
* **Creation**: Added `backend/rule-regression-radar.md` — F3 regression radar (analyze rules only; gather rules wait for their per-batch stats dedup fix): per (tenant, rule) daily hit rate from `RuleStats`, firing only when the trailing 7-day window has ≥5 hit sessions and ≥20 evaluated sessions, the rate is ≥2× the prior 28-day baseline AND the Wilson 95 % intervals are disjoint (deterministic `MetricsMath` primitive with hand-computed test vectors). Suppressed for deleted rules, rules younger than baseline+window (`CreatedAt` grace), rules edited inside the window (`UpdatedAt`), empty baselines, and the `RuleRegressionRadarDisabled` app-setting kill switch. Alert episodes live as `ruleregression|{ruleId}` tracker rows (one bell + one `RuleFrequencyRegression` ops event per episode; numbers refreshed while active; re-armed under 1.5× baseline, on fire-stop, or by the 30-day tracker retention). On-fire dimension correlation over osBuild/model/agentVersion/imeVersion with the "correlated, not necessarily causal" wording contract. Surfaces: `regressions[]` in the rule-stats response (MCP inherits), the analyze-rules page's 30-day sparkline and regression badge.
|
||||
|
||||
* **Creation**: Added `backend/device-journeys.md` — F2 device history / First-Time-Right: the device key (trim + case-fold serial, junk/placeholder exclusion incl. the agent's `Unknown` WMI sentinel), journey grouping over the terminal set Succeeded/Failed/Incomplete (first success ends a journey, a >30-day gap from the previous attempt's CompletedAt starts one, WhiteGlove's single session row is ONE attempt, open sessions are never attempts), the `DeviceHistories` chain (cap 20, written inline at the terminal seam, healed by the sweep, deleted-session refs pruned tombstone-driven, deliberately not in the per-session deletion manifest) and the additive `DeviceJourneyAggregates` daily rows (window rate = ratio of sums — no rolling row needed, 180-day retention). Surfaces: the session-detail "Attempt N" banner with server-computed attempt numbers (virtual-attempt rule for live sessions), the Fleet Health First-time-right section (≥20-journey gate, attempt histogram, repeat-devices violator list), and the MCP `get_device_history` tool (serial mode / fleet FTR mode).
|
||||
|
||||
## 2026-07-25
|
||||
|
||||
* **Update**: `agent/secureboot-ca2023-detection.md` — review follow-up on 90012d9 hardened the privilege scope, the repo's only privilege-adjustment code and therefore the precedent for any future one. The process token moved from a raw `IntPtr` to a `SafeHandle` assigned by the P/Invoke marshaler, so the CLR owns it from the moment `OpenProcessToken` returns and the critical finalizer releases it even when `Dispose` never runs — previously an async exception between the successful open and the caller's `using` leaked the handle *with the privilege enabled and no owner left to restore it*. `Dispose` became idempotent and now releases the handle in a `finally`, so a throwing restore cannot strand it; the restore obligation is recorded immediately after the successful `AdjustTokenPrivileges` with nothing throwable in between. The whole enable → read → restore sequence runs under a static gate because the privilege is adjusted on the *process* token: a concurrent reader would see "already enabled", correctly skip its own restore, and then lose the privilege mid-read when the first reader disposed, reporting `privilege_denied` on a machine that holds it.
|
||||
|
||||
## 2026-07-24
|
||||
|
||||
* **Creation**: Added `backend/server-actions-on-demand-diagnostics.md` — the ServerAction channel (queue on `Sessions.PendingActionsJson`, deliver piggy-backed on the next ingest response, at-least-once with `Type|RuleId|QueuedAt` dedup and sequential dispatch) and the on-demand log collection built on it: the portal's always-visible **Collect Logs** button, the new `EndpointPolicy.TenantAdminOrOperator` tier with the mandatory in-function per-type re-gate (Operators queue only `request_diagnostics`), the member-readable `diagnosticsUploadConfigured` feature flag, the admin quick-config path that must queue `rotate_config` before `request_diagnostics` (works against the deployed fleet — a one-time override would not, the mode gate is agent-side), sequence-based progress matching immune to agent clock skew, and the offboarding cascade's hosted-prefix sweep closing the latest-wins orphan-blob gap.
|
||||
|
||||
* **Update**: `agent/esp-policy-provider-stall.md` — the customer's follow-up on issue #106 corrected the field-case mechanics: `TrackingPoliciesCreated=1` WAS set under the foreign `ConfigMgr` provider, so the ESP ignores a foreign provider's completion values and waits on the Intune-registered `Sidecar` provider by name (the rename to `Sidecar` creates no tracking entries yet unblocks instantly). The completeness-keyed detector would have missed exactly this — the incident it was built for. The tripwire is now two-armed under one event type discriminated by `reason`: `provider_incomplete` (unchanged dwell on continuously-incomplete providers) and `sidecar_provider_missing` (≥ 1 Setup/Apps provider registered without `Sidecar`, completeness deliberately ignored, clock reset when Sidecar registers later). Sidecar-missing wins the `reason` when both arms cross in one round; the cross-restart fingerprint gains a `sidecarMissing` sentinel; payload adds `reason` and `sidecarMissingForMinutes`. Evidence for the name-keyed wait: Niehaus' documented Intune WAP payload always registers `Sidecar` (ConfigMgr only via a co-management settings policy) and Microsoft's ESP troubleshooting models app tracking exclusively through the Sidecar subkey.
|
||||
|
||||
* **Creation**: Added `agent/esp-policy-provider-stall.md` — a customer diagnosed that pre-provisioned devices with the ConfigMgr client as a blocking app occasionally park the user ESP at "Apps (Identifying)" for days: only a `ConfigMgr` provider is registered under `EnrollmentStatusTracking\Device\Setup\Apps\PolicyProviders` (no `Sidecar` = IME) and `TrackingPoliciesCreated` never becomes 1 — a wait the CSP defines without any timeout; renaming the key to `Sidecar` unblocks immediately. The agent now ships the always-on `esp_policy_provider_stalled` tripwire (60-s tick, 15-min continuous-incompleteness dwell, one-shot per provider, `StartupEventGate` fingerprint across restarts, purely observational via the InformationalEvent rail — no toggle, no ConfigVersion bump), generalized to any provider in both `Setup/Apps` and `DevicePreparation` kinds rather than hardcoding the ConfigMgr case. Companion change: `SOFTWARE\Microsoft\Windows\Autopilot` joins the gather-rule registry allowlist so tenants can target the subtree with custom rules (e.g. `not_exists` on the Sidecar key).
|
||||
|
||||
@@ -44,6 +60,10 @@
|
||||
|
||||
* **Creation**: Added `agent/endpoint-migration.md` — config-channel agent re-home (`MigrateToApiBaseUrl` on `GET agent/config`, ConfigVersion 33): backend or per-tenant API-URL moves without DNS indirection (impossible due to mTLS/Flex-TLS). Live-fetch-only with cache strip, `AgentEndpointMigrationRules` allowlist validated on both sides, one hop, kill-switch wins. Built as the lesson from the 2026-07-18 WEU→GWC cutover, which hard-cut all in-flight agents.
|
||||
|
||||
## 2026-07-26
|
||||
|
||||
* **Creation**: Added `backend/time-attribution.md` — the F1 enrollment time attribution: exact-partition invariant against the session's authoritative DurationSeconds, two-window WhiteGlove model, segment mapping from phase-declaration events, positive-evidence ESP-blocking join with "up to" what-if bounds, reboot spans from event gaps around `lastBootUtc`, and the SessionTimeBreakdowns / TimeAttributionAggregates tables with their terminal-seam + maintenance-sweep compute paths and read surfaces (web lane, fleet panel, MCP `get_time_attribution`).
|
||||
|
||||
## 2026-07-17
|
||||
|
||||
* **Creation**: Added `agent/build-counter-blob.md` — agent build numbers now come from a shared public counter blob (`buildcounter-v{N}.txt`, reserve-before-build with ETag-CAS) so local `build.ps1` and CI `build-agent.yml` builds can never mint the same number. Documents the CI parity fixes (SummaryDialog.exe.config in the ZIP, EXE-vs-manifest version check, hash-oracle update gated on stable cutover) and the Sigstore provenance attestation on CI-built ZIPs.
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Portal Navigation Prefetch & the SWA Runtime Bottleneck
|
||||
description: Why every navigation Link in the portal sets prefetch={false} — the App Router's viewport prefetch fires one RSC request per visible link against an HTTP/1.1 Static Web Apps runtime, and its latency tail starves the connection pool that serves the document and the route chunks.
|
||||
resource: src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx
|
||||
tags:
|
||||
- web
|
||||
- performance
|
||||
- nextjs
|
||||
- static-web-apps
|
||||
- observability
|
||||
timestamp: 2026-07-27T18:00:00+02:00
|
||||
---
|
||||
|
||||
# Context
|
||||
|
||||
The portal is a Next.js 15 App Router application hosted on Azure Static Web Apps. Every page under
|
||||
the authenticated shell is a client component behind `ProtectedRoute` that fetches its own data from
|
||||
the Function App (`autopilotmonitor-api-eu`) after mount. The RSC payload a route returns on
|
||||
navigation therefore carries no page data — only the shell.
|
||||
|
||||
`GlobalSidebar` and `Navbar` render the full navigation on every page. With the App Router's default
|
||||
prefetch behaviour, each visible `<Link>` triggers a background `GET /<route>?_rsc=<hash>` as soon as
|
||||
it enters the viewport.
|
||||
|
||||
# Mechanism
|
||||
|
||||
Three properties combine into a portal-wide freeze:
|
||||
|
||||
1. **Fan-out.** The navigation is fully rendered on each page load, so a single navigation issues one
|
||||
RSC request per visible link — measured at ~10,400 requests over 14 days.
|
||||
2. **Shared origin over HTTP/1.1.** `portal.autopilotmonitor.com` serves the document, the JS route
|
||||
chunks *and* the RSC payloads, and it negotiates HTTP/1.1 — not HTTP/2. Browsers cap concurrent
|
||||
connections per origin at six.
|
||||
3. **A heavy latency tail on the SSR runtime.** RSC responses are prerendered and edge-cached
|
||||
(`x-nextjs-cache: HIT`, `s-maxage=31536000`), so the median is fast. On a cache miss or cold start
|
||||
the Static Web Apps managed runtime queues them.
|
||||
|
||||
When the tail hits, the in-flight prefetches hold all six connections. The document request and the
|
||||
route chunks of the page the user actually wants then have no socket available, and the tab shows a
|
||||
loading spinner or a blank page until the queue drains. Nothing fails — the requests eventually
|
||||
return `200`, which is why the failure leaves no error trace anywhere.
|
||||
|
||||
# Evidence
|
||||
|
||||
Incident of 2026-07-27, 17:25–17:29 CEST. Browser-side dependencies during the freeze:
|
||||
|
||||
| Request | Duration | Status |
|
||||
| --- | --- | --- |
|
||||
| `/admin/presence?_rsc=…` | 151,461 ms | 200 |
|
||||
| `/gather-rules?_rsc=…` | 52,283 ms | 200 |
|
||||
| `/progress?_rsc=…` | 37,878 ms | 200 |
|
||||
| `/analyze-rules?_rsc=…` | 37,559 ms | 200 |
|
||||
| `/audit?_rsc=…` | 37,539 ms | 200 |
|
||||
| Backend API, same window | 1,600–2,200 ms | 200 |
|
||||
|
||||
The three ~37.5 s responses complete within 340 ms of each other — a queue draining, not independent
|
||||
slow requests. The corresponding `browserTimings` row for `/geographic-performance` reports
|
||||
`networkDuration` = 16 ms against `processingDuration` = 225,095 ms: the network delivered the
|
||||
document immediately and the browser then waited nearly four minutes for the `load` event.
|
||||
|
||||
Fourteen-day distribution of RSC prefetches: n = 10,428, p50 = 91 ms, p95 = 896 ms, max = 234,230 ms,
|
||||
38 over 10 s, 21 over 30 s.
|
||||
|
||||
# Diagnosing this class of problem
|
||||
|
||||
The backend Application Insights resource is the wrong place to look — a freeze appears there as an
|
||||
*absence* of requests, never as an error. Browser telemetry lives in a **separate** Application
|
||||
Insights resource addressed by `AUTOPILOT_MONITOR_WEB_APPINSIGHTS_ID`, which the MCP
|
||||
`query_backend_logs` tool does not reach. Query it through
|
||||
`.claude/commands/scripts/query-appinsights.sh` with that ID exported, and read `dependencies`
|
||||
(client-side fetches, including RSC prefetches) and `browserTimings` (the network/processing split of
|
||||
a document load).
|
||||
|
||||
# Contract
|
||||
|
||||
Navigation `<Link>` elements in `GlobalSidebar` and `Navbar` set `prefetch={false}`. The prefetched
|
||||
RSC payload buys nothing for these routes, so the only remaining cost of a click is the route's JS
|
||||
chunk, which the browser caches after first use.
|
||||
|
||||
Do not remove the prop to "speed up navigation" without first re-measuring the tail in the web
|
||||
Application Insights resource — the prefetch is cheap only while the runtime is warm.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx` — nav link rendering, rationale comment
|
||||
* `src/Web/autopilot-monitor-web/components/Navbar.tsx` — top-bar and notification links
|
||||
* [SWA and Oryx node pinning](../architecture.md) — hosting model for the web app
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-ESP-006",
|
||||
"title": "ESP Failure Recovered After User Retry",
|
||||
"description": "Detects the recovery story: the ESP reported a terminal failure (typically DeviceSetup/Apps), the user most likely pressed 'Try again' on the failure page, the failed step re-ran and completed successfully. The earlier failure did not stick — the enrollment continued past it.",
|
||||
"severity": "info",
|
||||
"category": "esp",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"baseConfidence": 85,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "advisory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "recovered",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "retry",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_retry_detected",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "category",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "category",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "failedSubcategory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "failedSubcategory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "likelyCulpritApps",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"dataField": "likelyCulpritApps",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "minutesSinceFailure",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "minutesSinceFailure",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{ "signal": "retry", "condition": "exists", "weight": 10 },
|
||||
{ "signal": "likelyCulpritApps", "condition": "exists", "weight": 5 }
|
||||
],
|
||||
"explanation": "The Enrollment Status Page reported a **failure** in the `{{category}}` phase (`{{failedSubcategory}}` subcategory, most likely app(s): `{{likelyCulpritApps}}`) — and then **recovered**: the failed step left its failed state and completed successfully about `{{minutesSinceFailure}}` min later.\n\nThat pattern is consistent with the user pressing **'Try again'** on the ESP failure page (the profile allows failure-page actions; the agent cannot observe the click itself). The failure the user saw was real, but it did not stick — the ESP re-ran the step, it succeeded, and the enrollment continued.\n\nRead the session timeline as one arc: `esp_failure_advisory` (the failure, with the likely culprit app) → `esp_failure_retry_detected` (the retry) → `esp_failure_recovered` (the recovery). If this pattern repeats across devices for the same app, that app's install is flaky on first attempt — fix the app rather than the devices. Any literal `{{…}}` placeholder means that detail was not recorded for this session.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Make the first install attempt stick",
|
||||
"steps": [
|
||||
"Check the likely culprit app(s) (`{{likelyCulpritApps}}`) in Intune: an app that fails on the first ESP pass but succeeds on retry usually has a dependency/ordering or detection-timing problem.",
|
||||
"If the app shows 'never started installing' in esp_failure_settle_started, verify its assignment and applicability rules — ESP counted it as required, but IME never processed it.",
|
||||
"If the pattern repeats across the tenant, consider removing the app from the ESP blocking list until the first-attempt install is reliable — users are seeing an avoidable failure screen.",
|
||||
"No device-side action is needed for this session: the failure recovered and the enrollment continued."
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": ["esp", "recovery", "try-again", "retry", "continue-anyway", "enrollment"]
|
||||
}
|
||||
+104
@@ -1884,6 +1884,110 @@
|
||||
"user-experience"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-ESP-006",
|
||||
"title": "ESP Failure Recovered After User Retry",
|
||||
"description": "Detects the recovery story: the ESP reported a terminal failure (typically DeviceSetup/Apps), the user most likely pressed 'Try again' on the failure page, the failed step re-ran and completed successfully. The earlier failure did not stick — the enrollment continued past it.",
|
||||
"severity": "info",
|
||||
"category": "esp",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"baseConfidence": 85,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "advisory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "recovered",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "retry",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_retry_detected",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "category",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "category",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "failedSubcategory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "failedSubcategory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "likelyCulpritApps",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"dataField": "likelyCulpritApps",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "minutesSinceFailure",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "minutesSinceFailure",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{
|
||||
"signal": "retry",
|
||||
"condition": "exists",
|
||||
"weight": 10
|
||||
},
|
||||
{
|
||||
"signal": "likelyCulpritApps",
|
||||
"condition": "exists",
|
||||
"weight": 5
|
||||
}
|
||||
],
|
||||
"explanation": "The Enrollment Status Page reported a **failure** in the `{{category}}` phase (`{{failedSubcategory}}` subcategory, most likely app(s): `{{likelyCulpritApps}}`) — and then **recovered**: the failed step left its failed state and completed successfully about `{{minutesSinceFailure}}` min later.\n\nThat pattern is consistent with the user pressing **'Try again'** on the ESP failure page (the profile allows failure-page actions; the agent cannot observe the click itself). The failure the user saw was real, but it did not stick — the ESP re-ran the step, it succeeded, and the enrollment continued.\n\nRead the session timeline as one arc: `esp_failure_advisory` (the failure, with the likely culprit app) → `esp_failure_retry_detected` (the retry) → `esp_failure_recovered` (the recovery). If this pattern repeats across devices for the same app, that app's install is flaky on first attempt — fix the app rather than the devices. Any literal `{{…}}` placeholder means that detail was not recorded for this session.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Make the first install attempt stick",
|
||||
"steps": [
|
||||
"Check the likely culprit app(s) (`{{likelyCulpritApps}}`) in Intune: an app that fails on the first ESP pass but succeeds on retry usually has a dependency/ordering or detection-timing problem.",
|
||||
"If the app shows 'never started installing' in esp_failure_settle_started, verify its assignment and applicability rules — ESP counted it as required, but IME never processed it.",
|
||||
"If the pattern repeats across the tenant, consider removing the app from the ESP blocking list until the first-attempt install is reliable — users are seeing an avoidable failure screen.",
|
||||
"No device-side action is needed for this session: the failure recovered and the enrollment continued."
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": [
|
||||
"esp",
|
||||
"recovery",
|
||||
"try-again",
|
||||
"retry",
|
||||
"continue-anyway",
|
||||
"enrollment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-ID-001",
|
||||
"title": "Expected Machine Certificate Not Found",
|
||||
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
using DeviceInfoCollector = AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo.DeviceInfoCollector;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.DeviceInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Audit Q2 — <see cref="DeviceInfoCollector.RefreshEspConfiguration"/>
|
||||
/// re-reads the ESP blocking lists and re-emits <c>esp_config_detected</c> exactly when the
|
||||
/// payload changed: the registry lists grow progressively (one timestamped subkey per CSP
|
||||
/// status write) and the user-scope lists appear only after sign-in, so the early emissions
|
||||
/// are structurally partial. The StartupEventGate keeps an unchanged re-read silent.
|
||||
/// Probe overrides are static slots → serialized with the other probe-driven suites.
|
||||
/// </summary>
|
||||
[Collection("SerialThreading")]
|
||||
public sealed class DeviceInfoCollectorEspRefreshTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 7, 27, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const string DeviceApp1 = "11111111-1111-1111-1111-111111111111";
|
||||
private const string DeviceApp2 = "22222222-2222-2222-2222-222222222222";
|
||||
private const string UserApp1 = "33333333-3333-3333-3333-333333333333";
|
||||
|
||||
private static EspTrackingInfoSnapshot Snapshot(string[] deviceApps, string[] userApps)
|
||||
{
|
||||
var all = deviceApps.Concat(userApps).Distinct().ToArray();
|
||||
return new EspTrackingInfoSnapshot(
|
||||
msiProductCodes: Array.Empty<string>(),
|
||||
modernAppPfns: Array.Empty<string>(),
|
||||
win32AppIds: all,
|
||||
userWin32AppIds: userApps,
|
||||
msiCount: 0,
|
||||
modernCount: 0,
|
||||
win32Count: all.Length);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<FakeSignalIngressSink.PostedSignal> EspConfigEvents(FakeSignalIngressSink ingress) =>
|
||||
ingress.Posted
|
||||
.Where(p => p.Kind == DecisionSignalKind.InformationalEvent
|
||||
&& p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == SharedEventTypes.EspConfigDetected)
|
||||
.ToList();
|
||||
|
||||
private static IReadOnlyList<string> TypedList(FakeSignalIngressSink.PostedSignal signal, string key)
|
||||
{
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(signal.TypedPayload);
|
||||
return Assert.IsAssignableFrom<IReadOnlyList<string>>(data[key]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Refresh_reemits_when_lists_grew_and_stays_silent_when_unchanged()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
var ingress = new FakeSignalIngressSink();
|
||||
var clock = new VirtualClock(T0);
|
||||
var collector = new DeviceInfoCollector(
|
||||
"session-1", "tenant-1",
|
||||
new InformationalEventPost(ingress, clock), logger,
|
||||
signalIngress: null, clock: clock,
|
||||
startupGate: new StartupEventGate(tmp.Path, logger));
|
||||
|
||||
using var skip = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_ => new EspFirstSyncSnapshot(
|
||||
skipUser: false, skipDevice: false,
|
||||
blockInStatusPage: null, syncFailureTimeoutMinutes: null));
|
||||
|
||||
// Early read (agent start / DeviceSetup detection): partial device list, empty user list.
|
||||
using (new EspTrackingInfoProbe.ScopedOverride(_ => Snapshot(new[] { DeviceApp1 }, Array.Empty<string>())))
|
||||
{
|
||||
collector.CollectEspConfiguration();
|
||||
}
|
||||
var afterEarly = EspConfigEvents(ingress);
|
||||
Assert.Single(afterEarly);
|
||||
Assert.Empty(TypedList(afterEarly[0], "espTrackedUserWin32AppIds"));
|
||||
|
||||
// Apps sub-phase opened: the device list grew and sign-in populated the user list.
|
||||
using (new EspTrackingInfoProbe.ScopedOverride(_ => Snapshot(new[] { DeviceApp1, DeviceApp2 }, new[] { UserApp1 })))
|
||||
{
|
||||
collector.RefreshEspConfiguration(DeviceInfoHost.EspConfigTriggerAppsUser);
|
||||
}
|
||||
var afterGrowth = EspConfigEvents(ingress);
|
||||
Assert.Equal(2, afterGrowth.Count);
|
||||
Assert.Contains(DeviceApp2, TypedList(afterGrowth[1], "espTrackedWin32AppIds"));
|
||||
Assert.Equal(new[] { UserApp1 }, TypedList(afterGrowth[1], "espTrackedUserWin32AppIds"));
|
||||
|
||||
// A later trigger with an UNCHANGED registry state re-reads but emits nothing —
|
||||
// the StartupEventGate suppresses the identical payload.
|
||||
using (new EspTrackingInfoProbe.ScopedOverride(_ => Snapshot(new[] { DeviceApp1, DeviceApp2 }, new[] { UserApp1 })))
|
||||
{
|
||||
collector.RefreshEspConfiguration(DeviceInfoHost.EspConfigTriggerAccountSetup);
|
||||
}
|
||||
Assert.Equal(2, EspConfigEvents(ingress).Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Concurrent_refreshes_never_emit_an_older_snapshot_after_a_newer_one()
|
||||
{
|
||||
// Codex review: the refresh triggers each run on their own Task.Run — without the
|
||||
// collector-side lock, two collects can interleave read-before-emit and the LATER
|
||||
// emission carries the OLDER (smaller) list. The backend keeps the latest
|
||||
// list-bearing emission as positive evidence, so that would silently shrink it.
|
||||
// The probe returns a strictly growing list per read; with the read→emit critical
|
||||
// section, emission order must equal read order — sizes strictly ascending.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
var ingress = new FakeSignalIngressSink();
|
||||
var clock = new VirtualClock(T0);
|
||||
var collector = new DeviceInfoCollector(
|
||||
"session-1", "tenant-1",
|
||||
new InformationalEventPost(ingress, clock), logger,
|
||||
signalIngress: null, clock: clock,
|
||||
startupGate: new StartupEventGate(tmp.Path, logger));
|
||||
|
||||
using var skip = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_ => new EspFirstSyncSnapshot(
|
||||
skipUser: false, skipDevice: false,
|
||||
blockInStatusPage: null, syncFailureTimeoutMinutes: null));
|
||||
|
||||
var readCounter = 0;
|
||||
using var tracking = new EspTrackingInfoProbe.ScopedOverride(_ =>
|
||||
{
|
||||
var n = Interlocked.Increment(ref readCounter);
|
||||
var ids = Enumerable.Range(1, n)
|
||||
.Select(i => $"{i:d8}-1111-2222-3333-444444444444")
|
||||
.ToArray();
|
||||
return Snapshot(ids, Array.Empty<string>());
|
||||
});
|
||||
|
||||
var tasks = Enumerable.Range(0, 12)
|
||||
.Select(i => Task.Run(() => collector.RefreshEspConfiguration($"stress-{i}")))
|
||||
.ToArray();
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
var sizes = EspConfigEvents(ingress)
|
||||
.Select(e => TypedList(e, "espTrackedWin32AppIds").Count)
|
||||
.ToList();
|
||||
// Every refresh saw a strictly larger list (gate never suppresses) and each
|
||||
// read→emit is atomic, so the emission sequence is exactly 1..12.
|
||||
Assert.Equal(Enumerable.Range(1, 12), sizes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Harness
|
||||
logFolder: Tmp.Path,
|
||||
patterns: new List<ImeLogPattern>(),
|
||||
logger: Logger);
|
||||
// Mirror ImeLogHost wiring: the historic-replay guard judges staleness against the
|
||||
// (virtual) agent clock so tests stay deterministic.
|
||||
Tracker.UtcNowProvider = () => Clock.UtcNow;
|
||||
}
|
||||
|
||||
public IReadOnlyList<FakeSignalIngressSink.PostedSignal> DecisionSignals(DecisionSignalKind kind) =>
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// Tracker-level historic-replay guard (session eaf3d8c4 part 2): source lines > 24 h older
|
||||
/// than now are content from a previous enrollment whose IME log survived on disk. The guard
|
||||
/// in <c>HandlePatternMatch</c> skips app-mutating actions for such lines so replayed apps
|
||||
/// never enter <c>_packageStates</c> / phase snapshots / persistence — keeping
|
||||
/// app_tracking_summary, culprit lists and final-status clean. Script actions pass through
|
||||
/// (the adapter suppresses their emissions), and SimulationMode bypasses the guard entirely.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerHistoricReplayGuardTests
|
||||
{
|
||||
private static readonly DateTime Now = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
private static List<ImeLogPattern> TestPatterns() => new List<ImeLogPattern>
|
||||
{
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-POLICIES", Category = "always", Enabled = true,
|
||||
Pattern = @"policies discovered = (?<policies>\[.*\])",
|
||||
Action = "policiesDiscovered",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-INSTALLED", Category = "always", Enabled = true,
|
||||
Pattern = @"app (?<id>[\w-]+) installed",
|
||||
Action = "updateStateInstalled",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-ESP-PHASE", Category = "always", Enabled = true,
|
||||
Pattern = @"In EspPhase: (?<espPhase>\w+)",
|
||||
Action = "espPhaseDetected",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-SCRIPT-START", Category = "always", Enabled = true,
|
||||
Pattern = @"script (?<id>[\w-]+) started",
|
||||
Action = "scriptStarted",
|
||||
Parameters = new Dictionary<string, string> { ["scriptType"] = "platform" },
|
||||
},
|
||||
};
|
||||
|
||||
private static ImeLogTracker BuildTracker(TempDirectory tmp)
|
||||
{
|
||||
var tracker = new ImeLogTracker(
|
||||
logFolder: tmp.Path,
|
||||
patterns: TestPatterns(),
|
||||
logger: new AgentLogger(tmp.Path, AgentLogLevel.Info));
|
||||
tracker.UtcNowProvider = () => Now;
|
||||
return tracker;
|
||||
}
|
||||
|
||||
private const string PoliciesLine = @"policies discovered = [{""Id"":""app-1"",""Name"":""App One""}]";
|
||||
|
||||
[Fact]
|
||||
public void Stale_policies_line_does_not_populate_package_states()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddDays(-7));
|
||||
|
||||
Assert.Empty(tracker.GetAllKnownPackageStates());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_relog_of_same_policies_is_tracked_after_stale_skip()
|
||||
{
|
||||
// IME re-logs the full policy JSON on every app-policy check-in — skipping the
|
||||
// stale copy cannot starve the current enrollment.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddDays(-7));
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddMinutes(-2));
|
||||
|
||||
var all = tracker.GetAllKnownPackageStates();
|
||||
var pkg = Assert.Single(all);
|
||||
Assert.Equal("app-1", pkg.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_state_update_against_fresh_app_is_skipped()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
var stateChanges = 0;
|
||||
tracker.OnAppStateChanged = (_, __, ___) => stateChanges++;
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddMinutes(-5)); // fresh discovery
|
||||
tracker.ProcessLogMessageForTest("app app-1 installed", Now.AddDays(-7)); // stale mutation
|
||||
|
||||
Assert.Equal(0, stateChanges);
|
||||
var pkg = Assert.Single(tracker.GetAllKnownPackageStates());
|
||||
Assert.NotEqual(AppInstallationState.Installed, pkg.InstallationState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_esp_phase_line_does_not_block_fresh_phase_detection()
|
||||
{
|
||||
// A stale "AccountSetup" would advance _currentPhaseOrder and make the fresh
|
||||
// enrollment's DeviceSetup bounce as "backward" — the guard must skip it.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
var phases = new List<string>();
|
||||
tracker.OnEspPhaseChanged = p => phases.Add(p);
|
||||
|
||||
tracker.ProcessLogMessageForTest("In EspPhase: AccountSetup", Now.AddDays(-7));
|
||||
tracker.ProcessLogMessageForTest("In EspPhase: DeviceSetup", Now.AddMinutes(-1));
|
||||
|
||||
Assert.Equal(new[] { "DeviceSetup" }, phases);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_script_line_still_reaches_script_handlers()
|
||||
{
|
||||
// Script tracker state is harmless for stale lines (adapter suppresses the
|
||||
// emissions; the stale-slot hardening covers leftovers) — the guard must not
|
||||
// swallow script actions.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
var started = new List<ScriptStartedInfo>();
|
||||
tracker.OnScriptStarted = i => started.Add(i);
|
||||
|
||||
tracker.ProcessLogMessageForTest("script aaaa1111-0000-0000-0000-000000000000 started", Now.AddDays(-7));
|
||||
|
||||
Assert.Single(started);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimulationMode_bypasses_the_guard()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
tracker.SimulationMode = true;
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddDays(-7));
|
||||
|
||||
Assert.Single(tracker.GetAllKnownPackageStates());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Lines_without_timestamp_are_not_treated_as_stale()
|
||||
{
|
||||
// Non-CMTrace lines carry no source timestamp (entry == null) — never stale.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, sourceTimestampUtc: null);
|
||||
|
||||
Assert.Single(tracker.GetAllKnownPackageStates());
|
||||
}
|
||||
}
|
||||
}
|
||||
+47
-51
@@ -15,7 +15,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
/// <item>a locked tree must abort WITHOUT deleting anything (old script force-deleted
|
||||
/// marker + state around the locks and orphaned the binaries),</item>
|
||||
/// <item>a straggler agent process (the incident's second instance) must be found by
|
||||
/// NAME, killed, and the cleanup must then complete,</item>
|
||||
/// NAME, killed, and the cleanup must then complete — simulated via function
|
||||
/// shadowing, NEVER by launching a renamed executable (EDR isolation, see the test),</item>
|
||||
/// <item>concurrent cleanup scripts must be serialized by the lock file.</item>
|
||||
/// </list>
|
||||
/// Timings are dialed down via the test-only BuildCleanupScript parameters so each case
|
||||
@@ -111,60 +112,55 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Straggler_process_holding_the_tree_is_killed_and_cleanup_completes()
|
||||
public void Straggler_agent_process_is_waited_on_and_force_killed_via_name_backstop()
|
||||
{
|
||||
// Full incident replay: a SECOND agent-named process (here: a renamed
|
||||
// powershell.exe, matching how only the process NAME identifies agents) holds a
|
||||
// file in the tree open. The old script waited on the launcher PID only and ran
|
||||
// its force-delete against the live locks. The hardened script must find the
|
||||
// straggler by name, kill it after the wait deadline, and then delete everything.
|
||||
var fakeExe = Path.Combine(_root, _fakeProcessName + ".exe");
|
||||
File.Copy(SystemPaths.PowerShell, fakeExe);
|
||||
// Incident replay of the second-instance mechanics WITHOUT spawning a process:
|
||||
// an earlier version of this test copied powershell.exe under the fake agent name
|
||||
// and launched it — textbook MITRE T1036.003 ("Rename Legitimate Utilities"),
|
||||
// which Defender for Endpoint answered with a machine ISOLATION on the dev box
|
||||
// (see tasks/lessons.md). Instead, a harness script shadows Get-Process /
|
||||
// Stop-Process with functions (PowerShell resolves functions before cmdlets, so
|
||||
// the dot-sourced cleanup script binds to the stubs): Get-Process reports a live
|
||||
// agent-named straggler on every probe; Stop-Process records the kill. This pins
|
||||
// the same contract — the script polls by NAME until the deadline, then
|
||||
// force-kills by NAME before deleting — with zero EDR-visible behavior.
|
||||
var probeLog = Path.Combine(_root, "probe.log");
|
||||
var cleanupScriptPath = Path.Combine(_root, $"cleanup-{Guid.NewGuid():N}.ps1");
|
||||
File.WriteAllText(cleanupScriptPath, BuildScript(processWaitSeconds: 2));
|
||||
|
||||
var lockTarget = Path.Combine(_basePath, "Agent", "fake.exe").Replace("'", "''");
|
||||
Process straggler = null;
|
||||
try
|
||||
{
|
||||
straggler = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = fakeExe,
|
||||
Arguments = "-NoProfile -Command \"$f = [IO.File]::Open('" + lockTarget +
|
||||
"', 'Open', 'Read', 'None'); Start-Sleep -Seconds 120\"",
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
var harness = @"
|
||||
$probeLog = 'LOG_PATH'
|
||||
$global:AmProbeCount = 0
|
||||
function Get-Process {
|
||||
[CmdletBinding()]
|
||||
param([string[]]$Name)
|
||||
if ($Name -contains 'FAKE_NAME') {
|
||||
$global:AmProbeCount++
|
||||
Add-Content -Path $probeLog -Value ('probe ' + $global:AmProbeCount)
|
||||
return [pscustomobject]@{ Name = 'FAKE_NAME'; Id = 424242 }
|
||||
}
|
||||
Microsoft.PowerShell.Management\Get-Process @PSBoundParameters
|
||||
}
|
||||
function Stop-Process {
|
||||
[CmdletBinding()]
|
||||
param([Parameter(ValueFromPipeline = $true)]$InputObject, [switch]$Force)
|
||||
process { Add-Content -Path $probeLog -Value ('kill ' + $InputObject.Id + ' force=' + $Force) }
|
||||
}
|
||||
. 'SCRIPT_PATH'
|
||||
"
|
||||
.Replace("LOG_PATH", probeLog)
|
||||
.Replace("FAKE_NAME", _fakeProcessName)
|
||||
.Replace("SCRIPT_PATH", cleanupScriptPath);
|
||||
|
||||
// Give the straggler time to acquire its file handle before cleanup starts.
|
||||
var acquireDeadline = DateTime.UtcNow.AddSeconds(20);
|
||||
var acquired = false;
|
||||
while (!acquired && DateTime.UtcNow < acquireDeadline)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (File.Open(Path.Combine(_basePath, "Agent", "fake.exe"),
|
||||
FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
|
||||
{
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
acquired = true; // share violation ⇒ straggler holds the file
|
||||
}
|
||||
if (!acquired) System.Threading.Thread.Sleep(200);
|
||||
}
|
||||
Assert.True(acquired, "straggler never acquired its file lock");
|
||||
RunScript(harness);
|
||||
|
||||
RunScript(BuildScript(processWaitSeconds: 2));
|
||||
|
||||
Assert.True(straggler.HasExited, "straggler must be force-killed by the name-based backstop");
|
||||
Assert.False(Directory.Exists(_basePath), "after the kill the tree must be fully deleted");
|
||||
Assert.Empty(Directory.GetFileSystemEntries(_root, "*.del-*"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { if (straggler != null && !straggler.HasExited) straggler.Kill(); } catch { }
|
||||
try { straggler?.Dispose(); } catch { }
|
||||
}
|
||||
var log = File.ReadAllLines(probeLog);
|
||||
Assert.Contains("kill 424242 force=True", log);
|
||||
Assert.True(Array.FindAll(log, l => l.StartsWith("probe")).Length >= 2,
|
||||
"script must poll by name in the wait loop AND re-probe for the kill backstop");
|
||||
Assert.False(Directory.Exists(_basePath), "after the kill the tree must be fully deleted");
|
||||
Assert.Empty(Directory.GetFileSystemEntries(_root, "*.del-*"));
|
||||
Assert.False(File.Exists(LockPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 4910a5a5 (2026-07-23) — post-terminal recovery story. Field sequence:
|
||||
/// DeviceSetup/Apps failed (culprit app never started installing) → settle window expired
|
||||
/// → terminal EspFailureDetected → the user pressed "Try again" ~23 min later → Apps left
|
||||
/// the failed state → 10/10 installed → DeviceSetup resolved. Pre-fix, that recovery only
|
||||
/// appeared as a generic <c>subcategory_state_change</c>. Coverage:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>esp_failure_retry_detected</c> fires once when the failed subcategory leaves
|
||||
/// the failed state AFTER the terminal fire.</item>
|
||||
/// <item><c>esp_failure_recovered</c> fires once when the category then resolves to
|
||||
/// success.</item>
|
||||
/// <item>An in-settle-window recovery keeps emitting only
|
||||
/// <c>esp_failure_settle_recovered</c> — the post-terminal pair stays silent.</item>
|
||||
/// <item>Apps failures carry <c>LikelyCulpritApps</c> on the EspFailureDetected args,
|
||||
/// never-started apps ranked first.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class EspFailureRecoveryStoryTests
|
||||
{
|
||||
private static readonly DateTime Fixed = new DateTime(2026, 7, 23, 4, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Fixture : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public FakeSignalIngressSink Sink { get; } = new FakeSignalIngressSink();
|
||||
public List<EspFailureDetectedEventArgs> EspFailures { get; } = new List<EspFailureDetectedEventArgs>();
|
||||
public ProvisioningStatusTracker Tracker { get; }
|
||||
|
||||
public Fixture(Func<IReadOnlyList<AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState>>? packageStatesProbe = null)
|
||||
{
|
||||
var clock = new VirtualClock(Fixed);
|
||||
var post = new InformationalEventPost(Sink, clock);
|
||||
var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info);
|
||||
Tracker = new ProvisioningStatusTracker(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: post,
|
||||
logger: logger,
|
||||
appxScanner: new FakeAppxDeploymentFailureScanner(),
|
||||
backgroundDispatcher: action => action(),
|
||||
packageStatesProbe: packageStatesProbe);
|
||||
Tracker.EspFailureDetected += (_, args) => EspFailures.Add(args);
|
||||
}
|
||||
|
||||
public IReadOnlyList<(string EventType, IReadOnlyDictionary<string, object> Data)> CapturedEvents()
|
||||
{
|
||||
return Sink.Posted
|
||||
.Where(p => p.Kind == DecisionSignalKind.InformationalEvent && p.Payload != null)
|
||||
.Select(p =>
|
||||
{
|
||||
var eventType = p.Payload!.TryGetValue(SignalPayloadKeys.EventType, out var et) ? et : "";
|
||||
var data = p.TypedPayload as IReadOnlyDictionary<string, object>
|
||||
?? (IReadOnlyDictionary<string, object>)new Dictionary<string, object>();
|
||||
return (eventType, data);
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>Drive the session-4910a5a5 shape up to the terminal fire.</summary>
|
||||
public void DriveToTerminalDeviceSetupAppsFailure()
|
||||
{
|
||||
Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (9 of 10 installed)""}
|
||||
}");
|
||||
Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""}
|
||||
}");
|
||||
Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Tracker.Dispose();
|
||||
Tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PostTerminalRetry_EmitsRetryDetected_Once()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
f.DriveToTerminalDeviceSetupAppsFailure();
|
||||
Assert.Single(f.EspFailures);
|
||||
|
||||
// "Try again": Apps leaves the failed state well after the settle window.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""}
|
||||
}");
|
||||
|
||||
var retry = Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_retry_detected");
|
||||
Assert.Equal("DeviceSetup", (string)retry.Data["category"]);
|
||||
Assert.Equal("Apps", (string)retry.Data["subcategory"]);
|
||||
Assert.Equal("failed", (string)retry.Data["previousState"]);
|
||||
Assert.Equal("inProgress", (string)retry.Data["newState"]);
|
||||
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", (string)retry.Data["failureType"]);
|
||||
Assert.True(retry.Data.ContainsKey("minutesSinceFailure"));
|
||||
|
||||
// A later unrelated transition does not re-emit.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed)""}
|
||||
}");
|
||||
Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_retry_detected");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PostTerminalRecovery_EmitsRecovered_OnCategorySuccess()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
f.DriveToTerminalDeviceSetupAppsFailure();
|
||||
|
||||
// Retry → all apps installed → Windows resolves the category boolean.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""}
|
||||
}");
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": true,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed)""}
|
||||
}");
|
||||
|
||||
var recovered = Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_recovered");
|
||||
Assert.Equal("DeviceSetup", (string)recovered.Data["category"]);
|
||||
Assert.Equal("Apps", (string)recovered.Data["failedSubcategory"]);
|
||||
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", (string)recovered.Data["failureType"]);
|
||||
Assert.True(recovered.Data.ContainsKey("minutesSinceFailure"));
|
||||
|
||||
// Duplicate category-success writes do not re-emit (fire-once).
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": true,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed, take 2)""}
|
||||
}");
|
||||
Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_recovered");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InSettleWindowRecovery_KeepsPostTerminalPairSilent()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (0 of 1 installed)""}
|
||||
}");
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""}
|
||||
}");
|
||||
// Recovery INSIDE the settle window — terminal fire is suppressed.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""}
|
||||
}");
|
||||
f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
|
||||
Assert.Empty(f.EspFailures);
|
||||
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": true,
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (1 of 1 installed)""}
|
||||
}");
|
||||
|
||||
var events = f.CapturedEvents();
|
||||
Assert.Contains(events, e => e.EventType == "esp_failure_settle_recovered");
|
||||
Assert.DoesNotContain(events, e => e.EventType == "esp_failure_retry_detected");
|
||||
Assert.DoesNotContain(events, e => e.EventType == "esp_failure_recovered");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AppsFailure_TerminalArgs_CarryLikelyCulprits_NeverStartedFirst()
|
||||
{
|
||||
var neverStarted = BuildApp("app-bg", "Teams Backgrounds v2",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown,
|
||||
downloadingSeen: false);
|
||||
var startedButPending = BuildApp("app-slow", "Slow Installer",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown,
|
||||
downloadingSeen: true);
|
||||
var installed = BuildApp("app-ok", "Falcon Sensor",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Installed,
|
||||
downloadingSeen: true);
|
||||
using var f = new Fixture(() => new[] { startedButPending, neverStarted, installed });
|
||||
|
||||
f.DriveToTerminalDeviceSetupAppsFailure();
|
||||
|
||||
var fired = Assert.Single(f.EspFailures);
|
||||
Assert.Equal(2, fired.LikelyCulpritApps.Count);
|
||||
// Never-started app ranked first — it starved the ESP gate without any IME event.
|
||||
Assert.Equal("Teams Backgrounds v2", fired.LikelyCulpritApps[0]);
|
||||
Assert.Equal("Slow Installer", fired.LikelyCulpritApps[1]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonAppsFailure_TerminalArgs_CarryNoCulprits()
|
||||
{
|
||||
var neverStarted = BuildApp("app-bg", "Teams Backgrounds v2",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown,
|
||||
downloadingSeen: false);
|
||||
using var f = new Fixture(() => new[] { neverStarted });
|
||||
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Working""}
|
||||
}");
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Failed""}
|
||||
}");
|
||||
f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
|
||||
|
||||
var fired = Assert.Single(f.EspFailures);
|
||||
Assert.Empty(fired.LikelyCulpritApps);
|
||||
}
|
||||
|
||||
private static AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState BuildApp(
|
||||
string id, string name,
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState state,
|
||||
bool downloadingSeen)
|
||||
{
|
||||
var pkg = new AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState(id, 0);
|
||||
pkg.UpdateName(name);
|
||||
pkg.UpdateIntent(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppIntent.Install);
|
||||
pkg.UpdateTargeted(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppTargeted.Device);
|
||||
// DownloadingOrInstallingSeen is derived inside UpdateState (Downloading..Installing
|
||||
// range) — drive it through a Downloading transition instead of setting it directly.
|
||||
if (downloadingSeen)
|
||||
pkg.UpdateState(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Downloading);
|
||||
if (state != AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown
|
||||
&& state != AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Downloading)
|
||||
pkg.UpdateState(state);
|
||||
return pkg;
|
||||
}
|
||||
}
|
||||
}
|
||||
+116
-10
@@ -74,14 +74,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
var previous = new Dictionary<string, DateTime> { ["setupApps|device|ConfigMgr"] = T0 };
|
||||
|
||||
var missing = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
EspPolicyProviderSnapshot.Empty, previous, new HashSet<string>(), T0.AddHours(2), Dwell);
|
||||
EspPolicyProviderSnapshot.Empty, previous, new HashSet<string>(), T0, false, T0.AddHours(2), Dwell);
|
||||
Assert.Empty(missing.UpdatedFirstSeenUtc);
|
||||
Assert.Empty(missing.NewlyStalled);
|
||||
Assert.Null(missing.UpdatedSidecarMissingSinceUtc); // arm 2 resets too
|
||||
Assert.Null(missing.SidecarMissingStalledForMinutes);
|
||||
|
||||
var registeredButEmpty = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(), previous, new HashSet<string>(), T0.AddHours(2), Dwell);
|
||||
Snapshot(), previous, new HashSet<string>(), T0, false, T0.AddHours(2), Dwell);
|
||||
Assert.Empty(registeredButEmpty.UpdatedFirstSeenUtc);
|
||||
Assert.Empty(registeredButEmpty.NewlyStalled);
|
||||
Assert.Null(registeredButEmpty.UpdatedSidecarMissingSinceUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -90,14 +93,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: null));
|
||||
|
||||
var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), T0, Dwell);
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Equal(T0, round1.UpdatedFirstSeenUtc["setupApps|device|ConfigMgr"]);
|
||||
Assert.Empty(round1.NewlyStalled);
|
||||
|
||||
var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, round1.UpdatedFirstSeenUtc, new HashSet<string>(), T0.AddMinutes(14), Dwell);
|
||||
snapshot, round1.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round1.UpdatedSidecarMissingSinceUtc, false, T0.AddMinutes(14), Dwell);
|
||||
Assert.Equal(T0, round2.UpdatedFirstSeenUtc["setupApps|device|ConfigMgr"]); // first-seen preserved
|
||||
Assert.Empty(round2.NewlyStalled);
|
||||
Assert.Null(round2.SidecarMissingStalledForMinutes); // 14 min < dwell on arm 2 as well
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -107,7 +112,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
var seen = new Dictionary<string, DateTime> { ["setupApps|device|ConfigMgr"] = T0 };
|
||||
|
||||
var eval = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, seen, new HashSet<string>(), T0.AddMinutes(15), Dwell);
|
||||
snapshot, seen, new HashSet<string>(), null, false, T0.AddMinutes(15), Dwell);
|
||||
|
||||
var stalled = Assert.Single(eval.NewlyStalled);
|
||||
Assert.Equal("ConfigMgr", stalled.Provider.Name);
|
||||
@@ -122,20 +127,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
// Completes mid-dwell → dropped from the map.
|
||||
var completed = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)), seen, new HashSet<string>(), T0.AddMinutes(10), Dwell);
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)), seen, new HashSet<string>(), null, false, T0.AddMinutes(10), Dwell);
|
||||
Assert.Empty(completed.UpdatedFirstSeenUtc);
|
||||
|
||||
// Re-appears incomplete much later → dwell restarts from the new now, no stall yet.
|
||||
var reappeared = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 0)), completed.UpdatedFirstSeenUtc,
|
||||
new HashSet<string>(), T0.AddHours(3), Dwell);
|
||||
new HashSet<string>(), null, false, T0.AddHours(3), Dwell);
|
||||
Assert.Equal(T0.AddHours(3), reappeared.UpdatedFirstSeenUtc[key]);
|
||||
Assert.Empty(reappeared.NewlyStalled);
|
||||
|
||||
// Disappearance behaves the same as completion.
|
||||
var disappeared = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("Sidecar", tracking: 1)), new Dictionary<string, DateTime> { [key] = T0 },
|
||||
new HashSet<string>(), T0.AddMinutes(20), Dwell);
|
||||
new HashSet<string>(), null, false, T0.AddMinutes(20), Dwell);
|
||||
Assert.False(disappeared.UpdatedFirstSeenUtc.ContainsKey(key));
|
||||
}
|
||||
|
||||
@@ -147,7 +152,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
var fired = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "setupApps|device|ConfigMgr" };
|
||||
|
||||
var eval = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, seen, fired, T0.AddHours(5), Dwell);
|
||||
snapshot, seen, fired, null, false, T0.AddHours(5), Dwell);
|
||||
|
||||
Assert.Empty(eval.NewlyStalled);
|
||||
Assert.True(eval.UpdatedFirstSeenUtc.ContainsKey("setupApps|device|ConfigMgr")); // still tracked
|
||||
@@ -162,7 +167,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
DevicePrep("ConfigMgr", installState: 1));
|
||||
|
||||
var eval = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), T0, Dwell);
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
|
||||
Assert.Equal(3, eval.UpdatedFirstSeenUtc.Count);
|
||||
Assert.Contains("setupApps|device|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys);
|
||||
@@ -179,6 +184,71 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
Assert.False(Snapshot(DevicePrep("Sidecar", installState: 3)).SidecarRegistered); // wrong kind
|
||||
Assert.False(EspPolicyProviderSnapshot.Empty.SidecarRegistered);
|
||||
}
|
||||
|
||||
// ------------------------------------------------- arm 2: sidecar-missing transition
|
||||
|
||||
[Fact]
|
||||
public void Issue106_foreign_provider_complete_but_sidecar_missing_stalls_after_dwell()
|
||||
{
|
||||
// The actual field case: ConfigMgr with TrackingPoliciesCreated=1 — "complete" by the
|
||||
// CSP value contract, arm 1 never fires — yet the ESP waits on the absent Sidecar.
|
||||
var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: 1));
|
||||
|
||||
var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Empty(round1.NewlyStalled); // arm 1 stays silent...
|
||||
Assert.Equal(T0, round1.UpdatedSidecarMissingSinceUtc); // ...but arm 2 starts its clock
|
||||
Assert.Null(round1.SidecarMissingStalledForMinutes);
|
||||
|
||||
var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, round1.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round1.UpdatedSidecarMissingSinceUtc, false, T0.AddMinutes(15), Dwell);
|
||||
Assert.Empty(round2.NewlyStalled);
|
||||
Assert.Equal(15.0, round2.SidecarMissingStalledForMinutes!.Value, precision: 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sidecar_registering_resets_the_sidecar_missing_clock()
|
||||
{
|
||||
// ConfigMgr first (legitimate startup ordering) → clock runs...
|
||||
var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)),
|
||||
new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Equal(T0, round1.UpdatedSidecarMissingSinceUtc);
|
||||
|
||||
// ...Sidecar joins → condition clears, state resets.
|
||||
var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1), SetupApps("Sidecar", tracking: 0)),
|
||||
round1.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round1.UpdatedSidecarMissingSinceUtc, false, T0.AddMinutes(10), Dwell);
|
||||
Assert.Null(round2.UpdatedSidecarMissingSinceUtc);
|
||||
Assert.Null(round2.SidecarMissingStalledForMinutes);
|
||||
|
||||
// Sidecar gone again much later → clock restarts from the new now, no instant stall.
|
||||
var round3 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)),
|
||||
round2.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round2.UpdatedSidecarMissingSinceUtc, false, T0.AddHours(3), Dwell);
|
||||
Assert.Equal(T0.AddHours(3), round3.UpdatedSidecarMissingSinceUtc);
|
||||
Assert.Null(round3.SidecarMissingStalledForMinutes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sidecar_missing_one_shot_and_deviceprep_only_scoping()
|
||||
{
|
||||
// Already fired → never re-reported, but the clock keeps running.
|
||||
var latched = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)),
|
||||
new Dictionary<string, DateTime>(), new HashSet<string>(), T0, true, T0.AddHours(5), Dwell);
|
||||
Assert.Null(latched.SidecarMissingStalledForMinutes);
|
||||
Assert.Equal(T0, latched.UpdatedSidecarMissingSinceUtc);
|
||||
|
||||
// DevicePreparation providers alone don't constitute the Setup/Apps wait — arm 2 off.
|
||||
var devicePrepOnly = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(DevicePrep("ConfigMgr", installState: 1)),
|
||||
new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Null(devicePrepOnly.UpdatedSidecarMissingSinceUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -243,7 +313,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
Assert.Equal(EspPolicyProviderStallDetector.SourceLabel, evt.Payload[SignalPayloadKeys.Source]);
|
||||
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(evt.TypedPayload);
|
||||
// Both arms cross in the same round (ConfigMgr incomplete AND Sidecar absent) →
|
||||
// ONE event; sidecar-missing is the dominant reason.
|
||||
Assert.Equal(EspPolicyProviderStallDetector.ReasonSidecarMissing, data["reason"]);
|
||||
Assert.Equal(false, data["sidecarRegistered"]);
|
||||
Assert.Equal(15.0, data["sidecarMissingForMinutes"]);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.DwellMinutes, data["dwellMinutes"]);
|
||||
var stalled = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object>>>(data["stalledProviders"]);
|
||||
var entry = Assert.Single(stalled);
|
||||
@@ -252,6 +326,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
Assert.Equal(15.0, entry["stalledForMinutes"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Issue106_configmgr_complete_without_sidecar_still_emits_sidecar_missing()
|
||||
{
|
||||
// Regression test for the customer's follow-up on issue #106: TrackingPoliciesCreated=1
|
||||
// WAS present under ConfigMgr — the provider looks "complete", arm 1 never fires, yet
|
||||
// the ESP hangs because Sidecar is absent. Arm 2 must catch exactly this.
|
||||
var configMgrComplete = new EspPolicyProviderSnapshot(new[]
|
||||
{
|
||||
new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps,
|
||||
trackingPoliciesCreated: 1, installationState: null),
|
||||
});
|
||||
using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => configMgrComplete);
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
|
||||
sut.Tick();
|
||||
Assert.Empty(StalledEvents(sink));
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
|
||||
var evt = Assert.Single(StalledEvents(sink));
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(evt.TypedPayload);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.ReasonSidecarMissing, data["reason"]);
|
||||
Assert.Equal(false, data["sidecarRegistered"]);
|
||||
Assert.Equal(15.0, data["sidecarMissingForMinutes"]);
|
||||
Assert.Empty(Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object>>>(data["stalledProviders"]));
|
||||
var providers = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object?>>>(data["providers"]);
|
||||
Assert.Equal(true, Assert.Single(providers)["complete"]); // the misleading "complete" is visible
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sidecar_complete_configmgr_incomplete_reports_only_configmgr_with_sidecar_flag()
|
||||
{
|
||||
@@ -264,7 +368,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
var evt = Assert.Single(StalledEvents(sink));
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(evt.TypedPayload);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.ReasonProviderIncomplete, data["reason"]); // arm 2 off — Sidecar is there
|
||||
Assert.Equal(true, data["sidecarRegistered"]);
|
||||
Assert.False(data.ContainsKey("sidecarMissingForMinutes"));
|
||||
var stalled = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object>>>(data["stalledProviders"]);
|
||||
Assert.Equal("ConfigMgr", Assert.Single(stalled)["name"]);
|
||||
var providers = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object?>>>(data["providers"]);
|
||||
|
||||
+34
@@ -161,6 +161,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
Assert.False(string.IsNullOrEmpty(snapshot.KekStatus));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SeSystemEnvironmentPrivilege is adjusted on the PROCESS token, so the enable →
|
||||
/// read → restore sequence is a process-wide side effect. Unserialized, a second
|
||||
/// reader observes "already enabled", skips its own restore, and then loses the
|
||||
/// privilege mid-read when the first reader disposes — surfacing as a privilege_denied
|
||||
/// that disagrees with the other readers. The machine's actual privilege state does
|
||||
/// not matter here: whatever it is, every reader must report the SAME status.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Read_is_serialized_across_concurrent_callers()
|
||||
{
|
||||
const int readers = 8;
|
||||
var snapshots = new UefiSecureBootCertSnapshot[readers];
|
||||
var failures = new System.Exception[readers];
|
||||
|
||||
var threads = new System.Threading.Thread[readers];
|
||||
for (var i = 0; i < readers; i++)
|
||||
{
|
||||
var index = i;
|
||||
threads[i] = new System.Threading.Thread(() =>
|
||||
{
|
||||
try { snapshots[index] = UefiSecureBootCertReader.Read(); }
|
||||
catch (System.Exception ex) { failures[index] = ex; }
|
||||
});
|
||||
threads[i].Start();
|
||||
}
|
||||
foreach (var thread in threads) thread.Join();
|
||||
|
||||
Assert.All(failures, Assert.Null);
|
||||
Assert.All(snapshots, s => Assert.False(string.IsNullOrEmpty(s.DbStatus)));
|
||||
Assert.All(snapshots, s => Assert.Equal(snapshots[0].DbStatus, s.DbStatus));
|
||||
Assert.All(snapshots, s => Assert.Equal(snapshots[0].KekStatus, s.KekStatus));
|
||||
}
|
||||
|
||||
private static byte[] Concat(params byte[][] parts)
|
||||
{
|
||||
var total = 0;
|
||||
|
||||
+36
@@ -37,5 +37,41 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
Assert.False(DeviceInfoHost.IsEndTrigger(DecisionSignalKind.EspPhaseChanged, null));
|
||||
Assert.False(DeviceInfoHost.IsEndTrigger(DecisionSignalKind.InformationalEvent, Phase("FinalizingSetup")));
|
||||
}
|
||||
|
||||
// Audit Q2 — the payload shape the adapter's apps sub-phase declaration produces on the
|
||||
// ingress rail (InformationalEventPost: eventType + declared phase enum name).
|
||||
private static IReadOnlyDictionary<string, string> SubPhaseDeclaration(string declaredPhase) =>
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ SignalPayloadKeys.EventType, AutopilotMonitor.Shared.Constants.EventTypes.PhaseTransition },
|
||||
{ AutopilotMonitor.Agent.V2.Core.Telemetry.Events.EventTimelineEmitter.PhaseParamKey, declaredPhase },
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Apps_subphase_declarations_classify_as_esp_config_refresh_triggers()
|
||||
{
|
||||
Assert.Equal(DeviceInfoHost.EspConfigTriggerAppsDevice,
|
||||
DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.InformationalEvent, SubPhaseDeclaration("AppsDevice")));
|
||||
Assert.Equal(DeviceInfoHost.EspConfigTriggerAppsUser,
|
||||
DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.InformationalEvent, SubPhaseDeclaration("AppsUser")));
|
||||
// Non-apps phase declarations (e.g. the reducer's Finalizing entry) never classify.
|
||||
Assert.Null(DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.InformationalEvent, SubPhaseDeclaration("FinalizingSetup")));
|
||||
// A phase_transition without a declared phase never classifies.
|
||||
Assert.Null(DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.InformationalEvent,
|
||||
new Dictionary<string, string> { { SignalPayloadKeys.EventType, AutopilotMonitor.Shared.Constants.EventTypes.PhaseTransition } }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AccountSetup_phase_classifies_as_the_user_setup_refresh_trigger()
|
||||
{
|
||||
Assert.Equal(DeviceInfoHost.EspConfigTriggerAccountSetup,
|
||||
DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.EspPhaseChanged, Phase("AccountSetup")));
|
||||
// DeviceSetup is the enrollment-start trigger, not an ESP-config refresh.
|
||||
Assert.Null(DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.EspPhaseChanged, Phase("DeviceSetup")));
|
||||
Assert.Null(DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.EspPhaseChanged, null));
|
||||
// Unrelated informational events never classify.
|
||||
Assert.Null(DeviceInfoHost.ClassifyEspConfigRefreshTrigger(DecisionSignalKind.InformationalEvent,
|
||||
new Dictionary<string, string> { { SignalPayloadKeys.EventType, "esp_config_detected" } }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
{
|
||||
/// <summary>
|
||||
/// Audit Q5 — WMI hardware values are trimmed at the source. BIOS serials become device
|
||||
/// identity keys downstream (F2 device history joins on TenantId + serial), so padding
|
||||
/// must never leave the agent; whitespace-only values collapse to the "Unknown" sentinel
|
||||
/// the retry logic already treats as unresolved.
|
||||
/// </summary>
|
||||
public sealed class HardwareInfoNormalizeTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(" PF60H941 ", "PF60H941")]
|
||||
[InlineData("PF60H941", "PF60H941")]
|
||||
[InlineData("\tLENOVO \r\n", "LENOVO")]
|
||||
[InlineData("Unknown", "Unknown")]
|
||||
public void Trims_whitespace_at_the_source(string raw, string expected)
|
||||
=> Assert.Equal(expected, HardwareInfo.NormalizeWmiValue(raw));
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("\t\r\n")]
|
||||
public void Empty_and_whitespace_only_collapse_to_the_Unknown_sentinel(string raw)
|
||||
=> Assert.Equal("Unknown", HardwareInfo.NormalizeWmiValue(raw));
|
||||
|
||||
[Fact]
|
||||
public void Null_collapses_to_the_Unknown_sentinel()
|
||||
=> Assert.Equal("Unknown", HardwareInfo.NormalizeWmiValue(null));
|
||||
}
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
{
|
||||
/// <summary>
|
||||
/// Historic-replay suppression for APP events (session eaf3d8c4 part 2): 147 of 157 app
|
||||
/// events in that session were replayed from a previous enrollment's IME log — all 10
|
||||
/// "installed apps" were phantoms. Stale app-state transitions and DO telemetry must be
|
||||
/// suppressed BEFORE the sub-phase declaration, the timing bookkeeping and the terminal
|
||||
/// DecisionSignal dedup, so fire-once flags stay unconsumed and no phantom
|
||||
/// AppInstallCompleted/Failed reaches the engine (which could spuriously re-arm the
|
||||
/// AdvisoryCompletion window). The script twin lives in
|
||||
/// <see cref="ImeLogTrackerAdapterHistoricReplayTests"/>.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerAdapterHistoricAppReplayTests
|
||||
{
|
||||
private static readonly DateTime ClockNow = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Stale_terminal_app_transition_is_fully_suppressed()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-old", 0);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Empty(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted));
|
||||
// Shadow download_progress (status=completed) and the summary snapshot ride the
|
||||
// same emit — both must be gone too.
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.DownloadProgress));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.AppTrackingSummary));
|
||||
// Timing bookkeeping must not be poisoned (feeds FinalStatusBuilder).
|
||||
Assert.False(adapter.AppTimings.ContainsKey("app-old"));
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_run_after_suppressed_stale_run_fires_signal_timing_and_subphase()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-X", 0);
|
||||
|
||||
// Fresh ESP phase so the sub-phase declaration is armed.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-5);
|
||||
adapter.TriggerEspPhaseFromTest("DeviceSetup");
|
||||
|
||||
// Stale replay of the same app — suppressed, must consume NEITHER the sub-phase
|
||||
// fire-once NOR the terminal-signal dedup for this appId.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Unknown, AppInstallationState.Installed);
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.PhaseTransition));
|
||||
Assert.Empty(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted));
|
||||
|
||||
// The app genuinely runs in THIS enrollment.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Unknown, AppInstallationState.Installing);
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-1);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.PhaseTransition)); // sub-phase fired fresh
|
||||
Assert.Single(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted)); // signal fired fresh
|
||||
var timing = adapter.AppTimings["app-X"];
|
||||
Assert.Equal(ClockNow.AddMinutes(-2), timing.StartedAtUtc); // fresh stamps, not stale
|
||||
Assert.Equal(ClockNow.AddMinutes(-1), timing.CompletedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_do_telemetry_is_suppressed_fresh_emits()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-do", 0);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerDoTelemetryFromTest(app);
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.DoTelemetry));
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-1);
|
||||
adapter.TriggerDoTelemetryFromTest(app);
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.DoTelemetry));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Oneshot_summary_is_shared_between_script_and_app_suppressions()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "aaa11111",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
adapter.TriggerAppStateFromTest(new AppPackageState("app-old", 0),
|
||||
AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FutureSkew_app_transition_still_emits()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-skew", 0);
|
||||
|
||||
// Mid-enrollment clock jump — genuine current activity, stamped to the clock.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddHours(2);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Equal(ClockNow, info.OccurredAtUtc);
|
||||
Assert.Equal("true", info.Payload!["derivedTimestamp"]);
|
||||
Assert.Single(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimulationMode_bypasses_suppression_for_apps_and_scripts()
|
||||
{
|
||||
// The --replay-log-dir dev tool intentionally feeds historic lines.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
f.Tracker.SimulationMode = true;
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerAppStateFromTest(new AppPackageState("app-sim", 0),
|
||||
AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "bbb22222",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
{
|
||||
/// <summary>
|
||||
/// Historic-replay handling (session eaf3d8c4): a previous enrollment's IME log surviving
|
||||
/// on disk made the agent replay week-old script activity — 156 phantom script_completed
|
||||
/// events with ~170 h durations (clock-clamped completion minus RAW stale start) and an
|
||||
/// immediate-upload flood. Script events whose source line is > 24 h stale are now
|
||||
/// suppressed entirely (one-shot <c>historic_ime_replay_detected</c> summary instead),
|
||||
/// and durations are computed from a timeline-consistent timestamp pair with a 24 h
|
||||
/// plausibility backstop.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerAdapterHistoricReplayTests
|
||||
{
|
||||
private static readonly DateTime ClockNow = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Suppression: stale (> 24 h past) source lines are a previous enrollment
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Stale_completion_is_suppressed_and_emits_oneshot_summary()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
var ancient = ClockNow.AddDays(-7);
|
||||
f.Tracker.LastMatchedLogTimestamp = ancient;
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "d78c1822",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Failed",
|
||||
StartedAtUtc = ancient.AddMinutes(-30),
|
||||
// Bootstrap marker in stdout — week-old evidence must not drive the
|
||||
// bootstrap-detected trace either.
|
||||
Stdout = "Bootstrap script version: v2.0",
|
||||
});
|
||||
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptFailed));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.AgentTrace));
|
||||
|
||||
var summary = f.InfoEvent(SharedEventTypes.HistoricImeReplayDetected);
|
||||
Assert.Equal(ancient.ToString("o"), summary.Payload!["earliestRejectedSourceTimestamp"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Summary_is_oneshot_across_started_and_completed_suppressions()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerScriptStartedFromTest(new ScriptStartedInfo { PolicyId = "aaa11111", ScriptType = "platform" });
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7).AddMinutes(1);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "aaa11111",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7).AddMinutes(2);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "bbb22222",
|
||||
ScriptType = "remediation",
|
||||
ScriptPart = "detection",
|
||||
ExitCode = 0,
|
||||
ComplianceResult = "True",
|
||||
});
|
||||
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptStarted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
// Exactly ONE summary regardless of how many replayed lines followed — and it dates
|
||||
// the replay window from the FIRST (earliest) suppressed line.
|
||||
var summary = Assert.Single(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
Assert.Equal(ClockNow.AddDays(-7).ToString("o"), summary.Payload!["earliestRejectedSourceTimestamp"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_events_after_suppression_emit_normally()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "aaa11111",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
|
||||
// The agent catches up to current log content — normal emission resumes.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "ccc33333",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
StartedAtUtc = ClockNow.AddMinutes(-3),
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Equal("ccc33333", info.Payload!["policyId"]);
|
||||
Assert.Equal("60.00", info.Payload["durationSeconds"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clock_fallback_without_source_timestamp_still_emits()
|
||||
{
|
||||
// derivedFromClock with NO rejected source ts (synthetic/callback path) is not a
|
||||
// replay — must emit, and must not trigger the summary.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "ddd44444",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Equal("true", info.Payload!["derivedTimestamp"]);
|
||||
Assert.False(info.Payload.ContainsKey("rejectedSourceTimestamp"));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Duration: timeline-consistent pair + plausibility backstop
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void FutureSkew_completion_emits_with_clamped_stamp_and_raw_pair_duration()
|
||||
{
|
||||
// Mid-enrollment clock jump (WhiteGlove +1h CMTrace skew case): the source ts is
|
||||
// rejected as future-skewed — NOT suppressed — and the duration must come from the
|
||||
// raw source pair (both ends on the same skewed timeline), not clamped-now − raw-start.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
var skewed = ClockNow.AddHours(2);
|
||||
f.Tracker.LastMatchedLogTimestamp = skewed;
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "eee55555",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
StartedAtUtc = skewed.AddSeconds(-360),
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Equal(ClockNow, info.OccurredAtUtc); // stamp: clamped to clock
|
||||
Assert.Equal("true", info.Payload!["derivedTimestamp"]);
|
||||
Assert.Equal(skewed.ToString("o"), info.Payload["rejectedSourceTimestamp"]);
|
||||
Assert.Equal("360.00", info.Payload["durationSeconds"]); // duration: raw pair
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FutureSkew_failed_platform_script_timeout_heuristic_uses_raw_pair()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
var skewed = ClockNow.AddHours(2);
|
||||
f.Tracker.LastMatchedLogTimestamp = skewed;
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "fff66666",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Failed",
|
||||
StartedAtUtc = skewed.AddMinutes(-26), // past the 25-min suspicion threshold
|
||||
});
|
||||
|
||||
var suspected = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected));
|
||||
Assert.Equal("1560.00", suspected.Payload!["durationSeconds"]);
|
||||
Assert.Equal(ClockNow, suspected.OccurredAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Implausible_duration_above_24h_is_omitted()
|
||||
{
|
||||
// Fresh (accepted) completion paired with an ancient start — e.g. a stale pending
|
||||
// slot a legacy path left behind. The duration would be a cross-run lie; omit it.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2);
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "abc12345",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Failed",
|
||||
StartedAtUtc = ClockNow.AddHours(-30),
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptFailed));
|
||||
Assert.False(info.Payload!.ContainsKey("durationSeconds"));
|
||||
Assert.False(info.Payload.ContainsKey("durationBasis"));
|
||||
// The 30 h "duration" must not read as a hung script either.
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tracker slot hardening: stale pending start must not pair with a fresh run
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Fresh_start_discards_stale_pending_slot_from_previous_enrollment()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
// Previous enrollment: replayed start line whose matching result never appeared
|
||||
// (the script was mid-run when that enrollment ended).
|
||||
f.Tracker.SeedPendingPlatformScriptForTesting("beef7777", exitCode: null, exitObservedAtUtc: null,
|
||||
startedAtUtc: ClockNow.AddDays(-7));
|
||||
|
||||
// Same policy genuinely re-runs now — the fresh start must begin a NEW execution.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddSeconds(-90);
|
||||
f.Tracker.HandlePlatformScriptStarted("beef7777");
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow;
|
||||
f.Tracker.CompletePlatformScriptFromImeResultForTesting("beef7777", "Success", ClockNow);
|
||||
|
||||
var info = f.InfoEvent(SharedEventTypes.ScriptCompleted);
|
||||
// Without the hardening the slot kept the 7-day-old start: the duration would have
|
||||
// been dropped by the 24 h backstop (or, before that, shown as ~168 h).
|
||||
Assert.Equal("90.00", info.Payload!["durationSeconds"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
+82
@@ -243,5 +243,87 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
|
||||
Assert.Equal(firstStamp, adapter.AppTimings["app-P"].StartedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Terminal_to_active_retry_rearms_CompletedAt_and_final_terminal_covers_total_occupancy()
|
||||
{
|
||||
// Audit Q1: the IME retry path (Error -> Downloading -> Installed is permitted by
|
||||
// AppPackageState.UpdateState). Pre-fix, CompletedAt froze at the first Error, so
|
||||
// the final Installed payload carried the first failure's completedAt/duration.
|
||||
using var f = new ImeLogTrackerAdapterFixture(T0);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-R", 0);
|
||||
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Unknown, AppInstallationState.Installing);
|
||||
f.Clock.Advance(TimeSpan.FromSeconds(5));
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Error);
|
||||
f.Clock.Advance(TimeSpan.FromSeconds(10));
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Error, AppInstallationState.Downloading);
|
||||
|
||||
// Re-armed while active again: the retry's download event carries no completedAt,
|
||||
// and the FIRST start stamp is kept.
|
||||
Assert.Null(adapter.AppTimings["app-R"].CompletedAtUtc);
|
||||
var retryDownload = f.InfoEvent(SharedEventTypes.AppDownloadStarted);
|
||||
Assert.False(retryDownload.Payload!.ContainsKey("completedAt"));
|
||||
Assert.False(retryDownload.Payload.ContainsKey("durationSeconds"));
|
||||
Assert.Equal(T0.ToString("o"), retryDownload.Payload["startedAt"]);
|
||||
|
||||
f.Clock.Advance(TimeSpan.FromSeconds(20));
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Downloading, AppInstallationState.Installed);
|
||||
|
||||
var timing = adapter.AppTimings["app-R"];
|
||||
Assert.Equal(T0, timing.StartedAtUtc);
|
||||
Assert.Equal(T0.AddSeconds(35), timing.CompletedAtUtc);
|
||||
Assert.Equal(35.0, timing.DurationSeconds);
|
||||
|
||||
var completed = Assert.Single(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Equal("35.00", completed.Payload!["durationSeconds"]);
|
||||
Assert.Equal(T0.AddSeconds(35).ToString("o"), completed.Payload["completedAt"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Skipped_then_retried_app_emits_positive_duration_from_retry_start()
|
||||
{
|
||||
// Production regression (session 66f1c5e1, audit Q1): a user-scope app evaluated
|
||||
// Skipped during DeviceSetup (terminal, no start observed), retried after sign-in
|
||||
// (Skipped -> Downloading is permitted), Installed later. Pre-fix payload:
|
||||
// completedAt frozen at the early Skip + startedAt from the retry ->
|
||||
// durationSeconds=-2342. Post-fix: retry re-arms, duration covers retry-start ->
|
||||
// final terminal.
|
||||
using var f = new ImeLogTrackerAdapterFixture(T0);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-S", 0);
|
||||
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Unknown, AppInstallationState.Skipped);
|
||||
Assert.Null(adapter.AppTimings["app-S"].StartedAtUtc);
|
||||
Assert.Equal(T0, adapter.AppTimings["app-S"].CompletedAtUtc);
|
||||
|
||||
f.Clock.Advance(TimeSpan.FromSeconds(60));
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Skipped, AppInstallationState.Downloading);
|
||||
f.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Downloading, AppInstallationState.Installed);
|
||||
|
||||
var timing = adapter.AppTimings["app-S"];
|
||||
Assert.Equal(T0.AddSeconds(60), timing.StartedAtUtc);
|
||||
Assert.Equal(T0.AddSeconds(90), timing.CompletedAtUtc);
|
||||
Assert.Equal(30.0, timing.DurationSeconds);
|
||||
|
||||
var completedEvents = f.InfoEvents(SharedEventTypes.AppInstallComplete);
|
||||
Assert.Equal(2, completedEvents.Count);
|
||||
Assert.Equal("30.00", completedEvents[1].Payload!["durationSeconds"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Inverted_endpoints_yield_null_DurationSeconds()
|
||||
{
|
||||
// Negative guard (audit Q1): endpoints stamped from out-of-order source lines must
|
||||
// never surface as a negative durationSeconds (observed -2342s in production). The
|
||||
// payload emitter omits the field when DurationSeconds is null.
|
||||
var inverted = new AppInstallTiming(startedAtUtc: T0.AddSeconds(100), completedAtUtc: T0);
|
||||
Assert.Null(inverted.DurationSeconds);
|
||||
|
||||
var wellOrdered = new AppInstallTiming(startedAtUtc: T0, completedAtUtc: T0.AddSeconds(100));
|
||||
Assert.Equal(100.0, wellOrdered.DurationSeconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+79
@@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Termination;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
@@ -70,5 +72,82 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Equal(0, bootToStart, 0);
|
||||
Assert.Equal(10, uptime, 0);
|
||||
}
|
||||
|
||||
// --- DescribeLateStart: outcome-calibrated severity + phrasing (fleet 2026-07-28) ---
|
||||
|
||||
private static void Describe(
|
||||
EnrollmentTerminationOutcome outcome, string oobeState,
|
||||
out EventSeverity severity, out string message, out string note)
|
||||
=> ObservationCoverage.DescribeLateStart(
|
||||
outcome, oobeState, bootToStartSeconds: 34 * 60, uptimeSeconds: 30,
|
||||
out severity, out message, out note);
|
||||
|
||||
[Fact]
|
||||
public void Failed_outcome_is_a_warning_with_the_hung_script_post_mortem_note()
|
||||
{
|
||||
Describe(EnrollmentTerminationOutcome.Failed, "in_progress",
|
||||
out var severity, out var message, out var note);
|
||||
|
||||
Assert.Equal(EventSeverity.Warning, severity);
|
||||
Assert.Contains("low coverage of the enrollment window", message);
|
||||
Assert.Contains("post-mortem", note);
|
||||
Assert.Contains("hung ahead of the bootstrap", note);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimedOut_outcome_is_a_warning_like_a_failure()
|
||||
{
|
||||
Describe(EnrollmentTerminationOutcome.TimedOut, "in_progress",
|
||||
out var severity, out _, out var note);
|
||||
|
||||
Assert.Equal(EventSeverity.Warning, severity);
|
||||
Assert.Contains("post-mortem", note);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Succeeded_outcome_is_an_info_about_tail_only_coverage_without_alarm_framing()
|
||||
{
|
||||
// The 659c3a90 shape: healthy enrollment, bootstrap merely ran late in the ESP queue.
|
||||
Describe(EnrollmentTerminationOutcome.Succeeded, "in_progress",
|
||||
out var severity, out _, out var note);
|
||||
|
||||
Assert.Equal(EventSeverity.Info, severity);
|
||||
Assert.Contains("succeeded", note);
|
||||
Assert.Contains("IME log replay", note);
|
||||
Assert.DoesNotContain("post-mortem", note);
|
||||
Assert.DoesNotContain("hung", note);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Oobe_completed_at_start_phrases_a_pure_post_mortem()
|
||||
{
|
||||
Describe(EnrollmentTerminationOutcome.Failed, "completed",
|
||||
out _, out var message, out _);
|
||||
|
||||
Assert.Contains("OOBE was already completed when the agent started", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Oobe_in_progress_at_start_phrases_the_live_tail()
|
||||
{
|
||||
Describe(EnrollmentTerminationOutcome.Failed, "in_progress",
|
||||
out _, out var message, out _);
|
||||
|
||||
Assert.Contains("OOBE was still in progress at agent start", message);
|
||||
Assert.Contains("final 30s live", message);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not_started")] // flips in after a mid-OOBE reboot — not interpretable
|
||||
[InlineData("unavailable")] // WinRT contract absent / read failed
|
||||
[InlineData("unknown_3")] // unexpected enum value
|
||||
public void Uninterpretable_oobe_states_add_no_claim_to_the_message(string oobeState)
|
||||
{
|
||||
Describe(EnrollmentTerminationOutcome.Failed, oobeState,
|
||||
out _, out var message, out _);
|
||||
|
||||
Assert.DoesNotContain("OOBE", message);
|
||||
Assert.EndsWith("low coverage of the enrollment window.", message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+10
-5
@@ -10,9 +10,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// the <c>app_tracking_summary</c> terminal event.
|
||||
/// <para>
|
||||
/// <b>StartedAtUtc</b> is recorded on the first lifecycle transition into Downloading /
|
||||
/// Installing / InProgress. <b>CompletedAtUtc</b> is recorded on the first transition to a
|
||||
/// terminal state (Installed / Skipped / Postponed / Error). Both are "set-once" — subsequent
|
||||
/// identical-state events do not overwrite an earlier stamp.
|
||||
/// Installing / InProgress and never overwritten. <b>CompletedAtUtc</b> is recorded on a
|
||||
/// transition to a terminal state (Installed / Skipped / Postponed / Error) and re-armed
|
||||
/// to <c>null</c> when the app goes terminal → active again (IME retry / re-evaluation,
|
||||
/// audit Q1) — so the final terminal event carries first-start → last-terminal.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class AppInstallTiming
|
||||
@@ -30,10 +31,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// <summary>
|
||||
/// Elapsed time from first Downloading/Installing transition to terminal state, in
|
||||
/// seconds. <c>null</c> when either endpoint is missing (e.g. app still installing,
|
||||
/// or the adapter missed the start event because it came online mid-install).
|
||||
/// or the adapter missed the start event because it came online mid-install) — and
|
||||
/// <c>null</c> when the endpoints are inverted (CompletedAt before StartedAt: log lines
|
||||
/// from different source files can be processed out of chronological order; observed in
|
||||
/// production as durationSeconds=-2342 on a Skipped-then-retried user-scope app,
|
||||
/// audit Q1). A negative duration is never a real measurement, so it is not emitted.
|
||||
/// </summary>
|
||||
public double? DurationSeconds =>
|
||||
StartedAtUtc.HasValue && CompletedAtUtc.HasValue
|
||||
StartedAtUtc.HasValue && CompletedAtUtc.HasValue && CompletedAtUtc.Value >= StartedAtUtc.Value
|
||||
? (CompletedAtUtc.Value - StartedAtUtc.Value).TotalSeconds
|
||||
: (double?)null;
|
||||
}
|
||||
|
||||
+51
-30
@@ -208,39 +208,60 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
}
|
||||
else
|
||||
{
|
||||
// Platform script: AgentExecutor.log entries create/enrich, IME log entries also create
|
||||
if (!_pendingPlatformScripts.ContainsKey(id))
|
||||
{
|
||||
// A fresh start for this policy (no pending entry) begins a NEW execution.
|
||||
// Clear any emitted-marker from a prior run so IME re-evaluations / retries of
|
||||
// the same platform-script policy within one agent lifetime are not silently
|
||||
// deduped away. Within a single run the start always precedes exit/result, so
|
||||
// this never clears the current run's own marker.
|
||||
_platformScriptResultEmitted.Remove(id);
|
||||
_pendingPlatformScripts[id] = new ScriptExecutionState
|
||||
{
|
||||
PolicyId = id,
|
||||
ScriptType = "platform",
|
||||
// Prefer the source CMTrace timestamp so a script that started before the
|
||||
// agent launched (replayed log content) is dated to its real start, not now.
|
||||
// Set only at slot creation: the start line fires twice (agentexecutor + ime
|
||||
// source) but the earliest wins.
|
||||
StartedAtUtc = LastMatchedLogTimestamp ?? DateTime.UtcNow,
|
||||
};
|
||||
// Live "running" indicator — same signal health scripts emit. Gated on slot
|
||||
// creation so the duplicate start line (agentexecutor + ime source) doesn't
|
||||
// double-emit for the same execution.
|
||||
try { OnScriptStarted?.Invoke(new ScriptStartedInfo { PolicyId = id, ScriptType = "platform" }); }
|
||||
catch (Exception ex) { _logger.Warning($"ImeLogTracker: OnScriptStarted handler threw: {ex.Message}"); }
|
||||
}
|
||||
_lastPlatformScriptPolicyId = id;
|
||||
// Started lines fire twice per script (agentexecutor + ime source) and carry no
|
||||
// outcome — the matching `platform script completed` line below carries result+exit
|
||||
// and stays on Info. Keep starts on Debug so Info reflects script outcomes only.
|
||||
_logger.Debug($"ImeLogTracker: platform script started: {id} (source: {source ?? "ime"})");
|
||||
HandlePlatformScriptStarted(id, source);
|
||||
}
|
||||
}
|
||||
|
||||
// Platform script: AgentExecutor.log entries create/enrich, IME log entries also create.
|
||||
// Internal so the stale-slot hardening is directly testable (the public path arrives via
|
||||
// a regex Match).
|
||||
internal void HandlePlatformScriptStarted(string id, string source = null)
|
||||
{
|
||||
// Stale-slot hardening (session eaf3d8c4): a replayed start line from a previous
|
||||
// enrollment whose matching result never appeared (script was mid-run when that
|
||||
// enrollment ended) leaves a pending slot with a days-old StartedAtUtc. When the
|
||||
// same policy genuinely re-runs now, the fresh start must begin a NEW execution —
|
||||
// otherwise the ancient start pairs with the fresh completion and the duration
|
||||
// lies. > 24 h mirrors the adapter's source-timestamp staleness clamp.
|
||||
var currentStartTs = LastMatchedLogTimestamp ?? DateTime.UtcNow;
|
||||
if (_pendingPlatformScripts.TryGetValue(id, out var pendingSlot)
|
||||
&& pendingSlot.StartedAtUtc.HasValue
|
||||
&& currentStartTs - pendingSlot.StartedAtUtc.Value > TimeSpan.FromHours(24))
|
||||
{
|
||||
_logger.Debug($"ImeLogTracker: discarding stale pending platform-script slot for {id} (started {pendingSlot.StartedAtUtc:o})");
|
||||
_pendingPlatformScripts.Remove(id);
|
||||
}
|
||||
if (!_pendingPlatformScripts.ContainsKey(id))
|
||||
{
|
||||
// A fresh start for this policy (no pending entry) begins a NEW execution.
|
||||
// Clear any emitted-marker from a prior run so IME re-evaluations / retries of
|
||||
// the same platform-script policy within one agent lifetime are not silently
|
||||
// deduped away. Within a single run the start always precedes exit/result, so
|
||||
// this never clears the current run's own marker.
|
||||
_platformScriptResultEmitted.Remove(id);
|
||||
_pendingPlatformScripts[id] = new ScriptExecutionState
|
||||
{
|
||||
PolicyId = id,
|
||||
ScriptType = "platform",
|
||||
// Prefer the source CMTrace timestamp so a script that started before the
|
||||
// agent launched (replayed log content) is dated to its real start, not now.
|
||||
// Set only at slot creation: the start line fires twice (agentexecutor + ime
|
||||
// source) but the earliest wins.
|
||||
StartedAtUtc = currentStartTs,
|
||||
};
|
||||
// Live "running" indicator — same signal health scripts emit. Gated on slot
|
||||
// creation so the duplicate start line (agentexecutor + ime source) doesn't
|
||||
// double-emit for the same execution.
|
||||
try { OnScriptStarted?.Invoke(new ScriptStartedInfo { PolicyId = id, ScriptType = "platform" }); }
|
||||
catch (Exception ex) { _logger.Warning($"ImeLogTracker: OnScriptStarted handler threw: {ex.Message}"); }
|
||||
}
|
||||
_lastPlatformScriptPolicyId = id;
|
||||
// Started lines fire twice per script (agentexecutor + ime source) and carry no
|
||||
// outcome — the matching `platform script completed` line below carries result+exit
|
||||
// and stays on Info. Keep starts on Debug so Info reflects script outcomes only.
|
||||
_logger.Debug($"ImeLogTracker: platform script started: {id} (source: {source ?? "ime"})");
|
||||
}
|
||||
|
||||
// Routes line-by-line script-data handlers (context / exitCode / output) to the right
|
||||
// accumulator based on the pattern's scriptType parameter. Health-script lines feed the
|
||||
// single-slot _pendingHealthScript; platform-script lines feed _pendingPlatformScripts.
|
||||
|
||||
+34
@@ -194,6 +194,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
}
|
||||
}
|
||||
|
||||
// Actions that mutate app/phase tracking state (_packageStates, _phasePackageSnapshots,
|
||||
// _currentPhaseOrder, DO telemetry) — the historic-replay guard skips these for source
|
||||
// lines from a previous enrollment so replayed apps never enter tracked state,
|
||||
// persistence, app_tracking_summary, culprit lists or final-status. Script actions are
|
||||
// deliberately NOT here: their tracker state is harmless (stale-slot hardening covers
|
||||
// leftovers) and the adapter suppresses their emissions. espphasedetected IS here — a
|
||||
// stale "In EspPhase: AccountSetup" would advance _currentPhaseOrder and make the fresh
|
||||
// enrollment's DeviceSetup lines bounce as "backward"; IME re-logs the phase constantly,
|
||||
// so fresh lines re-deliver it immediately.
|
||||
private static readonly HashSet<string> AppMutatingActions = new HashSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
"imeshutdown", "espphasedetected", "setcurrentapp",
|
||||
"updatestateinstalled", "updatestatedownloading", "updatestateinstalling",
|
||||
"updatestateskipped", "updatestateerror", "updatestatepostponed",
|
||||
"captureexitcode", "capturehresult", "captureappversion",
|
||||
"captureapptypewinget", "captureapptypemsi", "captureattemptnumber",
|
||||
"capturedetectionresult", "esptrackstatus", "policiesdiscovered",
|
||||
"ignorecompletedapp", "updatename", "updatewin32appstate",
|
||||
"cancelstuckandsetcurrent", "updatedotelemetry",
|
||||
};
|
||||
|
||||
private void HandlePatternMatch(CompiledPattern pattern, Match match, string message, CmTraceLogEntry entry)
|
||||
{
|
||||
LastMatchedPatternId = pattern.PatternId;
|
||||
@@ -206,6 +227,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
try { OnPatternMatched?.Invoke(pattern.PatternId); }
|
||||
catch (Exception ex) { _logger?.Warning($"OnPatternMatched handler threw: {ex.Message}"); }
|
||||
|
||||
// Historic-replay guard (session eaf3d8c4): a source line > 24 h older than now is
|
||||
// content from a previous enrollment whose IME log survived on disk. App-mutating
|
||||
// actions are skipped BEFORE _seenAppIds so replayed apps poison neither the tracked
|
||||
// state nor the phase-change ignore list. SimulationMode (--replay-log-dir dev tool)
|
||||
// replays historic logs on purpose and bypasses the guard.
|
||||
var isStaleReplayLine = !SimulationMode && entry != null
|
||||
&& (UtcNowProvider() - NormalizeUtc(entry.Timestamp)) > HistoricReplayThreshold;
|
||||
if (isStaleReplayLine && AppMutatingActions.Contains(pattern.Action?.ToLower() ?? string.Empty))
|
||||
{
|
||||
_logger.Debug($"ImeLogTracker: skipped app action '{pattern.Action}' for historic line ({entry.Timestamp:o})");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var id = match.Groups["id"]?.Value;
|
||||
|
||||
@@ -71,6 +71,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
public double SpeedFactor { get; set; } = 50;
|
||||
private DateTime _lastLogTimestamp = DateTime.MinValue;
|
||||
|
||||
// Source log lines older than this relative to now are content from a previous
|
||||
// enrollment (IME logs surviving a re-enrollment, session eaf3d8c4). Single source of
|
||||
// truth shared with the adapter's ResolveOccurredAt clamp and replay suppression.
|
||||
internal static readonly TimeSpan HistoricReplayThreshold = TimeSpan.FromHours(24);
|
||||
|
||||
// Deterministic-time seam for the historic-replay guard; production default is the
|
||||
// system clock, ImeLogHost rewires it to the agent IClock, tests to their VirtualClock.
|
||||
internal Func<DateTime> UtcNowProvider { get; set; } = () => DateTime.UtcNow;
|
||||
|
||||
/// <summary>
|
||||
/// CMTrace timestamps from IME are emitted in UTC, but DateTimeKind may arrive as
|
||||
/// Unspecified depending on the parser path. Normalize via SpecifyKind=Utc rather than
|
||||
/// ToUniversalTime() to avoid double-conversion when Kind is already UTC.
|
||||
/// </summary>
|
||||
internal static DateTime NormalizeUtc(DateTime value) =>
|
||||
value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc);
|
||||
|
||||
// Background task
|
||||
private Task _pollingTask;
|
||||
private CancellationTokenSource _cts;
|
||||
|
||||
+16
-1
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
@@ -38,16 +39,30 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// </summary>
|
||||
public string Category { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Session 4910a5a5: names of the tracked apps most likely behind an Apps-subcategory
|
||||
/// failure (capped, never-started apps ranked first — those starve ESP's app gate
|
||||
/// without ever producing an IME install event). Snapshot from
|
||||
/// <c>ProvisioningStatusTracker.SnapshotTrackedAppsNotCompleted</c> at failure-arming
|
||||
/// time; empty for non-Apps failures and event-log-derived failures. Propagated onto
|
||||
/// the EspTerminalFailure signal payload so the DecisionEngine's
|
||||
/// <c>esp_failure_advisory</c> / <c>enrollment_failed</c> events can name the culprit
|
||||
/// directly.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> LikelyCulpritApps { get; }
|
||||
|
||||
public EspFailureDetectedEventArgs(
|
||||
string failureType,
|
||||
string errorCode = null,
|
||||
string failedSubcategory = null,
|
||||
string category = null)
|
||||
string category = null,
|
||||
IReadOnlyList<string> likelyCulpritApps = null)
|
||||
{
|
||||
FailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType;
|
||||
ErrorCode = string.IsNullOrEmpty(errorCode) ? null : errorCode;
|
||||
FailedSubcategory = string.IsNullOrEmpty(failedSubcategory) ? null : failedSubcategory;
|
||||
Category = string.IsNullOrEmpty(category) ? null : category;
|
||||
LikelyCulpritApps = likelyCulpritApps ?? Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+13
-7
@@ -35,13 +35,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// (registered during Autopilot-into-co-management, see the troubleshooting registry paths in
|
||||
/// https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment). Microsoft
|
||||
/// warns that ESP policy providers are not aware of each other and that pre-provisioning is
|
||||
/// unsupported with co-management. Field case that motivated this probe: a pre-provisioned
|
||||
/// device with the ConfigMgr client installed as a blocking app ended up with ONLY
|
||||
/// <c>ConfigMgr</c> registered under <c>Device\Setup\Apps\PolicyProviders</c> (no
|
||||
/// <c>Sidecar</c>) and <c>TrackingPoliciesCreated</c> never set — the user ESP hung at
|
||||
/// "Apps (Identifying)" for hours to days; manually renaming the key to <c>Sidecar</c>
|
||||
/// unblocked it immediately. <see cref="EspPolicyProviderStallDetector"/> turns this contract
|
||||
/// into the <c>esp_policy_provider_stalled</c> tripwire.
|
||||
/// unsupported with co-management. Field case that motivated this probe (issue #106): a
|
||||
/// pre-provisioned device with the ConfigMgr client installed as a blocking app ended up with
|
||||
/// ONLY <c>ConfigMgr</c> registered under <c>Device\Setup\Apps\PolicyProviders</c> (no
|
||||
/// <c>Sidecar</c>) — and although <c>TrackingPoliciesCreated=1</c> WAS set under it, the user
|
||||
/// ESP hung at "Apps (Identifying)" for hours to days; manually renaming the key to
|
||||
/// <c>Sidecar</c> unblocked it immediately. So the ESP's Apps wait is keyed to the provider
|
||||
/// the Intune service registered at enrollment (always <c>Sidecar</c>; ConfigMgr only joins
|
||||
/// via a co-management settings policy — see the WAP payload documented in
|
||||
/// https://oofhours.com/2023/08/25/run-an-sccm-task-sequence-during-autopilot/), and a
|
||||
/// foreign provider's completion values do not satisfy it. Detection therefore keys on the
|
||||
/// provider NAME topology, not only on completeness:
|
||||
/// <see cref="EspPolicyProviderStallDetector"/> turns this contract into the two-arm
|
||||
/// <c>esp_policy_provider_stalled</c> tripwire.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Fail-soft like <see cref="EspTrackingInfoProbe"/>: any failure is Debug-logged and yields
|
||||
|
||||
+116
-27
@@ -13,18 +13,30 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Turns the ESP policy-provider contract read by <see cref="EspPolicyProviderProbe"/> into
|
||||
/// the one-shot <c>esp_policy_provider_stalled</c> Warning: a registered provider that stays
|
||||
/// continuously incomplete for <see cref="DwellMinutes"/> minutes is reported once, with the
|
||||
/// full provider table in the payload. See the probe's header for the CSP contract, the
|
||||
/// official sources, and the co-management field case (ConfigMgr registered without Sidecar
|
||||
/// → user ESP parked at "Apps (Identifying)" with no OS timeout) this tripwire exists for.
|
||||
/// the one-shot <c>esp_policy_provider_stalled</c> Warning via two independent arms, each
|
||||
/// with the same <see cref="DwellMinutes"/>-minute continuous-dwell rule and the full
|
||||
/// provider table in the payload:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Arm 1 — <c>reason=provider_incomplete</c></b>: a registered provider stays
|
||||
/// continuously incomplete (e.g. Sidecar itself never sets TrackingPoliciesCreated).</item>
|
||||
/// <item><b>Arm 2 — <c>reason=sidecar_provider_missing</c></b>: at least one
|
||||
/// <c>Setup\Apps</c> provider is registered but none of them is <c>Sidecar</c> (the IME).
|
||||
/// This is the actual co-management field-case signature (issue #106): the foreign
|
||||
/// <c>ConfigMgr</c> provider had <c>TrackingPoliciesCreated=1</c> — i.e. was "complete"
|
||||
/// by the CSP value contract — yet the user ESP stayed parked at "Apps (Identifying)"
|
||||
/// until the key was renamed to <c>Sidecar</c>. The ESP's Apps wait is keyed to the
|
||||
/// Intune-registered provider by NAME, so provider completeness alone must not gate this
|
||||
/// detection. See the probe's header for sources.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Dwell semantics: the dwell clock per provider starts when it is first observed incomplete
|
||||
/// and RESETS when the provider completes or disappears — only <i>continuously</i> incomplete
|
||||
/// providers fire. A missing root key / empty provider list is normal early enrollment and
|
||||
/// clears all dwell state (probe D4). Purely observational: emission goes through
|
||||
/// <see cref="InformationalEventPost"/> (InformationalEvent pass-through, dispatch-guard
|
||||
/// exempt), never into decision state.
|
||||
/// Dwell semantics: arm 1's clock per provider starts when it is first observed incomplete
|
||||
/// and RESETS when the provider completes or disappears; arm 2's clock starts when the
|
||||
/// sidecar-missing condition is first observed and RESETS when Sidecar registers (covers the
|
||||
/// legitimate "ConfigMgr registered before Sidecar" startup ordering) — only
|
||||
/// <i>continuously</i> bad states fire. A missing root key / empty provider list is normal
|
||||
/// early enrollment and clears all dwell state (probe D4). Purely observational: emission
|
||||
/// goes through <see cref="InformationalEventPost"/> (InformationalEvent pass-through,
|
||||
/// dispatch-guard exempt), never into decision state.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Duplicate suppression: in-process one-shot per provider key, plus a
|
||||
@@ -44,6 +56,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
|
||||
internal const string SourceLabel = "EspPolicyProviderStallDetector";
|
||||
|
||||
/// <summary>Payload <c>reason</c>: a registered provider stayed continuously incomplete (arm 1).</summary>
|
||||
internal const string ReasonProviderIncomplete = "provider_incomplete";
|
||||
|
||||
/// <summary>Payload <c>reason</c>: Setup\Apps providers registered but Sidecar absent (arm 2 — dominant when both arms fire in one round).</summary>
|
||||
internal const string ReasonSidecarMissing = "sidecar_provider_missing";
|
||||
|
||||
/// <summary>
|
||||
/// Sentinel token representing the arm-2 condition in the cross-restart fingerprint.
|
||||
/// Cannot collide with provider keys — those always contain <c>|</c> separators.
|
||||
/// </summary>
|
||||
private const string SidecarMissingFingerprintToken = "sidecarMissing";
|
||||
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
@@ -57,6 +81,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
private readonly HashSet<string> _firedKeys =
|
||||
new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Arm-2 dwell state: first observation of "setupApps providers without Sidecar"; null = condition not active.</summary>
|
||||
private DateTime? _sidecarMissingSinceUtc;
|
||||
|
||||
/// <summary>Arm-2 in-process one-shot latch (mirror of <see cref="_firedKeys"/>).</summary>
|
||||
private bool _sidecarMissingFired;
|
||||
|
||||
public EspPolicyProviderStallDetector(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
@@ -79,13 +109,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
internal void Tick()
|
||||
{
|
||||
var snapshot = EspPolicyProviderProbe.Read(_logger);
|
||||
var eval = EvaluateStallTransition(snapshot, _firstSeenIncompleteUtc, _firedKeys, _clock.UtcNow, _dwell);
|
||||
var eval = EvaluateStallTransition(
|
||||
snapshot, _firstSeenIncompleteUtc, _firedKeys,
|
||||
_sidecarMissingSinceUtc, _sidecarMissingFired, _clock.UtcNow, _dwell);
|
||||
|
||||
_firstSeenIncompleteUtc.Clear();
|
||||
foreach (var kv in eval.UpdatedFirstSeenUtc)
|
||||
_firstSeenIncompleteUtc[kv.Key] = kv.Value;
|
||||
_sidecarMissingSinceUtc = eval.UpdatedSidecarMissingSinceUtc;
|
||||
|
||||
if (eval.NewlyStalled.Count > 0)
|
||||
if (eval.NewlyStalled.Count > 0 || eval.SidecarMissingStalledForMinutes != null)
|
||||
EmitStalled(snapshot, eval);
|
||||
}
|
||||
|
||||
@@ -94,24 +127,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// registry, timer, or ingress (pattern: <c>PerformanceCollector.EvaluateDiskLowTransition</c>).
|
||||
/// </summary>
|
||||
/// <param name="snapshot">Current provider snapshot.</param>
|
||||
/// <param name="firstSeenIncompleteUtc">Dwell state from the previous round (provider key → first-seen-incomplete).</param>
|
||||
/// <param name="firstSeenIncompleteUtc">Arm-1 dwell state from the previous round (provider key → first-seen-incomplete).</param>
|
||||
/// <param name="alreadyFired">Provider keys already reported this run (never re-report).</param>
|
||||
/// <param name="sidecarMissingSinceUtc">Arm-2 dwell state from the previous round; null = condition was not active.</param>
|
||||
/// <param name="sidecarMissingAlreadyFired">Arm-2 one-shot latch (never re-report).</param>
|
||||
/// <param name="nowUtc">Evaluation timestamp.</param>
|
||||
/// <param name="dwell">Continuous-incompleteness threshold.</param>
|
||||
/// <param name="dwell">Continuous-condition threshold (shared by both arms).</param>
|
||||
internal static StallEvaluation EvaluateStallTransition(
|
||||
EspPolicyProviderSnapshot snapshot,
|
||||
IReadOnlyDictionary<string, DateTime> firstSeenIncompleteUtc,
|
||||
ISet<string> alreadyFired,
|
||||
DateTime? sidecarMissingSinceUtc,
|
||||
bool sidecarMissingAlreadyFired,
|
||||
DateTime nowUtc,
|
||||
TimeSpan dwell)
|
||||
{
|
||||
var result = new StallEvaluation();
|
||||
|
||||
// Missing root key or no registered providers = normal early enrollment: clear all
|
||||
// dwell state (a provider must be continuously OBSERVED incomplete to accrue dwell).
|
||||
// dwell state (a condition must be continuously OBSERVED to accrue dwell).
|
||||
if (!snapshot.HasData || snapshot.Providers.Count == 0)
|
||||
return result;
|
||||
|
||||
// Arm 1 — a registered provider continuously incomplete.
|
||||
foreach (var provider in snapshot.Providers)
|
||||
{
|
||||
if (provider.IsComplete)
|
||||
@@ -127,22 +165,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
result.NewlyStalled.Add(new StalledProvider(provider, stalledFor.TotalMinutes));
|
||||
}
|
||||
|
||||
// Arm 2 — Setup\Apps providers registered, but none of them is Sidecar. Deliberately
|
||||
// ignores provider completeness: the field case (issue #106) had ConfigMgr with
|
||||
// TrackingPoliciesCreated=1 and the ESP still waited. Sidecar registering later
|
||||
// (legitimate startup ordering) clears the condition and resets the clock.
|
||||
if (HasSetupAppsProvider(snapshot) && !snapshot.SidecarRegistered)
|
||||
{
|
||||
var since = sidecarMissingSinceUtc ?? nowUtc;
|
||||
result.UpdatedSidecarMissingSinceUtc = since;
|
||||
|
||||
var missingFor = nowUtc - since;
|
||||
if (missingFor >= dwell && !sidecarMissingAlreadyFired)
|
||||
result.SidecarMissingStalledForMinutes = missingFor.TotalMinutes;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static bool HasSetupAppsProvider(EspPolicyProviderSnapshot snapshot)
|
||||
=> snapshot.Providers.Any(p => p.Kind == EspPolicyProviderProbe.KindSetupApps);
|
||||
|
||||
private void EmitStalled(EspPolicyProviderSnapshot snapshot, StallEvaluation eval)
|
||||
{
|
||||
foreach (var stalled in eval.NewlyStalled)
|
||||
_firedKeys.Add(stalled.Provider.Key);
|
||||
if (eval.SidecarMissingStalledForMinutes != null)
|
||||
_sidecarMissingFired = true;
|
||||
|
||||
// Cross-restart dedup: fingerprint = sorted set of all fired keys that are still
|
||||
// incomplete. A restart into the same stall re-accrues dwell and lands on the same
|
||||
// fingerprint → suppressed; a new provider joining the stall changes it → re-emitted.
|
||||
// Cross-restart dedup: fingerprint = sorted set of all fired conditions that are
|
||||
// still active (incomplete providers + the sidecar-missing sentinel). A restart into
|
||||
// the same stall re-accrues dwell and lands on the same fingerprint → suppressed; a
|
||||
// new condition joining the stall changes it → re-emitted.
|
||||
var currentlyStalledKeys = snapshot.Providers
|
||||
.Where(p => !p.IsComplete && _firedKeys.Contains(p.Key))
|
||||
.Select(p => p.Key)
|
||||
.OrderBy(k => k, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
if (_sidecarMissingFired && HasSetupAppsProvider(snapshot) && !snapshot.SidecarRegistered)
|
||||
currentlyStalledKeys.Add(SidecarMissingFingerprintToken);
|
||||
currentlyStalledKeys.Sort(StringComparer.OrdinalIgnoreCase);
|
||||
var fingerprint = StartupEventGate.HashString(string.Join(";", currentlyStalledKeys));
|
||||
if (_startupGate != null &&
|
||||
!_startupGate.ShouldEmit(Constants.EventTypes.EspPolicyProviderStalled, fingerprint))
|
||||
@@ -153,11 +213,30 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
return;
|
||||
}
|
||||
|
||||
// Sidecar-missing is the dominant reason when both arms fire in one round: it
|
||||
// explains the ESP hang regardless of the foreign providers' completeness.
|
||||
var reason = eval.SidecarMissingStalledForMinutes != null
|
||||
? ReasonSidecarMissing
|
||||
: ReasonProviderIncomplete;
|
||||
|
||||
var messageParts = new List<string>();
|
||||
if (eval.SidecarMissingStalledForMinutes != null)
|
||||
{
|
||||
var registeredNames = string.Join(", ", snapshot.Providers
|
||||
.Where(p => p.Kind == EspPolicyProviderProbe.KindSetupApps)
|
||||
.Select(p => $"{p.Name} ({p.Scope})"));
|
||||
messageParts.Add(
|
||||
$"expected IME provider '{EspPolicyProviderProbe.SidecarProviderName}' not registered for " +
|
||||
$">= {DwellMinutes} min while Setup/Apps providers exist: {registeredNames}");
|
||||
}
|
||||
var stalledNames = string.Join(", ",
|
||||
eval.NewlyStalled.Select(s => $"{s.Provider.Name} ({s.Provider.Scope}/{s.Provider.Kind})"));
|
||||
if (eval.NewlyStalled.Count > 0)
|
||||
messageParts.Add($"provider(s) incomplete for >= {DwellMinutes} min: {stalledNames}");
|
||||
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["reason"] = reason,
|
||||
["dwellMinutes"] = DwellMinutes,
|
||||
["sidecarRegistered"] = snapshot.SidecarRegistered,
|
||||
["providers"] = snapshot.Providers.Select(p => new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
@@ -177,6 +256,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
["stalledForMinutes"] = Math.Round(s.StalledForMinutes, 1),
|
||||
}).ToList(),
|
||||
};
|
||||
if (eval.SidecarMissingStalledForMinutes != null)
|
||||
data["sidecarMissingForMinutes"] = Math.Round(eval.SidecarMissingStalledForMinutes.Value, 1);
|
||||
|
||||
var message =
|
||||
"ESP Setup/Apps tracking stalled: " + string.Join("; ", messageParts) +
|
||||
" — the ESP waits on its app-tracking providers without any timeout";
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
@@ -188,27 +273,31 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
Source = SourceLabel,
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
ImmediateUpload = true,
|
||||
Message = $"ESP policy provider(s) incomplete for >= {DwellMinutes} min: {stalledNames} " +
|
||||
$"(sidecarRegistered={snapshot.SidecarRegistered}) — the ESP waits on " +
|
||||
"Setup/Apps providers without any timeout",
|
||||
Message = message,
|
||||
Data = data
|
||||
});
|
||||
_startupGate?.MarkEmitted(Constants.EventTypes.EspPolicyProviderStalled);
|
||||
|
||||
_logger?.Warning(
|
||||
$"EspPolicyProviderStallDetector: {stalledNames} incomplete for >= {DwellMinutes} min " +
|
||||
$"(sidecarRegistered={snapshot.SidecarRegistered}).");
|
||||
$"EspPolicyProviderStallDetector: {string.Join("; ", messageParts)} " +
|
||||
$"(reason={reason}, sidecarRegistered={snapshot.SidecarRegistered}).");
|
||||
}
|
||||
|
||||
/// <summary>Result of one <see cref="EvaluateStallTransition"/> round.</summary>
|
||||
internal sealed class StallEvaluation
|
||||
{
|
||||
/// <summary>Next round's dwell state — only currently-incomplete providers survive.</summary>
|
||||
/// <summary>Next round's arm-1 dwell state — only currently-incomplete providers survive.</summary>
|
||||
public Dictionary<string, DateTime> UpdatedFirstSeenUtc { get; } =
|
||||
new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Providers that crossed the dwell threshold this round and were not yet reported.</summary>
|
||||
/// <summary>Providers that crossed the dwell threshold this round and were not yet reported (arm 1).</summary>
|
||||
public List<StalledProvider> NewlyStalled { get; } = new List<StalledProvider>();
|
||||
|
||||
/// <summary>Next round's arm-2 dwell state; null = sidecar-missing condition not active (reset).</summary>
|
||||
public DateTime? UpdatedSidecarMissingSinceUtc { get; set; }
|
||||
|
||||
/// <summary>Non-null when the sidecar-missing condition crossed the dwell threshold this round and was not yet reported (arm 2).</summary>
|
||||
public double? SidecarMissingStalledForMinutes { get; set; }
|
||||
}
|
||||
|
||||
internal readonly struct StalledProvider
|
||||
|
||||
+189
-1
@@ -116,6 +116,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// failure details that fire after the settle window expires.
|
||||
private Dictionary<string, Timer> _provisioningFailureSettleTimers;
|
||||
private Dictionary<string, EspFailureDetectedEventArgs> _provisioningFailureSettleArgs;
|
||||
// Session 4910a5a5 — post-terminal recovery story. Populated when the settle window
|
||||
// expires and EspFailureDetected actually fires (terminal path); consumed by the
|
||||
// esp_failure_retry_detected / esp_failure_recovered emitters so a late "Try again"
|
||||
// recovery (23 min after the failure in the field session) is visible on the timeline
|
||||
// instead of hiding inside a generic subcategory_state_change. Keyed by category-name.
|
||||
private Dictionary<string, EspFailureDetectedEventArgs> _provisioningFailureTerminalArgs;
|
||||
private Dictionary<string, DateTime> _provisioningFailureTerminalUtc;
|
||||
private HashSet<string> _postFailureRetryEmitted;
|
||||
private HashSet<string> _postFailureRecoveredEmitted;
|
||||
// Session 2bc884b6: AppX enrichment scan seams + fire-once guard. The scanner and the
|
||||
// dispatcher are injectable so tests never touch the real event log or the threadpool.
|
||||
private readonly IAppxDeploymentFailureScanner _appxScanner;
|
||||
@@ -285,6 +294,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_lastSubcategoryStates = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureSettleTimers = new Dictionary<string, Timer>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureSettleArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureTerminalArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureTerminalUtc = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
_postFailureRetryEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
_postFailureRecoveredEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
_deviceSetupProvisioningCompleteFired = false;
|
||||
_deviceSetupFallbackTimer = null;
|
||||
_accountSetupProvisioningCompleteFired = false;
|
||||
@@ -665,6 +678,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// null-ing here is correctness-neutral on the start-condition side.
|
||||
ResetAccountSetupFallbackTimer();
|
||||
_accountSetupProvisioningCompleteFired = true;
|
||||
MaybeEmitPostFailureRecovered("AccountSetupCategory.Status", "AccountSetup");
|
||||
try { AccountSetupProvisioningComplete?.Invoke(this, EventArgs.Empty); }
|
||||
catch (Exception ex) { _logger.Error("AccountSetupProvisioningComplete handler failed (fallback path)", ex); }
|
||||
}
|
||||
@@ -833,6 +847,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// null-ing here is correctness-neutral on the start-condition side.
|
||||
ResetDeviceSetupFallbackTimer();
|
||||
_deviceSetupProvisioningCompleteFired = true;
|
||||
MaybeEmitPostFailureRecovered("DeviceSetupCategory.Status", "DeviceSetup");
|
||||
try { DeviceSetupProvisioningComplete?.Invoke(this, EventArgs.Empty); }
|
||||
catch (Exception ex) { _logger.Error("DeviceSetupProvisioningComplete handler failed (fallback path)", ex); }
|
||||
}
|
||||
@@ -946,12 +961,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
var severity = categorySucceeded == false ? EventSeverity.Warning : EventSeverity.Info;
|
||||
EmitProvisioningEvent(categoryLabel, categorySucceeded, statusText, subcategories, severity);
|
||||
EmitRawRegistryDump(categoryLabel, jsonValue, $"category_resolved_{(categorySucceeded == true ? "success" : "failed")}");
|
||||
|
||||
// Session 4910a5a5: category resolving to success after a terminally
|
||||
// fired failure = the failure recovered (fallback-path successes call
|
||||
// this from their own timer handlers).
|
||||
if (categorySucceeded == true)
|
||||
{
|
||||
MaybeEmitPostFailureRecovered(categoryName, categoryLabel);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var transitions = DetectSubcategoryTransitions(categoryName, subcategories);
|
||||
StoreSubcategoryStates(categoryName, subcategories);
|
||||
|
||||
// Session 4910a5a5: after a terminally fired failure, the failed
|
||||
// subcategory leaving "failed" is the ESP re-running the step —
|
||||
// consistent with the user pressing "Try again" on the failure page.
|
||||
MaybeEmitPostFailureRetry(categoryName, categoryLabel, transitions);
|
||||
|
||||
var failureTransitions = transitions.Where(t => t.IsFailure).ToList();
|
||||
if (transitions.Count > 0)
|
||||
{
|
||||
@@ -1025,6 +1053,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
$"escalating as {failureTypeName} (errorCode={errorCode ?? "n/a"})");
|
||||
_logger.Warning($"Provisioning failure detected: {failureTypeName}");
|
||||
|
||||
// Session 4910a5a5: name the likely culprit app(s) directly on the failure args so
|
||||
// the DecisionEngine's advisory/terminal events carry them — never-started apps
|
||||
// first (they starve ESP's app gate without ever producing an IME install event),
|
||||
// capped at 3 names.
|
||||
IReadOnlyList<string> likelyCulpritApps = null;
|
||||
if (string.Equals(failedSubcategoryName, "Apps", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
likelyCulpritApps = SnapshotTrackedAppsNotCompleted()
|
||||
.OrderBy(a => a["everStartedInstalling"] is bool started && started ? 1 : 0)
|
||||
.Select(a => a["appName"]?.ToString())
|
||||
.Where(n => !string.IsNullOrEmpty(n))
|
||||
.Take(3)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
// Caller already holds _stateLock.
|
||||
ArmFailureSettleTimer(
|
||||
categoryName: categoryName,
|
||||
@@ -1032,7 +1075,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
failureType: failureTypeName,
|
||||
errorCode: errorCode,
|
||||
failedSubcategory: failedSubcategoryName,
|
||||
category: categoryLabel));
|
||||
category: categoryLabel,
|
||||
likelyCulpritApps: likelyCulpritApps));
|
||||
|
||||
// Settle-window owns the actual EspFailureDetected fire. The legacy "fire-after-emit"
|
||||
// path in CheckProvisioningStatus is now a no-op for this code-path; it stays in place
|
||||
@@ -1292,6 +1336,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_provisioningCategoriesResolved.Remove(categoryName);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Session 4910a5a5: remember the terminally fired failure so a LATE
|
||||
// recovery (outside the settle window — e.g. user pressing "Try again"
|
||||
// minutes later) can be told as esp_failure_retry_detected /
|
||||
// esp_failure_recovered instead of hiding in a generic state-change.
|
||||
_provisioningFailureTerminalArgs[categoryName] = args;
|
||||
_provisioningFailureTerminalUtc[categoryName] = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
|
||||
if (recovered)
|
||||
@@ -1382,6 +1435,133 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Session 4910a5a5: after a terminally fired failure (settle window expired,
|
||||
/// EspFailureDetected raised), the failed subcategory leaving the <c>failed</c> state
|
||||
/// means the ESP re-ran the step — consistent with the user pressing "Try again" on
|
||||
/// the ESP failure page (the agent cannot observe the click itself). Emits
|
||||
/// <c>esp_failure_retry_detected</c> once per terminally fired category. Caller holds
|
||||
/// <see cref="_stateLock"/>.
|
||||
/// </summary>
|
||||
private void MaybeEmitPostFailureRetry(string categoryName, string categoryLabel, List<SubcategoryTransition> transitions)
|
||||
{
|
||||
if (transitions == null || transitions.Count == 0) return;
|
||||
if (_postFailureRetryEmitted.Contains(categoryName)) return;
|
||||
if (_provisioningFailureTerminalArgs == null
|
||||
|| !_provisioningFailureTerminalArgs.TryGetValue(categoryName, out var terminalArgs))
|
||||
{
|
||||
return; // No terminal fire on record — in-window recovery is settle_recovered's job.
|
||||
}
|
||||
|
||||
SubcategoryTransition? retryTransition = null;
|
||||
foreach (var t in transitions)
|
||||
{
|
||||
if (string.Equals(t.OldState, "failed", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(t.NewState, "failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
retryTransition = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (retryTransition == null) return;
|
||||
var retry = retryTransition.Value;
|
||||
|
||||
_postFailureRetryEmitted.Add(categoryName);
|
||||
|
||||
var minutesSinceFailure = MinutesSinceTerminalFailure(categoryName);
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "category", categoryLabel },
|
||||
{ "subcategory", retry.SubcategoryName },
|
||||
{ "previousState", retry.OldState },
|
||||
{ "newState", retry.NewState },
|
||||
{ "failureType", terminalArgs.FailureType },
|
||||
{ "minutesSinceFailure", minutesSinceFailure },
|
||||
};
|
||||
if (!string.IsNullOrEmpty(terminalArgs.ErrorCode))
|
||||
data["errorCode"] = terminalArgs.ErrorCode;
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.EspFailureRetryDetected,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "EspAndHelloTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"ESP retry detected — {categoryLabel}/{retry.SubcategoryName} left the failed state " +
|
||||
$"({retry.OldState} → {retry.NewState}) {minutesSinceFailure} min after the failure — " +
|
||||
"consistent with the user pressing 'Try again' on the ESP failure page.",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Session 4910a5a5: the category behind a terminally fired failure reached success
|
||||
/// (categorySucceeded=true or the all-subcategories-succeeded fallback) — the failure
|
||||
/// demonstrably recovered. Emits <c>esp_failure_recovered</c> once per terminally
|
||||
/// fired category so the timeline tells "failed → retried → recovered" instead of
|
||||
/// leaving the stale failure as the last word. Caller holds <see cref="_stateLock"/>.
|
||||
/// </summary>
|
||||
private void MaybeEmitPostFailureRecovered(string categoryName, string categoryLabel)
|
||||
{
|
||||
if (_postFailureRecoveredEmitted == null || _postFailureRecoveredEmitted.Contains(categoryName)) return;
|
||||
if (_provisioningFailureTerminalArgs == null
|
||||
|| !_provisioningFailureTerminalArgs.TryGetValue(categoryName, out var terminalArgs))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_postFailureRecoveredEmitted.Add(categoryName);
|
||||
|
||||
var minutesSinceFailure = MinutesSinceTerminalFailure(categoryName);
|
||||
string subcategoryStatus = null;
|
||||
if (terminalArgs.FailedSubcategory != null
|
||||
&& _lastSubcategoryStates != null
|
||||
&& _lastSubcategoryStates.TryGetValue(categoryName, out var subs)
|
||||
&& subs.TryGetValue(terminalArgs.FailedSubcategory, out var currentState))
|
||||
{
|
||||
subcategoryStatus = currentState;
|
||||
}
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "category", categoryLabel },
|
||||
{ "failedSubcategory", terminalArgs.FailedSubcategory ?? "category-level" },
|
||||
{ "failureType", terminalArgs.FailureType },
|
||||
{ "minutesSinceFailure", minutesSinceFailure },
|
||||
};
|
||||
if (!string.IsNullOrEmpty(terminalArgs.ErrorCode))
|
||||
data["errorCode"] = terminalArgs.ErrorCode;
|
||||
if (subcategoryStatus != null)
|
||||
data["subcategoryState"] = subcategoryStatus;
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.EspFailureRecovered,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "EspAndHelloTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"ESP failure recovered — {categoryLabel}/{terminalArgs.FailedSubcategory ?? "category-level"} " +
|
||||
$"completed successfully {minutesSinceFailure} min after the failure; the earlier ESP failure no longer applies.",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
});
|
||||
}
|
||||
|
||||
private int MinutesSinceTerminalFailure(string categoryName)
|
||||
{
|
||||
if (_provisioningFailureTerminalUtc != null
|
||||
&& _provisioningFailureTerminalUtc.TryGetValue(categoryName, out var firedAt))
|
||||
{
|
||||
return Math.Max(0, (int)Math.Round((DateTime.UtcNow - firedAt).TotalMinutes));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private void EmitFailureSettleStarted(EspFailureDetectedEventArgs args)
|
||||
{
|
||||
var data = new Dictionary<string, object>
|
||||
@@ -1863,6 +2043,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_provisioningFailureSettleTimers = new Dictionary<string, Timer>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_provisioningFailureSettleArgs == null)
|
||||
_provisioningFailureSettleArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_provisioningFailureTerminalArgs == null)
|
||||
_provisioningFailureTerminalArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_provisioningFailureTerminalUtc == null)
|
||||
_provisioningFailureTerminalUtc = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_postFailureRetryEmitted == null)
|
||||
_postFailureRetryEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_postFailureRecoveredEmitted == null)
|
||||
_postFailureRecoveredEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+87
-28
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using Microsoft.Win32.SafeHandles;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
{
|
||||
@@ -56,6 +57,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
private const int InitialBufferSize = 64 * 1024;
|
||||
private const int RetryBufferSize = 256 * 1024;
|
||||
|
||||
// SeSystemEnvironmentPrivilege is adjusted on the PROCESS token, so enable → read →
|
||||
// restore is a process-wide side effect and must run as one atomic sequence. Without
|
||||
// this gate a second reader would observe "already enabled", correctly skip its own
|
||||
// restore, and then have the privilege disabled underneath it when the first reader
|
||||
// finishes — a spurious privilege_denied on a machine that actually holds it.
|
||||
private static readonly object FirmwareReadGate = new object();
|
||||
|
||||
/// <summary>
|
||||
/// Reads db and KEK and evaluates the certificate presence flags. Never throws;
|
||||
/// per-variable failures surface as <see cref="UefiSecureBootCertSnapshot.DbStatus"/> /
|
||||
@@ -67,6 +75,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
var snapshot = new UefiSecureBootCertSnapshot();
|
||||
try
|
||||
{
|
||||
// The critical section covers the whole privileged window, not just the
|
||||
// adjustment — see FirmwareReadGate. Only synchronous kernel calls and byte
|
||||
// scans run inside it, so it cannot block on anything else the agent does.
|
||||
lock (FirmwareReadGate)
|
||||
using (var privilege = SystemEnvironmentPrivilege.TryEnable())
|
||||
{
|
||||
if (!privilege.Enabled)
|
||||
@@ -168,23 +180,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
/// of the instance and restores the previous privilege state on dispose. SYSTEM holds
|
||||
/// the privilege (default-disabled); without elevation <see cref="Enabled"/> stays
|
||||
/// false and the caller reports <see cref="StatusPrivilegeDenied"/>.
|
||||
/// <para>
|
||||
/// <b>Crash-safety.</b> This is the repo's only privilege-adjustment code, so the
|
||||
/// ownership rules are spelled out: the token is held in a <see cref="SafeHandle"/>
|
||||
/// assigned by the P/Invoke marshaler, so the CLR owns it from the moment
|
||||
/// <c>OpenProcessToken</c> returns — it is released by the critical finalizer even if
|
||||
/// this instance never reaches the caller's <c>using</c> (async exception between the
|
||||
/// call and the assignment). <see cref="Dispose"/> is idempotent, never throws, and
|
||||
/// releases the handle in a <c>finally</c> so a failed restore cannot leak it. The
|
||||
/// restore obligation is recorded immediately after the successful adjustment, so no
|
||||
/// throw in between can strand the process with the privilege left enabled.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private sealed class SystemEnvironmentPrivilege : IDisposable
|
||||
{
|
||||
public bool Enabled { get; private set; }
|
||||
|
||||
private IntPtr _token = IntPtr.Zero;
|
||||
private SafeTokenHandle _token;
|
||||
private NativeMethods.TOKEN_PRIVILEGES _previousState;
|
||||
private bool _restoreNeeded;
|
||||
private bool _disposed;
|
||||
|
||||
public static SystemEnvironmentPrivilege TryEnable()
|
||||
{
|
||||
var instance = new SystemEnvironmentPrivilege();
|
||||
try
|
||||
{
|
||||
SafeTokenHandle token;
|
||||
if (!NativeMethods.OpenProcessToken(NativeMethods.GetCurrentProcess(),
|
||||
NativeMethods.TOKEN_ADJUST_PRIVILEGES | NativeMethods.TOKEN_QUERY, out instance._token))
|
||||
NativeMethods.TOKEN_ADJUST_PRIVILEGES | NativeMethods.TOKEN_QUERY, out token))
|
||||
{
|
||||
// Failed open still yields a (closed/invalid) SafeHandle instance.
|
||||
if (token != null) token.Dispose();
|
||||
return instance;
|
||||
}
|
||||
|
||||
instance._token = token;
|
||||
|
||||
NativeMethods.LUID luid;
|
||||
if (!NativeMethods.LookupPrivilegeValueW(null, "SeSystemEnvironmentPrivilege", out luid))
|
||||
@@ -197,52 +228,80 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
Attributes = NativeMethods.SE_PRIVILEGE_ENABLED
|
||||
};
|
||||
|
||||
NativeMethods.TOKEN_PRIVILEGES previousState;
|
||||
uint returnLength;
|
||||
if (!NativeMethods.AdjustTokenPrivileges(instance._token, false, ref newState,
|
||||
if (!NativeMethods.AdjustTokenPrivileges(token, false, ref newState,
|
||||
(uint)Marshal.SizeOf(typeof(NativeMethods.TOKEN_PRIVILEGES)),
|
||||
out instance._previousState, out returnLength))
|
||||
out previousState, out returnLength))
|
||||
return instance;
|
||||
|
||||
// AdjustTokenPrivileges returns TRUE even when the privilege is not held —
|
||||
// the real verdict is ERROR_NOT_ALL_ASSIGNED in the last error.
|
||||
// the real verdict is ERROR_NOT_ALL_ASSIGNED in the last error. Nothing was
|
||||
// adjusted in that case, so there is nothing to restore.
|
||||
if (Marshal.GetLastWin32Error() == ErrorNotAllAssigned)
|
||||
return instance;
|
||||
|
||||
// Restore only when the privilege was not already enabled before.
|
||||
instance._restoreNeeded = instance._previousState.PrivilegeCount > 0
|
||||
&& (instance._previousState.Attributes & NativeMethods.SE_PRIVILEGE_ENABLED) == 0;
|
||||
// The token is now modified: record the restore obligation before anything
|
||||
// else can throw. Restore only when the privilege was not already enabled.
|
||||
instance._previousState = previousState;
|
||||
instance._restoreNeeded = previousState.PrivilegeCount > 0
|
||||
&& (previousState.Attributes & NativeMethods.SE_PRIVILEGE_ENABLED) == 0;
|
||||
instance.Enabled = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Fail-soft: Enabled stays false.
|
||||
// Fail-soft: Enabled stays false. Whatever was acquired before the failure
|
||||
// is already tracked on the instance and released by Dispose.
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_token != IntPtr.Zero)
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
Enabled = false;
|
||||
|
||||
var token = _token;
|
||||
_token = null;
|
||||
if (token == null) return;
|
||||
|
||||
try
|
||||
{
|
||||
try
|
||||
if (_restoreNeeded)
|
||||
{
|
||||
if (_restoreNeeded)
|
||||
{
|
||||
NativeMethods.TOKEN_PRIVILEGES ignored;
|
||||
uint returnLength;
|
||||
NativeMethods.AdjustTokenPrivileges(_token, false, ref _previousState,
|
||||
(uint)Marshal.SizeOf(typeof(NativeMethods.TOKEN_PRIVILEGES)),
|
||||
out ignored, out returnLength);
|
||||
}
|
||||
_restoreNeeded = false;
|
||||
NativeMethods.TOKEN_PRIVILEGES ignored;
|
||||
uint returnLength;
|
||||
NativeMethods.AdjustTokenPrivileges(token, false, ref _previousState,
|
||||
(uint)Marshal.SizeOf(typeof(NativeMethods.TOKEN_PRIVILEGES)),
|
||||
out ignored, out returnLength);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best effort — the process token privilege state is not load-bearing
|
||||
// for anything else the agent does.
|
||||
}
|
||||
NativeMethods.CloseHandle(_token);
|
||||
_token = IntPtr.Zero;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best effort — the process token privilege state is not load-bearing for
|
||||
// anything else the agent does, and Dispose must never throw.
|
||||
}
|
||||
finally
|
||||
{
|
||||
token.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Owns a process-token handle. <c>ReleaseHandle</c> runs on the finalizer thread when
|
||||
/// <see cref="SystemEnvironmentPrivilege.Dispose"/> is skipped, so it must not throw —
|
||||
/// a single <c>CloseHandle</c> P/Invoke satisfies that.
|
||||
/// </summary>
|
||||
private sealed class SafeTokenHandle : SafeHandleZeroOrMinusOneIsInvalid
|
||||
{
|
||||
public SafeTokenHandle() : base(true) { }
|
||||
|
||||
protected override bool ReleaseHandle()
|
||||
{
|
||||
return NativeMethods.CloseHandle(handle);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -278,7 +337,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
|
||||
[DllImport("advapi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool OpenProcessToken(IntPtr processHandle, uint desiredAccess, out IntPtr tokenHandle);
|
||||
public static extern bool OpenProcessToken(IntPtr processHandle, uint desiredAccess, out SafeTokenHandle tokenHandle);
|
||||
|
||||
[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
@@ -286,7 +345,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
|
||||
[DllImport("advapi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool AdjustTokenPrivileges(IntPtr tokenHandle, bool disableAllPrivileges,
|
||||
public static extern bool AdjustTokenPrivileges(SafeTokenHandle tokenHandle, bool disableAllPrivileges,
|
||||
ref TOKEN_PRIVILEGES newState, uint bufferLength, out TOKEN_PRIVILEGES previousState, out uint returnLength);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
|
||||
+17
@@ -706,10 +706,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
/// the desktop via "Continue anyway".
|
||||
/// </para>
|
||||
/// </summary>
|
||||
// Serializes the ESP-config read+emit across ALL callers (CollectAll on the startup
|
||||
// ThreadPool task, CollectAtEnrollmentStart and RefreshEspConfiguration on their own
|
||||
// Task.Run) — without it, two concurrent collects can interleave read-before-emit and
|
||||
// the LATER emission can carry the OLDER (smaller) registry snapshot. The backend keeps
|
||||
// the latest list-bearing emission as positive blocking evidence, so an out-of-order
|
||||
// emission would silently shrink it (Codex review). Inside the lock each collect is
|
||||
// read→emit atomic: a later emission always carries an equal-or-newer snapshot.
|
||||
private readonly object _espConfigCollectLock = new object();
|
||||
|
||||
// Internal (not private) as a test seam: unlike CollectAll/CollectAtEnrollmentStart this
|
||||
// method touches no live system state once both probes are overridden, so tests can
|
||||
// drive the esp_config_detected event surface in isolation (InternalsVisibleTo).
|
||||
internal (bool? skipUserStatusPage, bool? skipDeviceStatusPage) CollectEspConfiguration()
|
||||
{
|
||||
lock (_espConfigCollectLock)
|
||||
{
|
||||
return CollectEspConfigurationLocked();
|
||||
}
|
||||
}
|
||||
|
||||
private (bool? skipUserStatusPage, bool? skipDeviceStatusPage) CollectEspConfigurationLocked()
|
||||
{
|
||||
bool? skipUser = null;
|
||||
bool? skipDevice = null;
|
||||
|
||||
+17
@@ -114,6 +114,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
CollectTpmStatus();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-reads the ESP configuration (skip flags + ESPTrackingInfo blocking lists) and
|
||||
/// re-emits <c>esp_config_detected</c> when the payload changed. Audit Q2: the earlier
|
||||
/// emissions (agent start, DeviceSetup detection) fire before the registry lists are
|
||||
/// complete — the ESP writes one timestamped subkey per CSP status write, so the
|
||||
/// device-scope list grows progressively and the user-scope <c>S-<SID></c> lists
|
||||
/// appear only after sign-in. Called by <see cref="Orchestration.DeviceInfoHost"/> as
|
||||
/// phase-driven one-shots (apps sub-phase opened, AccountSetup detected) — never on a
|
||||
/// timer. The StartupEventGate keeps an unchanged payload silent, so a re-collect that
|
||||
/// found nothing new emits nothing.
|
||||
/// </summary>
|
||||
public void RefreshEspConfiguration(string trigger)
|
||||
{
|
||||
_logger.Info($"EnrollmentTracker: re-collecting ESP configuration (trigger={trigger})");
|
||||
CollectEspConfiguration();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-collects device info that may change during enrollment (e.g. BitLocker enabled via policy).
|
||||
/// Called at enrollment complete / FinalizingSetup transition to capture final state.
|
||||
|
||||
+9
-5
@@ -77,11 +77,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
|
||||
protected override void Collect()
|
||||
{
|
||||
// Full path writes 18 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (4) + network delta (8: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests). cap=18 → HashHelpers.GetPrime(18)=23
|
||||
// buckets → no resize on the 18th key (cap=16 would land at 17 buckets and resize).
|
||||
var data = new Dictionary<string, object>(capacity: 18, StringComparer.Ordinal)
|
||||
// Full path writes 19 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (4) + network delta (9: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency). cap=19 →
|
||||
// HashHelpers.GetPrime(19)=23 buckets → no resize on the 19th key.
|
||||
var data = new Dictionary<string, object>(capacity: 19, StringComparer.Ordinal)
|
||||
{
|
||||
{ "agent_version", _agentVersion }
|
||||
};
|
||||
@@ -164,6 +164,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
data["net_total_bytes_up"] = currentNet.TotalBytesUp;
|
||||
data["net_total_bytes_down"] = currentNet.TotalBytesDown;
|
||||
data["net_total_requests"] = currentNet.RequestCount;
|
||||
// Cumulative latency sum: total/requests = session-wide average HTTP
|
||||
// round-trip. The backend projects that average onto the Session row
|
||||
// (AvgApiLatencyMs) — cumulative counters make the write idempotent.
|
||||
data["net_total_latency_ms"] = currentNet.TotalLatencyMs;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -37,8 +37,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// <see cref="StallProbeCollectorAdapter"/>. Owns its 60-s idle-check timer (the
|
||||
/// collector itself has no timer — it's a pure probe invoked from outside).</item>
|
||||
/// <item><b>EspPolicyProviderStallHost</b> — always-on 60-s wall-clock tick for the
|
||||
/// <c>esp_policy_provider_stalled</c> tripwire (registered EnrollmentStatusTracking CSP
|
||||
/// provider continuously incomplete ≥ 15 min — the no-timeout ESP wait).</item>
|
||||
/// two-arm <c>esp_policy_provider_stalled</c> tripwire (EnrollmentStatusTracking CSP
|
||||
/// provider continuously incomplete ≥ 15 min, OR Setup\Apps providers registered
|
||||
/// without the expected <c>Sidecar</c>/IME provider — the no-timeout ESP wait).</item>
|
||||
/// </list>
|
||||
/// Optional peripheral hosts (driven by <see cref="CollectorConfiguration"/> toggles):
|
||||
/// <list type="bullet">
|
||||
@@ -307,11 +308,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
}
|
||||
|
||||
// ESP policy-provider stall tripwire — always-on kernel host (no config gate, like
|
||||
// disk_space_low): a registered EnrollmentStatusTracking CSP provider that never
|
||||
// completes (e.g. co-management "ConfigMgr" without "Sidecar" on a pre-provisioned
|
||||
// device) parks the user ESP at "Apps (Identifying)" WITHOUT any OS timeout. Wall-clock
|
||||
// dwell, deliberately outside the StallProbe gate: the idle clock there resets on any
|
||||
// session activity, and the stall must accrue regardless of it.
|
||||
// disk_space_low): a provider that never completes, or co-management "ConfigMgr"
|
||||
// registered without the expected "Sidecar"/IME provider (issue #106 — even with
|
||||
// TrackingPoliciesCreated=1 the ESP ignores foreign provider names), parks the user
|
||||
// ESP at "Apps (Identifying)" WITHOUT any OS timeout. Wall-clock dwell, deliberately
|
||||
// outside the StallProbe gate: the idle clock there resets on any session activity,
|
||||
// and the stall must accrue regardless of it.
|
||||
hosts.Add(new EspPolicyProviderStallHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
|
||||
@@ -36,6 +36,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// <item><c>CollectAtEnd</c> — once, on <c>FinalizingSetup</c> or desktop arrival
|
||||
/// (whichever comes first; desktop arrival also covers no-ESP / WDP v2). Re-fetches
|
||||
/// BitLocker (commonly enabled via policy DURING enrollment) + the active NIC.</item>
|
||||
/// <item><c>RefreshEspConfiguration</c> — audit Q2, one-shot per trigger: the ESP
|
||||
/// blocking lists in the registry grow progressively (device scope) and the user-scope
|
||||
/// lists appear only after sign-in, so the early emissions are structurally partial.
|
||||
/// Re-collected when the apps sub-phase opens (first app activity in DeviceSetup /
|
||||
/// AccountSetup — the adapter's <c>phase_transition</c> declaration) and when the
|
||||
/// AccountSetup ESP phase itself is detected. Phase-driven one-shots, never a timer
|
||||
/// (no-heartbeat policy).</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Duplicate-event cost is zero: every re-collected event runs through the collector's
|
||||
@@ -46,6 +53,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
public string Name => "DeviceInfoCollector";
|
||||
|
||||
// ESP-config refresh trigger labels (audit Q2) — also the values passed to
|
||||
// DeviceInfoCollector.RefreshEspConfiguration for the agent-log line.
|
||||
internal const string EspConfigTriggerAppsDevice = "apps_phase_device";
|
||||
internal const string EspConfigTriggerAppsUser = "apps_phase_user";
|
||||
internal const string EspConfigTriggerAccountSetup = "account_setup_detected";
|
||||
|
||||
private readonly Monitoring.Telemetry.DeviceInfo.DeviceInfoCollector _collector;
|
||||
private readonly AgentLogger _logger;
|
||||
|
||||
@@ -58,6 +71,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
private int _endCollected;
|
||||
private int _disposed;
|
||||
|
||||
// Audit Q2 — fire-once latches per ESP-config refresh trigger. Guarded by _espConfigLock
|
||||
// instead of three Interlocked ints so the membership test + add stays one atomic step.
|
||||
private readonly object _espConfigLock = new object();
|
||||
private readonly HashSet<string> _espConfigRefreshesFired = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
public DeviceInfoHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
@@ -100,6 +118,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private void OnSignalPosted(DecisionSignalKind kind, IReadOnlyDictionary<string, string>? payload)
|
||||
{
|
||||
HandleEspConfigRefreshTrigger(kind, payload);
|
||||
|
||||
var startTrigger = IsEnrollmentStartTrigger(kind, payload);
|
||||
var endTrigger = IsEndTrigger(kind, payload);
|
||||
if (!startTrigger && !endTrigger) return;
|
||||
@@ -111,10 +131,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
var runEndCollect = endTrigger && Interlocked.Exchange(ref _endCollected, 1) == 0;
|
||||
if (!runStartRefresh && !runEndCollect) return;
|
||||
|
||||
// Both one-shots done → nothing left to observe.
|
||||
if (Volatile.Read(ref _enrollmentStartCollected) == 1 && Volatile.Read(ref _endCollected) == 1)
|
||||
Unsubscribe();
|
||||
|
||||
var trigger = kind == DecisionSignalKind.DesktopArrived ? "desktop_arrived" : "esp_phase_changed";
|
||||
_logger.Info($"DeviceInfoHost: trigger '{trigger}' — scheduling re-collect (enrollmentStart={runStartRefresh}, end={runEndCollect}).");
|
||||
|
||||
@@ -134,6 +150,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Audit Q2 — ESP-config refresh one-shots. Each trigger label fires at most once per
|
||||
/// agent run; the collector's StartupEventGate keeps an unchanged payload silent, so
|
||||
/// the worst case is one silent registry re-read per trigger. The subscription stays
|
||||
/// armed for the host's lifetime (unlike the start/end pair there is no "all triggers
|
||||
/// seen" point that is guaranteed to arrive — SkipUser enrollments never enter
|
||||
/// AccountSetup); the per-signal cost is a few dictionary lookups.
|
||||
/// </summary>
|
||||
private void HandleEspConfigRefreshTrigger(DecisionSignalKind kind, IReadOnlyDictionary<string, string>? payload)
|
||||
{
|
||||
var trigger = ClassifyEspConfigRefreshTrigger(kind, payload);
|
||||
if (trigger == null) return;
|
||||
|
||||
lock (_espConfigLock)
|
||||
{
|
||||
if (!_espConfigRefreshesFired.Add(trigger)) return;
|
||||
}
|
||||
|
||||
_logger.Info($"DeviceInfoHost: ESP-config refresh trigger '{trigger}' — scheduling re-collect.");
|
||||
Task.Run(() =>
|
||||
{
|
||||
try { _collector.RefreshEspConfiguration(trigger); }
|
||||
catch (Exception ex) { _logger.Warning($"DeviceInfoHost: ESP-config refresh threw: {ex.Message}"); }
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>First DeviceSetup phase signal — MDM enrollment has populated the registry surface.</summary>
|
||||
internal static bool IsEnrollmentStartTrigger(DecisionSignalKind kind, IReadOnlyDictionary<string, string>? payload)
|
||||
{
|
||||
@@ -156,6 +198,45 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
&& string.Equals(phase, nameof(EnrollmentPhase.FinalizingSetup), StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Audit Q2 — maps a posted signal to an ESP-config refresh trigger label, or
|
||||
/// <c>null</c> when the signal is not one. Two trigger families:
|
||||
/// <list type="bullet">
|
||||
/// <item>The adapter's apps sub-phase declaration (<c>phase_transition</c>
|
||||
/// InformationalEvent with phase AppsDevice/AppsUser — fired on the first app
|
||||
/// activity in an ESP phase): by then IME is actively processing the blocking set,
|
||||
/// so the device-scope ESPTrackingInfo lists are as complete as they get.</item>
|
||||
/// <item>The AccountSetup ESP phase (UserSetup detection): sign-in has happened, so
|
||||
/// the user-scope <c>S-<SID></c> lists exist now — production showed 88 of 89
|
||||
/// sessions with an EMPTY user list at the last pre-fix emission.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
internal static string? ClassifyEspConfigRefreshTrigger(DecisionSignalKind kind, IReadOnlyDictionary<string, string>? payload)
|
||||
{
|
||||
if (payload == null) return null;
|
||||
|
||||
if (kind == DecisionSignalKind.InformationalEvent
|
||||
&& payload.TryGetValue(SignalPayloadKeys.EventType, out var eventType)
|
||||
&& string.Equals(eventType, AutopilotMonitor.Shared.Constants.EventTypes.PhaseTransition, StringComparison.Ordinal)
|
||||
&& payload.TryGetValue(Telemetry.Events.EventTimelineEmitter.PhaseParamKey, out var declaredPhase))
|
||||
{
|
||||
if (string.Equals(declaredPhase, nameof(EnrollmentPhase.AppsDevice), StringComparison.Ordinal))
|
||||
return EspConfigTriggerAppsDevice;
|
||||
if (string.Equals(declaredPhase, nameof(EnrollmentPhase.AppsUser), StringComparison.Ordinal))
|
||||
return EspConfigTriggerAppsUser;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (kind == DecisionSignalKind.EspPhaseChanged
|
||||
&& payload.TryGetValue(SignalPayloadKeys.EspPhase, out var espPhase)
|
||||
&& string.Equals(espPhase, nameof(EnrollmentPhase.AccountSetup), StringComparison.Ordinal))
|
||||
{
|
||||
return EspConfigTriggerAccountSetup;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public void Stop() => Unsubscribe();
|
||||
|
||||
public void Dispose()
|
||||
|
||||
@@ -111,6 +111,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
logger: logger,
|
||||
matchLogPath: expandedMatchLogPath,
|
||||
stateDirectory: stateDirectory);
|
||||
// Historic-replay guard runs against the agent clock, not the raw system clock —
|
||||
// keeps the tracker's staleness verdicts consistent with the adapter's clamp.
|
||||
_tracker.UtcNowProvider = () => clock.UtcNow;
|
||||
|
||||
if (simulationMode)
|
||||
{
|
||||
|
||||
@@ -93,7 +93,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
using (obj)
|
||||
{
|
||||
manufacturer = obj["Manufacturer"]?.ToString() ?? Unknown;
|
||||
manufacturer = NormalizeWmiValue(obj["Manufacturer"]?.ToString());
|
||||
if (manufacturer.IndexOf("lenovo", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
using (var lenovoSearcher = new ManagementObjectSearcher("SELECT Version FROM Win32_ComputerSystemProduct"))
|
||||
@@ -103,7 +103,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
using (lenovoObj)
|
||||
{
|
||||
model = lenovoObj["Version"]?.ToString() ?? Unknown;
|
||||
model = NormalizeWmiValue(lenovoObj["Version"]?.ToString());
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -111,7 +111,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
else
|
||||
{
|
||||
model = obj["Model"]?.ToString() ?? Unknown;
|
||||
model = NormalizeWmiValue(obj["Model"]?.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,7 +132,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
using (obj)
|
||||
{
|
||||
serialNumber = obj["SerialNumber"]?.ToString() ?? Unknown;
|
||||
serialNumber = NormalizeWmiValue(obj["SerialNumber"]?.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -154,5 +154,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private static bool IsResolved(string value)
|
||||
=> !string.IsNullOrWhiteSpace(value)
|
||||
&& !string.Equals(value, Unknown, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Trims a raw WMI string value at the source (audit Q5: BIOS serials are used as device
|
||||
/// identity keys downstream — F2 device history joins on them — so padding must never
|
||||
/// leave the agent). Whitespace-only and empty values collapse to the "Unknown" sentinel,
|
||||
/// matching the null fallback so the retry logic treats them as unresolved.
|
||||
/// </summary>
|
||||
internal static string NormalizeWmiValue(string raw)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(raw)) return Unknown;
|
||||
return raw.Trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
@@ -267,6 +268,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
payload["failedSubcategory"] = failedSubcategory!;
|
||||
if (!string.IsNullOrEmpty(category))
|
||||
payload["category"] = category!;
|
||||
// Session 4910a5a5: culprit-app names ride the signal so the DecisionEngine's
|
||||
// esp_failure_advisory / enrollment_failed events can name them directly.
|
||||
var culprits = args?.LikelyCulpritApps;
|
||||
if (culprits != null && culprits.Count > 0)
|
||||
{
|
||||
payload["likelyCulpritApps"] = string.Join(", ", culprits);
|
||||
payload["likelyCulpritAppCount"] = culprits.Count.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
_ingress.Post(
|
||||
kind: DecisionSignalKind.EspTerminalFailure,
|
||||
|
||||
@@ -97,12 +97,24 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
// bootstrap marker line. Lets MCP report "which device runs which bootstrap version".
|
||||
private bool _bootstrapDetectedPosted;
|
||||
|
||||
// One-shot: historic script replay detected (a previous enrollment's IME log content
|
||||
// surviving on disk) — its script events are suppressed, and a single
|
||||
// historic_ime_replay_detected Info event explains the gap in the timeline.
|
||||
private bool _historicReplayNoted;
|
||||
private int _suppressedHistoricReplayEvents; // forensic count for the log line
|
||||
|
||||
// A platform script whose observed run duration reaches this is treated as having hit the
|
||||
// IME script-execution timeout (default ~30 min). 25 min leaves headroom below 30 so clock
|
||||
// jitter / the agent's late replay window can't miss it, while staying well above any
|
||||
// legitimately long install wrapper that completes on its own.
|
||||
private static readonly TimeSpan ScriptTimeoutSuspectedThreshold = TimeSpan.FromMinutes(25);
|
||||
|
||||
// Source log lines older than this relative to the clock are pathological — shared by
|
||||
// ResolveOccurredAt's staleness clamp, the historic-replay suppression, the duration
|
||||
// plausibility backstop AND the tracker's app-mutation guard (single source of truth
|
||||
// in ImeLogTracker so the layers can never drift apart).
|
||||
private static readonly TimeSpan StaleSourceThreshold = ImeLogTracker.HistoricReplayThreshold;
|
||||
|
||||
// Matches the deterministic marker line the bootstrap script writes via Write-Log,
|
||||
// e.g. "Bootstrap script version: v2.0" — see scripts/Bootstrap/Install-AutopilotMonitor.ps1.
|
||||
// Same shape as the web-side regex in utils/bootstrapVersion.ts; kept in sync intentionally.
|
||||
@@ -120,7 +132,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
|
||||
// Plan §5 Fix 4c — per-app install-lifecycle timing captured from state transitions.
|
||||
// StartedAtUtc on first Downloading/Installing/InProgress transition (set-once);
|
||||
// CompletedAtUtc on first terminal transition (Installed/Skipped/Postponed/Error).
|
||||
// CompletedAtUtc on terminal transition (Installed/Skipped/Postponed/Error), re-armed
|
||||
// to null on a terminal→active transition (IME retry, audit Q1) so the final terminal
|
||||
// stamp reflects the last outcome instead of freezing at the first one.
|
||||
// Surfaced via <see cref="AppTimings"/> to peripheral consumers (FinalStatusBuilder,
|
||||
// app_tracking_summary emission in EnrollmentTerminationHandler).
|
||||
private readonly Dictionary<string, (DateTime? Started, DateTime? Completed)> _appTimings =
|
||||
@@ -289,17 +303,11 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
rawSourceTimestamp = sourceTsOverride ?? _tracker.LastMatchedLogTimestamp;
|
||||
if (rawSourceTimestamp.HasValue)
|
||||
{
|
||||
// CMTrace log timestamps from IME are emitted in UTC, but DateTimeKind may
|
||||
// arrive as Unspecified depending on the parser path. Normalize via
|
||||
// SpecifyKind=Utc rather than ToUniversalTime() to avoid double-conversion
|
||||
// when Kind is already UTC.
|
||||
var asUtc = rawSourceTimestamp.Value.Kind == DateTimeKind.Utc
|
||||
? rawSourceTimestamp.Value
|
||||
: DateTime.SpecifyKind(rawSourceTimestamp.Value, DateTimeKind.Utc);
|
||||
var asUtc = NormalizeUtc(rawSourceTimestamp.Value);
|
||||
|
||||
var clockNow = _clock.UtcNow;
|
||||
var ageHours = (clockNow - asUtc).TotalHours;
|
||||
if (ageHours > 24.0 || ageHours < -1.0)
|
||||
if (ageHours > StaleSourceThreshold.TotalHours || ageHours < -1.0)
|
||||
{
|
||||
// Pathologically stale (>24h old) or in the future (>1h skew) — log,
|
||||
// fall back, and let the caller flag the event.
|
||||
@@ -325,6 +333,67 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
data["rejectedSourceTimestamp"] = rawSourceTs.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
// CMTrace Kind normalization — shared implementation lives on the tracker (used by its
|
||||
// historic-replay guard too).
|
||||
private static DateTime NormalizeUtc(DateTime value) => ImeLogTracker.NormalizeUtc(value);
|
||||
|
||||
/// <summary>
|
||||
/// True when an event's source log line is a historic replay: the source timestamp
|
||||
/// was rejected as stale (> <see cref="StaleSourceThreshold"/> in the PAST). Those lines
|
||||
/// are script runs and app installs from a previous enrollment whose IME log survived on
|
||||
/// disk (session eaf3d8c4: 156 replayed script completions + 147 replayed app events
|
||||
/// shown as current activity, immediate-upload flood → ingress_backpressure). Future-skew
|
||||
/// rejections (negative age — mid-enrollment clock jump) and clock-only fallbacks (no
|
||||
/// source timestamp at all) are NOT replay and must still emit. Emits the one-shot
|
||||
/// <c>historic_ime_replay_detected</c> summary on the first suppression; replay is
|
||||
/// chronological, so that first suppressed line carries the earliest rejected source
|
||||
/// timestamp. Bypassed in <see cref="ImeLogTracker.SimulationMode"/> — the
|
||||
/// --replay-log-dir dev tool feeds historic lines on purpose.
|
||||
/// </summary>
|
||||
private bool SuppressIfHistoricReplay(
|
||||
bool derivedFromClock, DateTime? rawSourceTs, string eventLabel, string entityId)
|
||||
{
|
||||
if (_tracker.SimulationMode) return false;
|
||||
if (!derivedFromClock || !rawSourceTs.HasValue) return false;
|
||||
var asUtc = NormalizeUtc(rawSourceTs.Value);
|
||||
var clockNow = _clock.UtcNow;
|
||||
var age = clockNow - asUtc;
|
||||
if (age <= StaleSourceThreshold) return false; // future-skew or fresh → emit
|
||||
|
||||
bool emitSummary;
|
||||
int suppressedSoFar;
|
||||
lock (_lock)
|
||||
{
|
||||
suppressedSoFar = ++_suppressedHistoricReplayEvents;
|
||||
emitSummary = !_historicReplayNoted;
|
||||
_historicReplayNoted = true;
|
||||
}
|
||||
|
||||
if (emitSummary)
|
||||
{
|
||||
var culture = System.Globalization.CultureInfo.InvariantCulture;
|
||||
// Standard batch upload on purpose — no immediateUpload; the whole point of the
|
||||
// suppression is to stop the replayed batch from flooding the ingest.
|
||||
_post.Emit(
|
||||
eventType: SharedEventTypes.HistoricImeReplayDetected,
|
||||
source: SourceLabel,
|
||||
message: "Historic IME activity from a previous enrollment detected in the log "
|
||||
+ $"(earliest source timestamp {asUtc:o}) — its script and app events are suppressed for this session.",
|
||||
severity: EventSeverity.Info,
|
||||
data: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["decision"] = "historic_ime_replay_suppressed",
|
||||
["earliestRejectedSourceTimestamp"] = asUtc.ToString("o", culture),
|
||||
["staleThresholdHours"] = StaleSourceThreshold.TotalHours.ToString("F0", culture),
|
||||
});
|
||||
}
|
||||
|
||||
_logger?.Debug(
|
||||
$"ImeAdapter: suppressed historic {eventLabel} for {entityId} "
|
||||
+ $"(source ts {asUtc:o}, age {age.TotalHours:F1}h, suppressed so far: {suppressedSoFar})");
|
||||
return true;
|
||||
}
|
||||
|
||||
private void EmitEspPhase(string phase)
|
||||
{
|
||||
if (string.IsNullOrEmpty(phase)) return;
|
||||
@@ -504,6 +573,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs);
|
||||
|
||||
// Historic replay (a previous enrollment's log) — do not emit at all. Placed BEFORE
|
||||
// the sub-phase declaration, the timing bookkeeping and the terminal-signal dedup so
|
||||
// a replayed app neither consumes the fire-once flags nor poisons _appTimings (which
|
||||
// feeds FinalStatusBuilder) nor posts a phantom DecisionSignal. The early return also
|
||||
// covers the shadow download_progress and the app_tracking_summary snapshot below.
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, $"app-state {oldState}->{newState}", app.Id))
|
||||
return;
|
||||
|
||||
// Plan §5 Fix 2: first app activity in a given ESP phase opens the sub-phase on
|
||||
// the UI timeline via a `phase_transition` declaration event. Fire-once per ESP
|
||||
// phase; emission happens BEFORE the actual app_* event so the UI sees the phase
|
||||
@@ -616,10 +693,20 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Plan §5 Fix 4c — set-once per-app install-lifecycle timing. Called for every
|
||||
/// state transition, no-op for duplicate stamps. Returns the up-to-date
|
||||
/// Plan §5 Fix 4c — per-app install-lifecycle timing. Called for every state
|
||||
/// transition, no-op for duplicate stamps. Returns the up-to-date
|
||||
/// <see cref="AppInstallTiming"/> so the emitter can inline the values on the
|
||||
/// current event's payload without a second dictionary lookup.
|
||||
/// <para>
|
||||
/// Audit Q1: a terminal→active transition (the IME retry path — <see
|
||||
/// cref="AppPackageState.UpdateState"/> permits e.g. Error→Downloading and
|
||||
/// Skipped→Downloading) re-arms <c>Completed</c> to <c>null</c>, so the NEXT terminal
|
||||
/// transition stamps a fresh CompletedAt. StartedAt keeps its first value — the final
|
||||
/// terminal payload then covers first-start → last-terminal (total occupancy across
|
||||
/// attempts) instead of freezing at the first failure. Observed pre-fix: a user-scope
|
||||
/// app Skipped early in DeviceSetup, retried after sign-in, emitted its final
|
||||
/// Installed event with the stale early CompletedAt (durationSeconds=-2342).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private AppInstallTiming UpdateAppTiming(string appId, AppInstallationState newState, DateTime now)
|
||||
{
|
||||
@@ -627,13 +714,20 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
{
|
||||
_appTimings.TryGetValue(appId, out var current);
|
||||
|
||||
// Terminal→active (retry): the earlier terminal stamp no longer describes this
|
||||
// app's outcome — re-arm so the next terminal transition stamps fresh.
|
||||
if (current.Completed != null && IsLifecycleActive(newState))
|
||||
{
|
||||
current.Completed = null;
|
||||
}
|
||||
|
||||
// First lifecycle-active transition → record StartedAt.
|
||||
if (current.Started == null && IsLifecycleActive(newState))
|
||||
{
|
||||
current.Started = now;
|
||||
}
|
||||
|
||||
// First terminal transition → record CompletedAt.
|
||||
// Terminal transition → record CompletedAt (fresh after a re-arm).
|
||||
if (current.Completed == null && IsCompletedState(newState))
|
||||
{
|
||||
current.Completed = now;
|
||||
@@ -819,6 +913,11 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
if (!string.IsNullOrEmpty(patternId)) data["patternId"] = patternId!;
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs);
|
||||
|
||||
// Historic replay — a week-old DO summary is not this enrollment's download.
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, SharedEventTypes.DoTelemetry, app.Id))
|
||||
return;
|
||||
|
||||
TagDerivedTimestamp(data, derivedFromClock, rawSourceTs);
|
||||
|
||||
var label = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name;
|
||||
@@ -910,13 +1009,27 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
now = ResolveOccurredAt(script.ExitObservedAtUtc, out derivedFromClock, out rawSourceTs);
|
||||
else
|
||||
now = ResolveOccurredAt(out derivedFromClock, out rawSourceTs);
|
||||
|
||||
// Historic replay (a previous enrollment's log) — do not emit at all. The early
|
||||
// return also skips MaybeEmitBootstrapDetected / MaybeEmitScriptTimeoutSuspected,
|
||||
// which would otherwise fire on week-old evidence.
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, eventType, script.PolicyId))
|
||||
return;
|
||||
|
||||
TagDerivedTimestamp(data, derivedFromClock, rawSourceTs);
|
||||
|
||||
// Per-script run duration (start line → this completion, `now`). Surfaced on every
|
||||
// Per-script run duration (start line → this completion). Surfaced on every
|
||||
// script completion so the UI can flag slow / inefficient scripts, not just timed-out
|
||||
// ones. Omitted when the start line was never observed (StartedAtUtc null); clamped at 0
|
||||
// against clock skew between the (possibly clock-derived) completion stamp and start.
|
||||
//
|
||||
// When the completion timestamp was clock-derived because the source line's timestamp
|
||||
// was rejected (future-skew CMTrace clock jump — the stale case never reaches here),
|
||||
// `now` lives on the clock's timeline while StartedAtUtc was captured RAW from the
|
||||
// source log. Mixing the two produced absurd durations (session eaf3d8c4: 170 h), so
|
||||
// the duration uses the raw source pair — both ends on the same timeline, the
|
||||
// difference is the true runtime. The event stamp stays `now` (timeline position).
|
||||
//
|
||||
// L19 (delta review 2026-07-02): the same field name carries two different semantics —
|
||||
// start→own-end-signal (≈ script runtime) vs HS-SCRIPT-START→HS-NEW-RESULT (the whole
|
||||
// cycle INCLUDING IME's 30 s–3 min batched reporting latency, i.e. systematically
|
||||
@@ -924,10 +1037,16 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
// consumers never compare the two as equals. The emitting handler stamps the basis on
|
||||
// the state (health-script early-signal completions are script_runtime too); the
|
||||
// scriptType fallback covers states from paths that predate the field.
|
||||
var completionForDuration = derivedFromClock && rawSourceTs.HasValue
|
||||
? NormalizeUtc(rawSourceTs.Value)
|
||||
: now;
|
||||
if (script.StartedAtUtc.HasValue)
|
||||
{
|
||||
var durationSeconds = (now - script.StartedAtUtc.Value).TotalSeconds;
|
||||
if (durationSeconds >= 0)
|
||||
var durationSeconds = (completionForDuration - script.StartedAtUtc.Value).TotalSeconds;
|
||||
// Plausibility backstop: IME's script execution timeout is ~30 min; anything above
|
||||
// 24 h is a cross-run artifact (e.g. a stale pending start slot from a previous
|
||||
// enrollment paired with a fresh completion) — omit rather than lie.
|
||||
if (durationSeconds >= 0 && durationSeconds <= StaleSourceThreshold.TotalSeconds)
|
||||
{
|
||||
data["durationSeconds"] = durationSeconds.ToString("F2", culture);
|
||||
data["durationBasis"] = script.DurationBasis
|
||||
@@ -979,7 +1098,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
$"ImeAdapter: script completed policyId={shortId} type={script.ScriptType ?? "?"} result={script.Result ?? "?"} exit={(script.ExitCode.HasValue ? script.ExitCode.Value.ToString() : "n/a")}");
|
||||
|
||||
MaybeEmitBootstrapDetected(script, now);
|
||||
MaybeEmitScriptTimeoutSuspected(script, now);
|
||||
MaybeEmitScriptTimeoutSuspected(script, now, completionForDuration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -991,15 +1110,19 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
/// late start / pre-failed ESP. Advisory only: no state mutation, the underlying
|
||||
/// <c>script_failed</c> is still emitted separately. Health (remediation) scripts are
|
||||
/// excluded — they re-run on their own cadence and have a different timeout regime.
|
||||
/// <paramref name="durationEndUtc"/> is the timeline-consistent completion for duration
|
||||
/// math (raw source ts on rejected-timestamp emissions); <paramref name="occurredAtUtc"/>
|
||||
/// stamps the emitted event.
|
||||
/// </summary>
|
||||
private void MaybeEmitScriptTimeoutSuspected(ScriptExecutionState script, DateTime completionUtc)
|
||||
private void MaybeEmitScriptTimeoutSuspected(ScriptExecutionState script, DateTime occurredAtUtc, DateTime durationEndUtc)
|
||||
{
|
||||
if (IsRemediation(script.ScriptType)) return; // platform scripts only
|
||||
if (!script.StartedAtUtc.HasValue) return; // no start ⇒ no duration
|
||||
if (!string.Equals(script.Result, "Failed", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var duration = completionUtc - script.StartedAtUtc.Value;
|
||||
var duration = durationEndUtc - script.StartedAtUtc.Value;
|
||||
if (duration < ScriptTimeoutSuspectedThreshold) return;
|
||||
if (duration > StaleSourceThreshold) return; // implausible — cross-run artifact, not a hung script
|
||||
|
||||
if (!_tracker.TryClaimScriptTimeoutSuspected(script.PolicyId)) return; // restart-safe dedup per policyId (persisted tracker state)
|
||||
|
||||
@@ -1026,7 +1149,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
message: $"Platform script {shortId} ran {durationMinutes:F0} min and was marked Failed — suspected IME script timeout blocking the enrollment.",
|
||||
severity: EventSeverity.Warning,
|
||||
data: data,
|
||||
occurredAtUtc: completionUtc);
|
||||
occurredAtUtc: occurredAtUtc);
|
||||
|
||||
_logger?.Info(
|
||||
$"ImeAdapter: script_timeout_suspected policyId={shortId} duration={durationMinutes:F1}min result={script.Result} espPhase={espPhase ?? "(none)"}");
|
||||
@@ -1081,6 +1204,12 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
if (!string.IsNullOrEmpty(patternId)) data["patternId"] = patternId!;
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs);
|
||||
|
||||
// Historic replay — suppressing here also kills the immediateUpload flood the
|
||||
// replayed batch caused (session eaf3d8c4: 3× ingress_backpressure).
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, SharedEventTypes.ScriptStarted, info.PolicyId))
|
||||
return;
|
||||
|
||||
TagDerivedTimestamp(data, derivedFromClock, rawSourceTs);
|
||||
|
||||
var label = IsRemediation(info.ScriptType) ? "Health script" : "Platform script";
|
||||
|
||||
+14
-8
@@ -623,12 +623,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Emits a one-shot <c>agent_late_start</c> Warning when the agent had low observation
|
||||
/// Emits a one-shot <c>agent_late_start</c> event when the agent had low observation
|
||||
/// coverage: it started a long time after device boot AND lived only briefly before this
|
||||
/// terminal outcome (gate in <see cref="ObservationCoverage"/>). Such a session looks like a
|
||||
/// normal multi-minute failure in the list (StartedAt is back-dated to the earliest replayed
|
||||
/// IME-log event) but the agent actually observed only the end-state — so the operator should
|
||||
/// read the diagnosis as a post-mortem reconstruction, not live evidence. No state mutation.
|
||||
/// IME-log event) but the agent actually observed only the end-state. Severity and phrasing
|
||||
/// are outcome-calibrated via <see cref="ObservationCoverage.DescribeLateStart"/>: a failure
|
||||
/// stays a Warning with the post-mortem/hung-script note, a success is an Info about
|
||||
/// tail-only coverage (fleet 2026-07-28: tenant 659c3a90 collected 12 Warnings on healthy
|
||||
/// enrollments whose bootstrap merely ran late). No state mutation.
|
||||
/// </summary>
|
||||
private void MaybeEmitAgentLateStart(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
@@ -642,21 +645,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
out var bootToStartSeconds, out var uptimeSeconds))
|
||||
return;
|
||||
|
||||
ObservationCoverage.DescribeLateStart(
|
||||
args.Outcome, _oobeStateAtAgentStart, bootToStartSeconds, uptimeSeconds,
|
||||
out var severity, out var message, out var note);
|
||||
|
||||
_logger.Info(
|
||||
$"EnrollmentTerminationHandler: low observation coverage — agent started " +
|
||||
$"{bootToStartSeconds / 60.0:F1} min after boot and lived {uptimeSeconds:F0}s; " +
|
||||
$"emitting agent_late_start.");
|
||||
$"emitting agent_late_start ({severity}, outcome {args.Outcome}).");
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _configuration.SessionId,
|
||||
TenantId = _configuration.TenantId,
|
||||
EventType = Constants.EventTypes.AgentLateStart,
|
||||
Severity = EventSeverity.Warning,
|
||||
Severity = severity,
|
||||
Source = "EnrollmentTerminationHandler",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"Agent started {bootToStartSeconds / 60.0:F0} min after boot and observed only " +
|
||||
$"{uptimeSeconds:F0}s before terminating ({args.Outcome}) — low coverage of the enrollment window.",
|
||||
Message = message,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "bootToAgentStartSeconds", Math.Round(bootToStartSeconds, 1) },
|
||||
@@ -665,7 +671,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
{ "agentStartUtc", _agentStartTimeUtc.ToString("o") },
|
||||
{ "oobeStateAtAgentStart", _oobeStateAtAgentStart },
|
||||
{ "outcome", args.Outcome.ToString() },
|
||||
{ "note", "The agent's bootstrap (an Intune platform script) ran only near the end of the enrollment, so the agent observed the already-decided end-state rather than the failure window. Treat the diagnosis as a post-mortem; check for a platform/remediation script that hung ahead of the bootstrap (see script_timeout_suspected)." },
|
||||
{ "note", note },
|
||||
},
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
{
|
||||
@@ -76,5 +78,57 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
uptimeSeconds = Math.Max(0, uptime.TotalSeconds);
|
||||
return bootToStart >= LateStartBootThreshold && uptime <= LowCoverageUptime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Severity + operator-facing phrasing for the <c>agent_late_start</c> event, calibrated
|
||||
/// by fleet data (2026-07-28): the gate alone is outcome-blind and attached a
|
||||
/// "post-mortem / check for a hung script" Warning to sessions that in fact succeeded —
|
||||
/// there the late bootstrap is deployment hygiene, not an alarm. The OOBE state refines
|
||||
/// the phrasing but must never gate: the canonical hung-script failure terminates on the
|
||||
/// still-shown ESP screen, i.e. with OOBE reading <c>in_progress</c>.
|
||||
/// </summary>
|
||||
/// <param name="outcome">Terminal outcome the enrollment reached.</param>
|
||||
/// <param name="oobeStateAtAgentStart"><see cref="Monitoring.Interop.OobeStateReader"/>
|
||||
/// sample taken at THIS process's start. Only <c>completed</c> / <c>in_progress</c> are
|
||||
/// interpreted; <c>not_started</c> flips in after mid-OOBE reboots and <c>completed</c> is
|
||||
/// routine for post-desktop restarts, so no claim is phrased from the other values.</param>
|
||||
/// <param name="bootToStartSeconds">From <see cref="IsLowObservationCoverage"/>.</param>
|
||||
/// <param name="uptimeSeconds">From <see cref="IsLowObservationCoverage"/>.</param>
|
||||
internal static void DescribeLateStart(
|
||||
EnrollmentTerminationOutcome outcome,
|
||||
string oobeStateAtAgentStart,
|
||||
double bootToStartSeconds,
|
||||
double uptimeSeconds,
|
||||
out EventSeverity severity,
|
||||
out string message,
|
||||
out string note)
|
||||
{
|
||||
var succeeded = outcome == EnrollmentTerminationOutcome.Succeeded;
|
||||
severity = succeeded ? EventSeverity.Info : EventSeverity.Warning;
|
||||
|
||||
message = $"Agent started {bootToStartSeconds / 60.0:F0} min after boot and observed only " +
|
||||
$"{uptimeSeconds:F0}s before terminating ({outcome}) — low coverage of the enrollment window.";
|
||||
switch (oobeStateAtAgentStart)
|
||||
{
|
||||
case "completed":
|
||||
message += " OOBE was already completed when the agent started — the diagnosis is a " +
|
||||
"post-mortem reconstruction, not live observation.";
|
||||
break;
|
||||
case "in_progress":
|
||||
message += $" OOBE was still in progress at agent start — the agent did observe the " +
|
||||
$"final {uptimeSeconds:F0}s live.";
|
||||
break;
|
||||
}
|
||||
|
||||
note = succeeded
|
||||
? "The agent's bootstrap (an Intune platform script) ran late in the enrollment, so only the " +
|
||||
"final stretch was observed live; earlier timeline entries are reconstructed from IME log " +
|
||||
"replay. The enrollment itself succeeded. If this recurs across sessions, review which " +
|
||||
"platform/remediation scripts run ahead of the bootstrap (see script_timeout_suspected)."
|
||||
: "The agent's bootstrap (an Intune platform script) ran only near the end of the enrollment, " +
|
||||
"so the agent observed the already-decided end-state rather than the failure window. Treat " +
|
||||
"the diagnosis as a post-mortem; check for a platform/remediation script that hung ahead of " +
|
||||
"the bootstrap (see script_timeout_suspected).";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Serialization;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 4910a5a5 (2026-07-23) — advisory recovery. Field failure: DeviceSetup/Apps
|
||||
/// failed (culprit app never started installing), the advisory defang armed the
|
||||
/// <c>advisory_completion</c> window, the user pressed "Try again" ~23 min later, the apps
|
||||
/// re-ran to 10/10 and <c>DeviceSetupProvisioningComplete</c> arrived — yet the window
|
||||
/// fired 30 min after the advisory and un-defanged the stale failure
|
||||
/// (<c>enrollment_failed: esp_terminal_failure</c>) on a live enrollment mid-AccountSetup,
|
||||
/// because the advisory variant was exempt from ALL re-arm/rebase guards. Coverage:
|
||||
/// <list type="bullet">
|
||||
/// <item>The advisory records the failed category + culprit apps.</item>
|
||||
/// <item>A ProvisioningComplete of the SAME category sets the resolved fact + emits
|
||||
/// <c>esp_failure_advisory_resolved</c>; a different category does not.</item>
|
||||
/// <item>Deadline fire after recovery + enforcement progress → re-arm, not Failed.</item>
|
||||
/// <item>Deadline fire after recovery without progress → Failed with the truthful
|
||||
/// <c>esp_recovered_without_completion_evidence</c> reason and
|
||||
/// LastFailureTrigger=DeadlineFired (likely-stuck promotion off).</item>
|
||||
/// <item>Unrecovered advisory keeps the legacy un-defang exactly (regression guard).</item>
|
||||
/// <item>Reboot re-bases a recovered-advisory window (and still does NOT touch an
|
||||
/// unrecovered one).</item>
|
||||
/// <item>Snapshot round-trip + census surfacing of the new facts.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class AdvisoryRecoveryTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 7, 23, 3, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
private static DecisionSignal MakeSignal(
|
||||
long ordinal,
|
||||
DecisionSignalKind kind,
|
||||
DateTime occurredAtUtc,
|
||||
IReadOnlyDictionary<string, string>? payload = null)
|
||||
{
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: kind,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
sourceOrigin: "test",
|
||||
evidence: new Evidence(EvidenceKind.Synthetic, $"{kind}-{ordinal}", "test"),
|
||||
payload: payload);
|
||||
}
|
||||
|
||||
private static DecisionSignal DeadlineFired(long ordinal, DateTime occurredAtUtc) =>
|
||||
MakeSignal(ordinal, DecisionSignalKind.DeadlineFired, occurredAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.AdvisoryCompletion });
|
||||
|
||||
private static DecisionEffect? FindTimelineEffect(DecisionStep step, string eventType) =>
|
||||
step.Effects.FirstOrDefault(e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry
|
||||
&& e.Parameters != null
|
||||
&& e.Parameters.TryGetValue("eventType", out var et)
|
||||
&& et == eventType);
|
||||
|
||||
private static ActiveDeadline? FindDeadline(DecisionState state, string name) =>
|
||||
state.Deadlines.FirstOrDefault(d => d.Name == name);
|
||||
|
||||
/// <summary>
|
||||
/// Replays the session-4910a5a5 shape up to (but excluding) the ESP terminal failure:
|
||||
/// Classic UserDriven, ContinueAnyway profile, Hello disabled, DeviceSetup entered,
|
||||
/// AccountSetup entered (04:18, BEFORE the 04:34 DeviceSetup/Apps failure — the IME
|
||||
/// phase line landed while device apps were still at 9/10).
|
||||
/// </summary>
|
||||
private static DecisionState SetupSession(DecisionEngine engine)
|
||||
{
|
||||
var state = DecisionState.CreateInitial("sess-4910a5a5", "tenant-e46bc88e", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
5, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.SkipUserEsp] = "false",
|
||||
[SignalPayloadKeys.SkipDeviceEsp] = "false",
|
||||
[SignalPayloadKeys.EspSyncFailureTimeoutMinutes] = "60",
|
||||
[SignalPayloadKeys.EspAllowContinueAnyway] = "true",
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
8, DecisionSignalKind.HelloPolicyDetected, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "false" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(6),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(48),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
return state;
|
||||
}
|
||||
|
||||
private static DecisionStep ApplyDeviceSetupAppsFailure(DecisionEngine engine, DecisionState state, long ordinal = 50) =>
|
||||
engine.Reduce(state, MakeSignal(
|
||||
ordinal, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(64),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Apps_Failed",
|
||||
["failedSubcategory"] = "Apps",
|
||||
["category"] = "DeviceSetup",
|
||||
["likelyCulpritApps"] = "Oriflame Cosmectics - Teams Backgroundsv v2",
|
||||
["likelyCulpritAppCount"] = "1",
|
||||
}));
|
||||
|
||||
private static DecisionStep ApplyDeviceSetupResolved(DecisionEngine engine, DecisionState state, long ordinal = 60) =>
|
||||
engine.Reduce(state, MakeSignal(
|
||||
ordinal, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(87)));
|
||||
|
||||
// ================================================== advisory arming context ====
|
||||
|
||||
[Fact]
|
||||
public void Advisory_RecordsFailedCategory_AndCulpritApps()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
|
||||
var step = ApplyDeviceSetupAppsFailure(engine, state);
|
||||
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
Assert.Equal("DeviceSetup", step.NewState.EspAdvisoryFailureCategory?.Value);
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureResolvedUtc);
|
||||
|
||||
var advisory = FindTimelineEffect(step, "esp_failure_advisory");
|
||||
Assert.NotNull(advisory);
|
||||
Assert.Equal("Oriflame Cosmectics - Teams Backgroundsv v2", advisory!.Parameters!["likelyCulpritApps"]);
|
||||
}
|
||||
|
||||
// ===================================================== recovery hook ====
|
||||
|
||||
[Fact]
|
||||
public void DeviceSetupComplete_SameCategory_ResolvesAdvisory_AndEmitsStoryEvent()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
|
||||
var step = ApplyDeviceSetupResolved(engine, state);
|
||||
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureResolvedUtc);
|
||||
Assert.Equal(60, step.NewState.EspAdvisoryFailureResolvedUtc!.SourceSignalOrdinal);
|
||||
// The anchor itself stays — the advisory happened; it just no longer blocks.
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
var resolved = FindTimelineEffect(step, "esp_failure_advisory_resolved");
|
||||
Assert.NotNull(resolved);
|
||||
Assert.Equal("DeviceSetup", resolved!.Parameters!["category"]);
|
||||
Assert.Equal("23", resolved.Parameters["minutesSinceAdvisory"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeviceSetupComplete_DifferentCategory_DoesNotResolve()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
// Advisory for AccountSetup — DeviceSetup resolving must not defuse it.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
50, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(64),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_AccountSetup_Apps_Failed",
|
||||
["failedSubcategory"] = "Apps",
|
||||
["category"] = "AccountSetup",
|
||||
})).NewState;
|
||||
|
||||
var step = ApplyDeviceSetupResolved(engine, state);
|
||||
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureResolvedUtc);
|
||||
Assert.Null(FindTimelineEffect(step, "esp_failure_advisory_resolved"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeviceSetupComplete_Duplicate_ResolvesOnlyOnce()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
// Post-reboot duplicate (AnchorAlreadySet passthrough branch).
|
||||
var duplicate = engine.Reduce(state, MakeSignal(
|
||||
65, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(92)));
|
||||
|
||||
Assert.Equal(60, duplicate.NewState.EspAdvisoryFailureResolvedUtc!.SourceSignalOrdinal);
|
||||
Assert.Null(FindTimelineEffect(duplicate, "esp_failure_advisory_resolved"));
|
||||
}
|
||||
|
||||
// ================================================ deadline fire semantics ====
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFire_AfterRecovery_WithEnforcementProgress_ReArmsInsteadOfFailing()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
// Post-reboot IME re-assert of AccountSetup (session shape: 05:02:59, ordinal 654)
|
||||
// — user-phase enforcement demonstrably progressed since the arming at ordinal 50.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(92),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94)));
|
||||
|
||||
// Not failed — the window re-arms with fresh baselines instead.
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.NotEqual(SessionStage.Failed, step.NewState.Stage);
|
||||
var rearmed = FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion);
|
||||
Assert.NotNull(rearmed);
|
||||
Assert.Equal(T0.AddMinutes(94).AddMinutes(30), rearmed!.DueAtUtc);
|
||||
Assert.Contains(step.Effects, e => e.Kind == DecisionEffectKind.ScheduleDeadline);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFire_AfterRecovery_NoProgress_FailsWithRecoveredReason()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94)));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
|
||||
var failed = FindTimelineEffect(step, "enrollment_failed");
|
||||
Assert.NotNull(failed);
|
||||
// Truthful reason: the original failure recovered — do not blame it.
|
||||
Assert.Equal("esp_recovered_without_completion_evidence", failed!.Parameters!["reason"]);
|
||||
Assert.Equal("true", failed.Parameters["failureRecovered"]);
|
||||
Assert.Equal("advisory_recovered_window_expired_without_completion_evidence", failed.Parameters["advisoryReason"]);
|
||||
// Likely-stuck app promotion stays off: apps are not stuck, they succeeded.
|
||||
Assert.Equal(nameof(DecisionSignalKind.DeadlineFired), step.NewState.LastFailureTrigger?.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFire_WithoutRecovery_KeepsLegacyUndefang()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
// No recovery — even with enforcement progress the advisory variant must
|
||||
// un-defang exactly as before (its anchor failure never un-happened).
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(92),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94)));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
var failed = FindTimelineEffect(step, "enrollment_failed");
|
||||
Assert.NotNull(failed);
|
||||
Assert.Equal("esp_terminal_failure", failed!.Parameters!["reason"]);
|
||||
Assert.Equal(nameof(DecisionSignalKind.EspTerminalFailure), step.NewState.LastFailureTrigger?.Value);
|
||||
}
|
||||
|
||||
// ======================================================= reboot rebase ====
|
||||
|
||||
[Fact]
|
||||
public void Reboot_AfterRecovery_RebasesAdvisoryWindow()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92)));
|
||||
|
||||
Assert.Contains(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.CancelDeadline
|
||||
&& e.CancelDeadlineName == DeadlineNames.AdvisoryCompletion);
|
||||
var rebased = FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion);
|
||||
Assert.NotNull(rebased);
|
||||
Assert.Equal(T0.AddMinutes(92).AddMinutes(30), rebased!.DueAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reboot_WithoutRecovery_DoesNotTouchAdvisoryWindow()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
var originalDue = FindDeadline(state, DeadlineNames.AdvisoryCompletion)!.DueAtUtc;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92)));
|
||||
|
||||
Assert.DoesNotContain(step.Effects, e => e.Kind == DecisionEffectKind.CancelDeadline);
|
||||
Assert.Equal(originalDue, FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion)!.DueAtUtc);
|
||||
}
|
||||
|
||||
// ============================================== full session-4910a5a5 replay ====
|
||||
|
||||
[Fact]
|
||||
public void Session4910a5a5_FailureRecoveryRebootFire_DoesNotFailTheLiveEnrollment()
|
||||
{
|
||||
// Field ordinals: 527 EspTerminalFailure → 611 DeviceSetupProvisioningComplete
|
||||
// (post "Try again", apps 10/10) → 635 SystemRebootObserved → 654 AccountSetup
|
||||
// re-assert → 671 DeadlineFired. Pre-fix outcome: enrollment_failed at the fire.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state, ordinal: 527).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state, ordinal: 611).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
635, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
654, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(93),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(671, T0.AddMinutes(94)));
|
||||
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.NotEqual(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.NotNull(FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion));
|
||||
}
|
||||
|
||||
// =============================================== serialization + census ====
|
||||
|
||||
[Fact]
|
||||
public void SnapshotRoundTrip_PreservesRecoveryFacts()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var roundtripped = StateSerializer.Deserialize(StateSerializer.Serialize(state));
|
||||
|
||||
Assert.Equal("DeviceSetup", roundtripped.EspAdvisoryFailureCategory?.Value);
|
||||
Assert.Equal(state.EspAdvisoryFailureResolvedUtc!.Value, roundtripped.EspAdvisoryFailureResolvedUtc?.Value);
|
||||
Assert.Equal(state.EspAdvisoryFailureResolvedUtc.SourceSignalOrdinal, roundtripped.EspAdvisoryFailureResolvedUtc?.SourceSignalOrdinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacySnapshot_WithoutRecoveryFacts_DeserializesToNull()
|
||||
{
|
||||
// Simulate a snapshot written by a pre-fix agent: strip the new properties from
|
||||
// the JSON entirely — deserialization must default them to null (additive compat).
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var legacy = Newtonsoft.Json.Linq.JObject.Parse(StateSerializer.Serialize(state));
|
||||
foreach (var prop in legacy.Properties()
|
||||
.Where(p => p.Name.IndexOf("EspAdvisoryFailureResolved", StringComparison.OrdinalIgnoreCase) >= 0
|
||||
|| p.Name.IndexOf("EspAdvisoryFailureCategory", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
.ToList())
|
||||
{
|
||||
prop.Remove();
|
||||
}
|
||||
|
||||
var roundtripped = StateSerializer.Deserialize(legacy.ToString());
|
||||
Assert.Null(roundtripped.EspAdvisoryFailureResolvedUtc);
|
||||
Assert.Null(roundtripped.EspAdvisoryFailureCategory);
|
||||
// The pre-existing advisory anchor still round-trips.
|
||||
Assert.NotNull(roundtripped.EspAdvisoryFailureRecordedUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Census_SurfacesAdvisoryResolved()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var census = DecisionStateSignalCensus.Build(state);
|
||||
|
||||
Assert.Contains("esp_advisory_failure_resolved", census.SignalsSeen);
|
||||
Assert.True(census.SignalTimestamps.ContainsKey("espAdvisoryFailureResolved"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,10 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// state-change-only dedupe anchor for the completion_waiting event → 17.
|
||||
// Session 772fe502 (2026-07-13) added `helloWizardStartedUtc` — genuine
|
||||
// Shell-Core 62404 wizard launch; vetoes the policy-disabled Hello skip → 18.
|
||||
Assert.Equal(18, facts.Count);
|
||||
// Session 4910a5a5 (2026-07-23) added `espAdvisoryFailureResolvedUtc` (the failed
|
||||
// category recovered — lifts the advisory's re-arm/rebase guard exemption) and
|
||||
// `espAdvisoryFailureCategory` (which category the advisory was for) → 20.
|
||||
Assert.Equal(20, facts.Count);
|
||||
Assert.All(facts.Values, v => Assert.Null(v));
|
||||
}
|
||||
|
||||
@@ -236,10 +239,12 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// `completionWaitingFingerprint` as the state-change-only dedupe anchor for
|
||||
// the completion_waiting event → 17. Session 772fe502 (2026-07-13) added
|
||||
// `helloWizardStartedUtc` — the genuine Shell-Core 62404 wizard-launch fact
|
||||
// that vetoes/retracts the policy-disabled Hello skip → 18. If this number
|
||||
// ever changes, both the count expectation AND the actual snapshot output
|
||||
// need to evolve in lockstep.
|
||||
Assert.Equal(18, expectedFactKeys.Count);
|
||||
// that vetoes/retracts the policy-disabled Hello skip → 18. Session 4910a5a5
|
||||
// (2026-07-23) added `espAdvisoryFailureResolvedUtc` + `espAdvisoryFailureCategory`
|
||||
// — advisory-recovery facts that lift the AdvisoryCompletion guard exemption → 20.
|
||||
// If this number ever changes, both the count expectation AND the actual snapshot
|
||||
// output need to evolve in lockstep.
|
||||
Assert.Equal(20, expectedFactKeys.Count);
|
||||
|
||||
var snapshot = DecisionStateSnapshotBuilder.Build(DecisionState.CreateInitial("s", "t"));
|
||||
var facts = (Dictionary<string, object?>)snapshot["facts"]!;
|
||||
|
||||
+333
@@ -0,0 +1,333 @@
|
||||
using System.Text.Json;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Functions.Apps;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// F1 PR1 (insights spec §F1, source-data audit Q2/Q3): app identity on the name-keyed
|
||||
/// summary row and the ESP-blocking join.
|
||||
///
|
||||
/// Ingest side: <c>appId</c> is adopted from the app events; a second appId under the same
|
||||
/// name flags the row (<see cref="AppInstallSummary.AppIdCollision"/>) — in-batch and
|
||||
/// cross-batch. Store side: sentinel-gated columns + sticky collision. Join side:
|
||||
/// <see cref="EspBlockingSets"/> is positive evidence only, and
|
||||
/// <see cref="TableStorageService.ShouldStampEspBlocking"/> never stamps collision rows,
|
||||
/// id-less rows, or non-members. Aggregate side: collision rows leave every per-app fleet
|
||||
/// aggregate with a disclosed exclusion count.
|
||||
/// </summary>
|
||||
public class AppInstallIdentityAndEspBlockingTests
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 7, 26, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const string IdA = "aaaaaaaa-1111-2222-3333-444444444444";
|
||||
private const string IdB = "bbbbbbbb-1111-2222-3333-444444444444";
|
||||
|
||||
// ── ingest fold: appId adoption + in-batch collision ────────────────────
|
||||
|
||||
private static Dictionary<string, AppInstallAggregationState> Aggregate(params EnrollmentEvent[] events)
|
||||
{
|
||||
var summaries = new Dictionary<string, AppInstallAggregationState>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var evt in events)
|
||||
EventIngestProcessor.AggregateAppInstallEvent(evt, "tenant", "session", summaries);
|
||||
return summaries;
|
||||
}
|
||||
|
||||
private static EnrollmentEvent AppEvent(string eventType, string? appId = null, string appName = "App")
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = eventType,
|
||||
Timestamp = T0,
|
||||
Data = new Dictionary<string, object> { ["appName"] = appName },
|
||||
};
|
||||
if (appId != null) evt.Data["appId"] = appId;
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ingest_AdoptsAppId_FromFirstObservingEvent()
|
||||
{
|
||||
var s = Aggregate(
|
||||
AppEvent("app_install_started", IdA),
|
||||
AppEvent("app_install_completed", IdA))["App"].Summary;
|
||||
|
||||
Assert.Equal(IdA, s.AppId);
|
||||
Assert.False(s.AppIdCollision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ingest_SameName_DifferentAppId_FlagsCollision_FirstSeenIdWins()
|
||||
{
|
||||
// Device- + user-scope assignment of the same display name in one batch (audit Q3:
|
||||
// observed in production as two "Company Portal" appIds in one session).
|
||||
var s = Aggregate(
|
||||
AppEvent("app_install_started", IdA),
|
||||
AppEvent("app_install_started", IdB))["App"].Summary;
|
||||
|
||||
Assert.True(s.AppIdCollision);
|
||||
Assert.Equal(IdA, s.AppId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ingest_AppIdCaseDrift_IsNotACollision()
|
||||
{
|
||||
var s = Aggregate(
|
||||
AppEvent("app_install_started", IdA),
|
||||
AppEvent("app_install_completed", IdA.ToUpperInvariant()))["App"].Summary;
|
||||
|
||||
Assert.False(s.AppIdCollision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ingest_EventsWithoutAppId_LeaveSentinelEmpty()
|
||||
{
|
||||
var s = Aggregate(AppEvent("app_install_completed"))["App"].Summary;
|
||||
Assert.Equal(string.Empty, s.AppId);
|
||||
Assert.False(s.AppIdCollision);
|
||||
}
|
||||
|
||||
// ── reconcile: cross-batch identity ─────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_AdoptsStoredAppId_WhenIncomingBatchHasNone()
|
||||
{
|
||||
var existing = new TableEntity("tenant", "session_App") { ["AppId"] = IdA };
|
||||
var summary = new AppInstallSummary { AppName = "App", StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.Equal(IdA, summary.AppId);
|
||||
Assert.False(summary.AppIdCollision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_CrossBatchDifferentAppId_FlagsCollision_StoredIdWins()
|
||||
{
|
||||
var existing = new TableEntity("tenant", "session_App") { ["AppId"] = IdA };
|
||||
var summary = new AppInstallSummary { AppName = "App", AppId = IdB, StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.True(summary.AppIdCollision);
|
||||
Assert.Equal(IdA, summary.AppId); // first-seen (stored) identity wins, matching the fold rule
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_CollisionIsSticky_AndEspBlockingVerdictIsAdopted()
|
||||
{
|
||||
var existing = new TableEntity("tenant", "session_App")
|
||||
{
|
||||
["AppId"] = IdA,
|
||||
["AppIdCollision"] = true,
|
||||
["EspBlocking"] = true,
|
||||
};
|
||||
var summary = new AppInstallSummary { AppName = "App", AppId = IdA, StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.True(summary.AppIdCollision); // one-way sticky
|
||||
Assert.True(summary.EspBlocking); // terminal-path verdict survives late batches
|
||||
}
|
||||
|
||||
// ── entity builder: sentinel gating ─────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void EntityBuilder_IdentityColumns_AreSentinelGated()
|
||||
{
|
||||
var empty = TableStorageService.BuildAppInstallSummaryEntity(
|
||||
new AppInstallSummary { AppName = "App" }, "rk");
|
||||
// Absent columns → Merge-mode preserves prior values / stays unknown.
|
||||
Assert.False(empty.ContainsKey("AppId"));
|
||||
Assert.False(empty.ContainsKey("EspBlocking"));
|
||||
Assert.False(empty.ContainsKey("AppIdCollision"));
|
||||
|
||||
var full = TableStorageService.BuildAppInstallSummaryEntity(
|
||||
new AppInstallSummary { AppName = "App", AppId = IdA, EspBlocking = true, AppIdCollision = true }, "rk");
|
||||
Assert.Equal(IdA, full.GetString("AppId"));
|
||||
Assert.True(full.GetBoolean("EspBlocking"));
|
||||
Assert.True(full.GetBoolean("AppIdCollision"));
|
||||
}
|
||||
|
||||
// ── EspBlockingSets: positive-evidence parsing ──────────────────────────
|
||||
|
||||
private static Dictionary<string, object> Lists(
|
||||
object? win32 = null, object? userWin32 = null, object? msi = null, object? pfn = null,
|
||||
long? win32Count = null)
|
||||
{
|
||||
var d = new Dictionary<string, object> { ["source"] = "registry_firstsync" };
|
||||
if (win32 != null) d["espTrackedWin32AppIds"] = win32;
|
||||
if (userWin32 != null) d["espTrackedUserWin32AppIds"] = userWin32;
|
||||
if (msi != null) d["espTrackedMsiProductCodes"] = msi;
|
||||
if (pfn != null) d["espTrackedModernAppPfns"] = pfn;
|
||||
if (win32Count.HasValue) d["espTrackedWin32Count"] = win32Count.Value;
|
||||
return d;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sets_PayloadWithoutListKeys_YieldsNull_NoListsIsNotEmptyLists()
|
||||
{
|
||||
Assert.Null(EspBlockingSets.FromEventData(null));
|
||||
Assert.Null(EspBlockingSets.FromEventData(Lists())); // probe found no Diagnostics key
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sets_MembershipIsCaseInsensitive_AcrossAllFourLists()
|
||||
{
|
||||
// Storage rehydration shape: List<object> (EventDataNormalizer).
|
||||
var sets = EspBlockingSets.FromEventData(Lists(
|
||||
win32: new List<object> { IdA },
|
||||
msi: new List<object> { "{11111111-2222-3333-4444-555555555555}" },
|
||||
pfn: new List<object> { "Contoso.App_abc123" }))!;
|
||||
|
||||
Assert.True(sets.Contains(IdA.ToUpperInvariant()));
|
||||
Assert.True(sets.Contains("{11111111-2222-3333-4444-555555555555}"));
|
||||
Assert.True(sets.Contains("Contoso.App_abc123"));
|
||||
Assert.False(sets.Contains(IdB)); // absent ⇒ unknown at the caller, never false here
|
||||
Assert.False(sets.Contains(null));
|
||||
Assert.Equal(3, sets.ListedCount);
|
||||
Assert.False(sets.IsTruncated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sets_EmittedCountAboveListLength_MeansTruncated()
|
||||
{
|
||||
var sets = EspBlockingSets.FromEventData(Lists(
|
||||
win32: new List<object> { IdA }, win32Count: 60))!;
|
||||
|
||||
Assert.True(sets.IsTruncated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sets_UserScopeSubset_IsExposedForCoverageLabeling()
|
||||
{
|
||||
var sets = EspBlockingSets.FromEventData(Lists(
|
||||
win32: new List<object> { IdA, IdB },
|
||||
userWin32: new List<object> { IdB }))!;
|
||||
|
||||
Assert.Contains(IdB, sets.UserWin32AppIds);
|
||||
Assert.DoesNotContain(IdA, sets.UserWin32AppIds);
|
||||
}
|
||||
|
||||
// ── terminal-path stamping predicate ────────────────────────────────────
|
||||
|
||||
private static TableEntity Row(string? appId = null, bool? collision = null, bool? espBlocking = null)
|
||||
{
|
||||
var row = new TableEntity("tenant", "session_App");
|
||||
if (appId != null) row["AppId"] = appId;
|
||||
if (collision.HasValue) row["AppIdCollision"] = collision.Value;
|
||||
if (espBlocking.HasValue) row["EspBlocking"] = espBlocking.Value;
|
||||
return row;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stamp_OnlyPositiveMembers_NeverIdlessCollidedOrAlreadyStamped()
|
||||
{
|
||||
var sets = EspBlockingSets.FromEventData(Lists(win32: new List<object> { IdA }))!;
|
||||
|
||||
Assert.True(TableStorageService.ShouldStampEspBlocking(Row(IdA), sets));
|
||||
Assert.False(TableStorageService.ShouldStampEspBlocking(Row(), sets)); // no identity → unknown
|
||||
Assert.False(TableStorageService.ShouldStampEspBlocking(Row(IdB), sets)); // absent ⇒ unknown, never false
|
||||
Assert.False(TableStorageService.ShouldStampEspBlocking(Row(IdA, collision: true), sets)); // ambiguous identity
|
||||
Assert.False(TableStorageService.ShouldStampEspBlocking(Row(IdA, espBlocking: true), sets)); // idempotent
|
||||
}
|
||||
|
||||
// ── per-app fleet aggregates: collision rows leave, disclosed ───────────
|
||||
|
||||
private static AppInstallSummary Summary(
|
||||
string appName, string status = "Succeeded", bool collision = false, int duration = 30)
|
||||
=> new()
|
||||
{
|
||||
TenantId = "00000000-0000-0000-0000-000000000fa1",
|
||||
SessionId = "s-1",
|
||||
AppName = appName,
|
||||
Status = status,
|
||||
TerminalState = status == "Succeeded" ? "Installed" : "Error",
|
||||
DurationSeconds = duration,
|
||||
StartedAt = DateTime.UtcNow.AddHours(-2),
|
||||
AppIdCollision = collision,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void AppMetricsPayload_ExcludesCollisionRowsFromGroups_DisclosesCount()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
Summary("Clean App"),
|
||||
Summary("Clean App"),
|
||||
Summary("Company Portal", collision: true),
|
||||
Summary("Company Portal", status: "Failed", collision: true),
|
||||
};
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(MetricsMath.BuildAppMetricsPayload(summaries));
|
||||
|
||||
Assert.Equal(1, root.GetProperty("totalApps").GetInt32()); // collided name forms no group
|
||||
Assert.Equal(4, root.GetProperty("totalInstalls").GetInt32()); // raw row count unchanged
|
||||
Assert.Equal(2, root.GetProperty("totalCollisionExcluded").GetInt32());
|
||||
Assert.Empty(root.GetProperty("topFailingApps").EnumerateArray()); // the collided Failed row ranks nowhere
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AppsList_ExcludesCollisionRows_DisclosesCount()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
Summary("Clean App"),
|
||||
Summary("Company Portal", collision: true),
|
||||
};
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var apps = root.GetProperty("apps").EnumerateArray().ToList();
|
||||
Assert.Single(apps);
|
||||
Assert.Equal("Clean App", apps[0].GetProperty("appName").GetString());
|
||||
Assert.Equal(2, root.GetProperty("totalInstalls").GetInt32());
|
||||
Assert.Equal(1, root.GetProperty("collisionExcluded").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppAnalytics_ExcludesCollisionRows_DisclosesCount()
|
||||
{
|
||||
var repo = new Mock<ISessionRepository>();
|
||||
repo.Setup(r => r.GetSessionAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ReturnsAsync((SessionSummary?)null);
|
||||
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
Summary("Company Portal"),
|
||||
Summary("Company Portal", status: "Failed", collision: true), // the second identity's failure
|
||||
};
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(await AppsAnalyticsHelper.BuildAnalyticsResponseAsync(
|
||||
summaries, repo.Object, "Company Portal", days: 30));
|
||||
|
||||
Assert.Equal(1, root.GetProperty("collisionExcluded").GetInt32());
|
||||
var stats = root.GetProperty("summary");
|
||||
Assert.Equal(1, stats.GetProperty("totalInstalls").GetInt32());
|
||||
Assert.Equal(0, stats.GetProperty("failed").GetInt32()); // ambiguous failure not attributed
|
||||
Assert.Equal(0d, stats.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppAnalytics_AllRowsCollided_ReturnsEmptyShape_WithDisclosure()
|
||||
{
|
||||
var repo = new Mock<ISessionRepository>();
|
||||
repo.Setup(r => r.GetSessionAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ReturnsAsync((SessionSummary?)null);
|
||||
|
||||
var summaries = new List<AppInstallSummary> { Summary("Company Portal", collision: true) };
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(await AppsAnalyticsHelper.BuildAnalyticsResponseAsync(
|
||||
summaries, repo.Object, "Company Portal", days: 30));
|
||||
|
||||
Assert.Equal(1, root.GetProperty("collisionExcluded").GetInt32());
|
||||
Assert.Equal(0, root.GetProperty("summary").GetProperty("totalInstalls").GetInt32());
|
||||
}
|
||||
}
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// PR0 (2026-07-26, source-data audit §0.5 of tasks/insights-expansion-spec.md).
|
||||
///
|
||||
/// Ingest side: the agent emits <c>app_install_completed</c> for EVERY terminal transition —
|
||||
/// including Skipped/Postponed no-ops (V1 wire parity; production: 78 % of zero-duration rows
|
||||
/// were skips). The payload's <c>state</c> field must be persisted as
|
||||
/// <see cref="AppInstallSummary.TerminalState"/> so metrics can exclude skips from durations
|
||||
/// and rates.
|
||||
///
|
||||
/// Store side: <see cref="TableStorageService.ReconcileAppInstallSummaryWithExisting"/> pins the
|
||||
/// cross-batch contract — earliest StartedAt wins, terminal status is sticky, and the Q4
|
||||
/// out-of-order case (terminal batch first, started batch later) recomputes the duration.
|
||||
/// </summary>
|
||||
public class AppInstallTerminalStateAndReconcileTests
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 7, 26, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// ── ingest: TerminalState fold ──────────────────────────────────────────
|
||||
|
||||
private static Dictionary<string, AppInstallAggregationState> Aggregate(params EnrollmentEvent[] events)
|
||||
{
|
||||
var summaries = new Dictionary<string, AppInstallAggregationState>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var evt in events)
|
||||
EventIngestProcessor.AggregateAppInstallEvent(evt, "tenant", "session", summaries);
|
||||
return summaries;
|
||||
}
|
||||
|
||||
private static EnrollmentEvent AppEvent(string eventType, DateTime ts, string? state = null, string appName = "App")
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = eventType,
|
||||
Timestamp = ts,
|
||||
Data = new Dictionary<string, object> { ["appName"] = appName },
|
||||
};
|
||||
if (state != null) evt.Data["state"] = state;
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Installed")]
|
||||
[InlineData("Skipped")]
|
||||
[InlineData("Postponed")]
|
||||
public void Completed_PersistsPayloadState(string state)
|
||||
{
|
||||
var s = Aggregate(AppEvent("app_install_completed", T0, state))["App"].Summary;
|
||||
|
||||
Assert.Equal("Succeeded", s.Status); // backward-compatible status
|
||||
Assert.Equal(state, s.TerminalState); // new precision column
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Completed_WithoutState_LeavesSentinelEmpty_NeverGuessed()
|
||||
{
|
||||
var s = Aggregate(AppEvent("app_install_completed", T0))["App"].Summary;
|
||||
Assert.Equal(string.Empty, s.TerminalState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Completed_WithNonTerminalOrGarbageState_LeavesSentinelEmpty()
|
||||
{
|
||||
// A payload state outside the terminal set (bug or drift) must not be persisted.
|
||||
var s = Aggregate(AppEvent("app_install_completed", T0, "Downloading"))["App"].Summary;
|
||||
Assert.Equal(string.Empty, s.TerminalState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_SetsError()
|
||||
{
|
||||
var s = Aggregate(AppEvent("app_install_failed", T0))["App"].Summary;
|
||||
Assert.Equal("Failed", s.Status);
|
||||
Assert.Equal("Error", s.TerminalState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailedThenCompleted_RetrySucceeds_LastTerminalWins()
|
||||
{
|
||||
// IME retry path (audit Q1): Error → retry → Installed. The later completed event is
|
||||
// the authoritative outcome for Status AND TerminalState.
|
||||
var s = Aggregate(
|
||||
AppEvent("app_install_failed", T0),
|
||||
AppEvent("app_install_completed", T0.AddMinutes(10), "Installed"))["App"].Summary;
|
||||
|
||||
Assert.Equal("Succeeded", s.Status);
|
||||
Assert.Equal("Installed", s.TerminalState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SkippedEvent_SetsSkipped_ButNeverOverridesStrongerTerminal()
|
||||
{
|
||||
var fresh = Aggregate(AppEvent("app_install_skipped", T0))["App"].Summary;
|
||||
Assert.Equal("Succeeded", fresh.Status);
|
||||
Assert.Equal("Skipped", fresh.TerminalState);
|
||||
|
||||
var afterFailure = Aggregate(
|
||||
AppEvent("app_install_failed", T0),
|
||||
AppEvent("app_install_skipped", T0.AddSeconds(5)))["App"].Summary;
|
||||
Assert.Equal("Error", afterFailure.TerminalState);
|
||||
}
|
||||
|
||||
// ── store: cross-batch reconcile ────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_Q4_OutOfOrder_RecomputesDurationFromStoredCompletedAt()
|
||||
{
|
||||
// Terminal batch landed first: row has CompletedAt but no usable duration.
|
||||
var existing = new TableEntity("tenant", "session_App")
|
||||
{
|
||||
["Status"] = "Succeeded",
|
||||
["StartedAt"] = new DateTimeOffset(T0.AddMinutes(5)), // late (completed-event) stamp
|
||||
["CompletedAt"] = new DateTimeOffset(T0.AddMinutes(5)),
|
||||
};
|
||||
// Started batch arrives now with the true (earlier) start.
|
||||
var summary = new AppInstallSummary
|
||||
{
|
||||
AppName = "App", SessionId = "session", TenantId = "tenant",
|
||||
Status = "InProgress", StartedAt = T0,
|
||||
};
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.Equal(T0, summary.StartedAt); // earlier start kept
|
||||
Assert.Equal(T0.AddMinutes(5), summary.CompletedAt); // endpoint adopted from the row
|
||||
Assert.Equal(300, summary.DurationSeconds); // recomputed — was permanently unset before PR0
|
||||
Assert.Equal("Succeeded", summary.Status); // sticky terminal (below)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_TerminalStatusIsSticky_AgainstLateInProgressBatch()
|
||||
{
|
||||
var existing = new TableEntity("tenant", "session_App") { ["Status"] = "Failed" };
|
||||
var summary = new AppInstallSummary { AppName = "App", Status = "InProgress", StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.Equal("Failed", summary.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_TerminalToTerminal_StaysWithIncomingEvidence()
|
||||
{
|
||||
// Failed row + succeeded batch (successful retry) → the new terminal wins.
|
||||
var existing = new TableEntity("tenant", "session_App") { ["Status"] = "Failed" };
|
||||
var summary = new AppInstallSummary { AppName = "App", Status = "Succeeded", StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.Equal("Succeeded", summary.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_Q4Guard_DoesNotFireWithoutStoredCompletedAt()
|
||||
{
|
||||
var existing = new TableEntity("tenant", "session_App") { ["Status"] = "InProgress" };
|
||||
var summary = new AppInstallSummary { AppName = "App", Status = "InProgress", StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.Null(summary.CompletedAt);
|
||||
Assert.Equal(0, summary.DurationSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reconcile_Q4Guard_IgnoresStoredCompletedAtBeforeIncomingStart()
|
||||
{
|
||||
// Negative interval (clock mess) must not produce a negative duration.
|
||||
var existing = new TableEntity("tenant", "session_App")
|
||||
{
|
||||
["Status"] = "Succeeded",
|
||||
["CompletedAt"] = new DateTimeOffset(T0.AddMinutes(-10)),
|
||||
};
|
||||
var summary = new AppInstallSummary { AppName = "App", Status = "InProgress", StartedAt = T0 };
|
||||
|
||||
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
|
||||
|
||||
Assert.Null(summary.CompletedAt);
|
||||
Assert.Equal(0, summary.DurationSeconds);
|
||||
}
|
||||
|
||||
// ── entity builder: sentinel gating for the new column ─────────────────
|
||||
|
||||
[Fact]
|
||||
public void EntityBuilder_TerminalState_IsSentinelGated()
|
||||
{
|
||||
var withState = new AppInstallSummary { AppName = "App", TerminalState = "Skipped" };
|
||||
var entity = TableStorageService.BuildAppInstallSummaryEntity(withState, "rk");
|
||||
Assert.Equal("Skipped", entity.GetString("TerminalState"));
|
||||
|
||||
var withoutState = new AppInstallSummary { AppName = "App" };
|
||||
var empty = TableStorageService.BuildAppInstallSummaryEntity(withoutState, "rk");
|
||||
// Column absent → Merge-mode preserves a prior batch's terminal state.
|
||||
Assert.False(empty.ContainsKey("TerminalState"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Apps;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the terminal-only failure-rate convention on the Apps dashboard aggregations
|
||||
/// (<see cref="AppsAnalyticsHelper"/>): every failureRate is Failed / (Failed + Succeeded) —
|
||||
/// finished installs only. "InProgress" rows (still installing, or orphaned by a session that
|
||||
/// died mid-install) sit in the install counts but never dilute a rate, matching the enrollment
|
||||
/// success-rate convention. Responses are anonymous projections, so tests assert on the
|
||||
/// JSON wire shape like the sibling suites.
|
||||
/// </summary>
|
||||
public class AppsAnalyticsFailureRateTests
|
||||
{
|
||||
private const string TenantId = "00000000-0000-0000-0000-000000000fa1";
|
||||
|
||||
private static AppInstallSummary A(
|
||||
string status,
|
||||
string appName = "Contoso App",
|
||||
DateTime? startedAt = null,
|
||||
string sessionId = "s-1",
|
||||
string appVersion = "")
|
||||
=> new()
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = sessionId,
|
||||
AppName = appName,
|
||||
AppVersion = appVersion,
|
||||
Status = status,
|
||||
StartedAt = startedAt ?? DateTime.UtcNow.AddHours(-2),
|
||||
DurationSeconds = status == "Succeeded" ? 30 : 0,
|
||||
};
|
||||
|
||||
// ── /apps/list ──────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void List_FailureRate_IsOverFinishedInstalls_InProgressExcluded()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
A("Failed"),
|
||||
A("Succeeded"),
|
||||
A("InProgress"),
|
||||
A("InProgress"),
|
||||
};
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var app = root.GetProperty("apps")[0];
|
||||
Assert.Equal(4, app.GetProperty("totalInstalls").GetInt32());
|
||||
// 1 / (1 + 1) = 50% (over finished), NOT 1 / 4 = 25% (over all installs).
|
||||
Assert.Equal(50d, app.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void List_FailureRate_IsZero_WhenNothingFinishedYet()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary> { A("InProgress"), A("InProgress") };
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
Assert.Equal(0d, root.GetProperty("apps")[0].GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void List_Trend_UsesFinishedRates_AndGatesOnFinishedSamples()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var firstHalf = now.AddDays(-20); // days: 30 → midpoint at now - 15d
|
||||
var secondHalf = now.AddDays(-5);
|
||||
|
||||
var summaries = new List<AppInstallSummary>();
|
||||
// First half: 1 failed / 5 finished = 20%.
|
||||
summaries.Add(A("Failed", startedAt: firstHalf));
|
||||
for (var i = 0; i < 4; i++) summaries.Add(A("Succeeded", startedAt: firstHalf));
|
||||
// Second half: 3 failed / 5 finished = 60% — plus 10 in-flight installs that
|
||||
// previously dragged the half's rate down to 20% and masked the regression.
|
||||
for (var i = 0; i < 3; i++) summaries.Add(A("Failed", startedAt: secondHalf));
|
||||
for (var i = 0; i < 2; i++) summaries.Add(A("Succeeded", startedAt: secondHalf));
|
||||
for (var i = 0; i < 10; i++) summaries.Add(A("InProgress", startedAt: secondHalf));
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var app = root.GetProperty("apps")[0];
|
||||
Assert.Equal("worsening", app.GetProperty("trend").GetString());
|
||||
Assert.Equal(40d, app.GetProperty("trendDelta").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void List_Trend_StaysStable_WhenAHalfLacksFiveFinishedInstalls()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var summaries = new List<AppInstallSummary>();
|
||||
// First half: 5 finished. Second half: 4 finished + many in-flight — the old
|
||||
// install-count gate would have passed; the finished-sample gate must not.
|
||||
for (var i = 0; i < 5; i++) summaries.Add(A("Succeeded", startedAt: now.AddDays(-20)));
|
||||
for (var i = 0; i < 4; i++) summaries.Add(A("Failed", startedAt: now.AddDays(-5)));
|
||||
for (var i = 0; i < 10; i++) summaries.Add(A("InProgress", startedAt: now.AddDays(-5)));
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var app = root.GetProperty("apps")[0];
|
||||
Assert.Equal("stable", app.GetProperty("trend").GetString());
|
||||
Assert.Equal(JsonValueKind.Null, app.GetProperty("trendDelta").ValueKind);
|
||||
}
|
||||
|
||||
// ── /apps/{appName}/analytics ───────────────────────────────────────────
|
||||
|
||||
private static async Task<JsonElement> BuildAnalyticsAsync(
|
||||
List<AppInstallSummary> summaries, ISessionRepository? sessionRepo = null)
|
||||
{
|
||||
if (sessionRepo == null)
|
||||
{
|
||||
var mock = new Mock<ISessionRepository>();
|
||||
mock.Setup(r => r.GetSessionAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ReturnsAsync((SessionSummary?)null);
|
||||
sessionRepo = mock.Object;
|
||||
}
|
||||
var result = await AppsAnalyticsHelper.BuildAnalyticsResponseAsync(
|
||||
summaries, sessionRepo, "Contoso App", days: 30);
|
||||
return JsonSerializer.SerializeToElement(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Analytics_SummaryVersionAndTimeSeriesRates_AreOverFinishedInstalls()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
A("Failed", appVersion: "1.0"),
|
||||
A("Succeeded", appVersion: "1.0"),
|
||||
A("InProgress", appVersion: "1.0"),
|
||||
A("InProgress", appVersion: "1.0"),
|
||||
};
|
||||
|
||||
var root = await BuildAnalyticsAsync(summaries);
|
||||
|
||||
// Summary: 1 / (1 + 1) = 50%, NOT 1 / 4 = 25%.
|
||||
var summary = root.GetProperty("summary");
|
||||
Assert.Equal(4, summary.GetProperty("totalInstalls").GetInt32());
|
||||
Assert.Equal(50d, summary.GetProperty("failureRate").GetDouble());
|
||||
|
||||
var version = root.GetProperty("versionBreakdown")[0];
|
||||
Assert.Equal(4, version.GetProperty("installs").GetInt32());
|
||||
Assert.Equal(50d, version.GetProperty("failureRate").GetDouble());
|
||||
|
||||
// All four installs land in today's bucket → same terminal-only rate there.
|
||||
var buckets = root.GetProperty("timeSeries").EnumerateArray().ToList();
|
||||
var todayBucket = buckets.Single(b => b.GetProperty("installs").GetInt32() > 0);
|
||||
Assert.Equal(50d, todayBucket.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
// ── PR0 (2026-07-26): skip / unmeasured classification ──────────────────
|
||||
|
||||
[Fact]
|
||||
public void List_Skips_LeaveRateAndDurations_AndAreReportedSeparately()
|
||||
{
|
||||
var skip = A("Succeeded");
|
||||
skip.TerminalState = "Skipped";
|
||||
skip.DurationSeconds = 0;
|
||||
var unmeasured = A("Succeeded");
|
||||
unmeasured.TerminalState = "Installed";
|
||||
unmeasured.DurationSeconds = 0;
|
||||
|
||||
var summaries = new List<AppInstallSummary> { A("Failed"), A("Succeeded"), skip, unmeasured };
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
var app = root.GetProperty("apps")[0];
|
||||
|
||||
Assert.Equal(4, app.GetProperty("totalInstalls").GetInt32());
|
||||
Assert.Equal(2, app.GetProperty("succeeded").GetInt32()); // real install + unmeasured
|
||||
Assert.Equal(1, app.GetProperty("skipped").GetInt32());
|
||||
Assert.Equal(1, app.GetProperty("unmeasured").GetInt32());
|
||||
// 1 / (1 + 2) — the skip is not an attempt.
|
||||
Assert.Equal(33.3d, app.GetProperty("failureRate").GetDouble());
|
||||
// avg over the single measured row (30s) — the zero must not drag it to 15.
|
||||
Assert.Equal(30d, app.GetProperty("avgDurationSeconds").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Analytics_Summary_CarriesSkippedAndUnmeasured_AndP95OverMeasuredOnly()
|
||||
{
|
||||
var skip = A("Succeeded");
|
||||
skip.TerminalState = "Skipped";
|
||||
skip.DurationSeconds = 0;
|
||||
var unmeasured = A("Succeeded");
|
||||
unmeasured.TerminalState = "Installed";
|
||||
unmeasured.DurationSeconds = 0;
|
||||
var measured = A("Succeeded");
|
||||
measured.TerminalState = "Installed";
|
||||
measured.DurationSeconds = 100;
|
||||
|
||||
var root = await BuildAnalyticsAsync(new List<AppInstallSummary> { skip, unmeasured, measured });
|
||||
var summary = root.GetProperty("summary");
|
||||
|
||||
Assert.Equal(2, summary.GetProperty("succeeded").GetInt32());
|
||||
Assert.Equal(1, summary.GetProperty("skipped").GetInt32());
|
||||
Assert.Equal(1, summary.GetProperty("unmeasured").GetInt32());
|
||||
Assert.Equal(100d, summary.GetProperty("avgDurationSeconds").GetDouble());
|
||||
Assert.Equal(100, summary.GetProperty("p95DurationSeconds").GetInt32());
|
||||
Assert.Equal(0d, summary.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Analytics_DeviceModelBreakdown_RatesAndSampleFloor_UseFinishedInstalls()
|
||||
{
|
||||
var sessionRepo = new Mock<ISessionRepository>();
|
||||
sessionRepo.Setup(r => r.GetSessionAsync(TenantId, It.IsAny<string>()))
|
||||
.ReturnsAsync((string _, string sessionId) => new SessionSummary
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = sessionId,
|
||||
Manufacturer = "Contoso",
|
||||
Model = sessionId.StartsWith("s-a") ? "ModelA" : "ModelB",
|
||||
});
|
||||
|
||||
var summaries = new List<AppInstallSummary>();
|
||||
// ModelA: 5 finished (1 failed → 20%) + 3 in-flight → ranked, rate undiluted.
|
||||
summaries.Add(A("Failed", sessionId: "s-a-0"));
|
||||
for (var i = 1; i <= 4; i++) summaries.Add(A("Succeeded", sessionId: $"s-a-{i}"));
|
||||
for (var i = 5; i <= 7; i++) summaries.Add(A("InProgress", sessionId: $"s-a-{i}"));
|
||||
// ModelB: 4 finished + 6 in-flight — 10 installs, but below the 5-finished floor.
|
||||
for (var i = 0; i <= 3; i++) summaries.Add(A("Failed", sessionId: $"s-b-{i}"));
|
||||
for (var i = 4; i <= 9; i++) summaries.Add(A("InProgress", sessionId: $"s-b-{i}"));
|
||||
|
||||
var root = await BuildAnalyticsAsync(summaries, sessionRepo.Object);
|
||||
|
||||
var breakdown = root.GetProperty("deviceModelBreakdown");
|
||||
var model = Assert.Single(breakdown.EnumerateArray());
|
||||
Assert.Equal("ModelA", model.GetProperty("model").GetString());
|
||||
Assert.Equal(8, model.GetProperty("installs").GetInt32());
|
||||
Assert.Equal(20d, model.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,14 @@ public class AppsProjectionEquivalenceTests
|
||||
["AppType"] = "Win32",
|
||||
["AppVersion"] = "2.1.0",
|
||||
["Status"] = "Failed",
|
||||
// PR0/PR1 classification columns. The fixture MUST populate every column a Build*
|
||||
// method reads: a column absent here passes the equivalence assert trivially (both
|
||||
// sides map to default) — exactly how the missing TerminalState projection slipped
|
||||
// through and zeroed totalSkipped in production (2026-07-27).
|
||||
["TerminalState"] = "Skipped",
|
||||
["AppId"] = "9e8d7c6b-5a49-4832-9182-736451234567",
|
||||
["EspBlocking"] = true,
|
||||
["AppIdCollision"] = true,
|
||||
["StartedAt"] = new DateTimeOffset(DateTime.UtcNow.AddHours(-6)),
|
||||
["CompletedAt"] = new DateTimeOffset(DateTime.UtcNow.AddHours(-5)),
|
||||
["DurationSeconds"] = 340,
|
||||
@@ -83,6 +91,9 @@ public class AppsProjectionEquivalenceTests
|
||||
|
||||
Assert.Equal(fromFull.AppName, fromProjected.AppName);
|
||||
Assert.Equal(fromFull.Status, fromProjected.Status);
|
||||
// IsSkipTerminalState / skip classification input — see the fixture comment.
|
||||
Assert.Equal(fromFull.TerminalState, fromProjected.TerminalState);
|
||||
Assert.Equal(fromFull.AppIdCollision, fromProjected.AppIdCollision);
|
||||
Assert.Equal(fromFull.StartedAt, fromProjected.StartedAt);
|
||||
Assert.Equal(fromFull.DurationSeconds, fromProjected.DurationSeconds);
|
||||
Assert.Equal(fromFull.DownloadBytes, fromProjected.DownloadBytes);
|
||||
@@ -122,6 +133,10 @@ public class AppsProjectionEquivalenceTests
|
||||
Assert.Equal(fromFull.FailureMessage, fromProjected.FailureMessage);
|
||||
Assert.Equal(fromFull.ExitCode, fromProjected.ExitCode);
|
||||
Assert.Equal(fromFull.DetectionResult, fromProjected.DetectionResult);
|
||||
Assert.Equal(fromFull.TerminalState, fromProjected.TerminalState);
|
||||
Assert.Equal(fromFull.AppId, fromProjected.AppId);
|
||||
Assert.Equal(fromFull.EspBlocking, fromProjected.EspBlocking);
|
||||
Assert.Equal(fromFull.AppIdCollision, fromProjected.AppIdCollision);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
using AutopilotMonitor.Functions.Functions.Infrastructure;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the pure helpers of <see cref="AzureMonitorAlertWebhookFunction"/> — the anonymous,
|
||||
/// secret-gated bridge that turns Azure Monitor Common Alert Schema notifications into ops
|
||||
/// events. Load-bearing bits: the secret gate is fail-closed (an unset app setting rejects
|
||||
/// everything), severity mapping can never escalate on schema drift (unknown → Info), and
|
||||
/// parsing is defensive (only the alert rule name is mandatory).
|
||||
/// </summary>
|
||||
public class AzureMonitorAlertWebhookFunctionTests
|
||||
{
|
||||
// ── SecretMatches ──────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void SecretMatches_accepts_equal_secrets()
|
||||
{
|
||||
Assert.True(AzureMonitorAlertWebhookFunction.SecretMatches("s3cret-value", "s3cret-value"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("wrong", "right")]
|
||||
[InlineData("right-but-longer", "right")]
|
||||
[InlineData("", "configured")]
|
||||
[InlineData(null, "configured")]
|
||||
public void SecretMatches_rejects_mismatch_or_missing_provided(string? provided, string configured)
|
||||
{
|
||||
Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches(provided, configured));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
public void SecretMatches_fails_closed_when_not_configured(string? configured)
|
||||
{
|
||||
// While the app setting is unset, NOTHING matches — not even an empty provided value.
|
||||
Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches("anything", configured));
|
||||
Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches("", configured));
|
||||
Assert.False(AzureMonitorAlertWebhookFunction.SecretMatches(null, configured));
|
||||
}
|
||||
|
||||
// ── MapSeverity ────────────────────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("Sev0", OpsEventSeverity.Critical)]
|
||||
[InlineData("Sev1", OpsEventSeverity.Error)]
|
||||
[InlineData("Sev2", OpsEventSeverity.Warning)]
|
||||
[InlineData("Sev3", OpsEventSeverity.Info)]
|
||||
[InlineData("Sev4", OpsEventSeverity.Info)]
|
||||
public void MapSeverity_maps_azure_scale_to_ops_scale(string azure, string expected)
|
||||
{
|
||||
Assert.Equal(expected, AzureMonitorAlertWebhookFunction.MapSeverity(azure));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData("Sev5")]
|
||||
[InlineData("Critical")] // someone posting the ops scale directly must not pass through
|
||||
public void MapSeverity_falls_back_to_info_on_unknown_values(string? azure)
|
||||
{
|
||||
Assert.Equal(OpsEventSeverity.Info, AzureMonitorAlertWebhookFunction.MapSeverity(azure));
|
||||
}
|
||||
|
||||
// ── TryParse ───────────────────────────────────────────────────────────
|
||||
|
||||
private const string FullPayload = """
|
||||
{
|
||||
"schemaId": "azureMonitorCommonAlertSchema",
|
||||
"data": {
|
||||
"essentials": {
|
||||
"alertId": "/subscriptions/xxx/providers/Microsoft.AlertsManagement/alerts/abc",
|
||||
"alertRule": "Portal RSC prefetch tail",
|
||||
"severity": "Sev2",
|
||||
"signalType": "Log",
|
||||
"monitorCondition": "Fired",
|
||||
"monitoringService": "Log Alerts V2",
|
||||
"alertTargetIDs": [
|
||||
"/subscriptions/xxx/resourcegroups/rg-autopilotmonitor-prd-gwc/providers/microsoft.insights/components/autopilotmonitorweb"
|
||||
],
|
||||
"description": "RSC prefetches exceeding 10s",
|
||||
"firedDateTime": "2026-07-27T15:30:00.000Z"
|
||||
},
|
||||
"alertContext": {
|
||||
"condition": {
|
||||
"windowSize": "PT1H",
|
||||
"allOf": [
|
||||
{
|
||||
"searchQuery": "dependencies | where name contains '_rsc='",
|
||||
"metricValue": 18.0,
|
||||
"operator": "GreaterThan",
|
||||
"threshold": "0"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
[Fact]
|
||||
public void TryParse_extracts_essentials_and_metric_value()
|
||||
{
|
||||
var alert = AzureMonitorAlertWebhookFunction.TryParse(FullPayload);
|
||||
|
||||
Assert.NotNull(alert);
|
||||
Assert.Equal("Portal RSC prefetch tail", alert!.AlertRule);
|
||||
Assert.Equal("Sev2", alert.Severity);
|
||||
Assert.Equal("Fired", alert.MonitorCondition);
|
||||
Assert.Equal("RSC prefetches exceeding 10s", alert.Description);
|
||||
Assert.Equal("Log Alerts V2", alert.MonitoringService);
|
||||
// No targetResourceName in the payload → resource name segment of the first target ID.
|
||||
Assert.Equal("autopilotmonitorweb", alert.TargetResource);
|
||||
Assert.Equal(18.0, alert.MetricValue);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryParse_prefers_explicit_target_resource_name()
|
||||
{
|
||||
const string payload = """
|
||||
{"data":{"essentials":{"alertRule":"r","targetResourceName":"explicit-name",
|
||||
"alertTargetIDs":["/sub/x/other-name"]}}}
|
||||
""";
|
||||
var alert = AzureMonitorAlertWebhookFunction.TryParse(payload);
|
||||
Assert.Equal("explicit-name", alert!.TargetResource);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryParse_minimal_payload_needs_only_the_alert_rule()
|
||||
{
|
||||
var alert = AzureMonitorAlertWebhookFunction.TryParse(
|
||||
"""{"data":{"essentials":{"alertRule":"minimal"}}}""");
|
||||
|
||||
Assert.NotNull(alert);
|
||||
Assert.Equal("minimal", alert!.AlertRule);
|
||||
Assert.Null(alert.Severity);
|
||||
Assert.Equal("Fired", alert.MonitorCondition); // default when absent
|
||||
Assert.Null(alert.Description);
|
||||
Assert.Null(alert.MetricValue);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("not json at all")]
|
||||
[InlineData("{}")] // no data
|
||||
[InlineData("""{"data":{}}""")] // no essentials
|
||||
[InlineData("""{"data":{"essentials":{}}}""")] // no alertRule
|
||||
[InlineData("""{"data":{"essentials":{"alertRule":""}}}""")] // empty alertRule
|
||||
[InlineData("""{"data":{"essentials":{"alertRule":123}}}""")] // wrong type
|
||||
public void TryParse_rejects_unusable_payloads(string? body)
|
||||
{
|
||||
Assert.Null(AzureMonitorAlertWebhookFunction.TryParse(body!));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryParse_tolerates_malformed_alert_context()
|
||||
{
|
||||
// alertContext with unexpected shapes must not break essentials extraction.
|
||||
const string payload = """
|
||||
{"data":{"essentials":{"alertRule":"r","severity":"Sev0"},
|
||||
"alertContext":{"condition":{"allOf":"not-an-array"}}}}
|
||||
""";
|
||||
var alert = AzureMonitorAlertWebhookFunction.TryParse(payload);
|
||||
Assert.NotNull(alert);
|
||||
Assert.Equal("Sev0", alert!.Severity);
|
||||
Assert.Null(alert.MetricValue);
|
||||
}
|
||||
}
|
||||
@@ -183,4 +183,13 @@ public class ConfigRedactionTests
|
||||
public void CanViewSecrets_PureDelegatedReader_False()
|
||||
=> Assert.False(GetTenantConfigurationFunction.CanViewSecrets(
|
||||
new RequestContext { IsDelegatedReader = true, TenantId = TenantA, TargetTenantId = TenantB }));
|
||||
|
||||
[Theory]
|
||||
[InlineData(Constants.TenantRoles.Operator)]
|
||||
[InlineData(Constants.TenantRoles.Viewer)]
|
||||
public void CanViewSecrets_OwnTenantOperatorOrViewer_False(string role)
|
||||
// Member-tier read-only Settings view (config GET is MemberRead): an own-tenant
|
||||
// Operator/Viewer arrives with IsTenantAdmin=false ⇒ always the redacted copy.
|
||||
=> Assert.False(GetTenantConfigurationFunction.CanViewSecrets(
|
||||
new RequestContext { UserRole = role, TenantId = TenantA, TargetTenantId = TenantA }));
|
||||
}
|
||||
|
||||
@@ -324,4 +324,42 @@ public class CrossTenantAccessTests
|
||||
Assert.NotNull(entry);
|
||||
Assert.Equal(TenantScoping.QueryParam, entry.TenantScoping);
|
||||
}
|
||||
|
||||
// ── Cross-tenant rule WRITES: global/rules/{gather,analyze}/{ruleId} ──
|
||||
// Bug (2026-07-24): a Global Admin editing a foreign tenant's rule via the JWT-scoped
|
||||
// rules/{type}/{ruleId} route silently upserted into the GA's OWN tenant (200 → false "saved");
|
||||
// the reload re-read the foreign tenant and showed the unchanged rule. Fix: dedicated cross-tenant
|
||||
// write routes. They MUST be GlobalAdminOnly (a write tier → no delegated-read rescue, so a
|
||||
// tenant-admin / delegated caller can never cross into another tenant's rules) + QueryParam (the
|
||||
// tenant is named by ?tenantId=, not the {ruleId} segment). A downgrade to a tenant tier, or a slip
|
||||
// to TenantScoping.None (which would ignore ?tenantId= and re-introduce the home-tenant misroute),
|
||||
// is caught here.
|
||||
|
||||
[Theory]
|
||||
[InlineData("PUT", "global/rules/gather/{ruleId}")]
|
||||
[InlineData("DELETE", "global/rules/gather/{ruleId}")]
|
||||
[InlineData("PUT", "global/rules/analyze/{ruleId}")]
|
||||
[InlineData("DELETE", "global/rules/analyze/{ruleId}")]
|
||||
public void GlobalRuleWriteRoutes_AreGlobalAdminOnly_QueryParam(string httpMethod, string routeTemplate)
|
||||
{
|
||||
var entry = EndpointAccessPolicyCatalog.Entries
|
||||
.FirstOrDefault(e => e.HttpMethod == httpMethod.ToUpperInvariant() && e.RouteTemplate == routeTemplate);
|
||||
|
||||
Assert.NotNull(entry);
|
||||
Assert.Equal(EndpointPolicy.GlobalAdminOnly, entry!.Policy);
|
||||
Assert.Equal(TenantScoping.QueryParam, entry.TenantScoping);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("PUT", "/api/global/rules/gather/rule-1")]
|
||||
[InlineData("DELETE", "/api/global/rules/gather/rule-1")]
|
||||
[InlineData("PUT", "/api/global/rules/analyze/rule-1")]
|
||||
[InlineData("DELETE", "/api/global/rules/analyze/rule-1")]
|
||||
public void GlobalRuleWrite_CrossTenant_GlobalAdmin_IsAllowed(string httpMethod, string path)
|
||||
{
|
||||
var (isBlocked, targetTenantId) = SimulateQueryParamCheck(httpMethod, path, TenantA, hasGlobalScope: true, queryTenantId: TenantB);
|
||||
|
||||
Assert.False(isBlocked, $"Global Admin should reach the target tenant on the cross-tenant write route: {httpMethod} {path}");
|
||||
Assert.Equal(TenantB, targetTenantId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,8 +75,9 @@ public class DeletionManifestBuilderTests
|
||||
Assert.Equal(2, c["classifierVerdictsByIdLevel"]);
|
||||
Assert.Equal(11, c["signalsByKind"]);
|
||||
|
||||
// 15 cascade tables + tombstone = 16 steps; no inventory steps when no contributions row.
|
||||
Assert.Equal(16, manifest.Steps.Count);
|
||||
// 16 cascade tables (incl. SessionTimeBreakdowns, F1 PR2) + tombstone = 17 steps;
|
||||
// no inventory steps when no contributions row.
|
||||
Assert.Equal(17, manifest.Steps.Count);
|
||||
Assert.Equal(DeletionStepClass.Final, manifest.Steps.Last().Class);
|
||||
}
|
||||
|
||||
@@ -102,7 +103,8 @@ public class DeletionManifestBuilderTests
|
||||
.WithSessionsByStage(1, propValue: "stage_marker")
|
||||
.WithDeadEndsByReason(1, propValue: "deadend_marker")
|
||||
.WithClassifierVerdictsByIdLevel(1, propValue: "classifier_marker")
|
||||
.WithSignalsByKind(1, propValue: "signalkind_marker"));
|
||||
.WithSignalsByKind(1, propValue: "signalkind_marker")
|
||||
.WithSessionTimeBreakdown(propValue: "breakdown_marker"));
|
||||
|
||||
var manifest = await NewBuilder(reader).BuildAsync(
|
||||
TenantId, SessionId, "admin_delete",
|
||||
@@ -145,8 +147,8 @@ public class DeletionManifestBuilderTests
|
||||
new DeletionRetentionContext { TenantRetentionDays = 90 });
|
||||
|
||||
var nonTombstoneSteps = manifest.Steps.Where(s => s.Class != DeletionStepClass.Final).ToList();
|
||||
// Without a contributions row, neither inventory step is emitted → exactly 15 cascade-table steps.
|
||||
Assert.Equal(15, nonTombstoneSteps.Count);
|
||||
// Without a contributions row, neither inventory step is emitted → exactly 16 cascade-table steps.
|
||||
Assert.Equal(16, nonTombstoneSteps.Count);
|
||||
foreach (var step in nonTombstoneSteps)
|
||||
{
|
||||
Assert.Equal(0, step.RowCount);
|
||||
@@ -175,8 +177,8 @@ public class DeletionManifestBuilderTests
|
||||
new DeletionActor { Type = "admin", Actor = "alice@example.com" },
|
||||
new DeletionRetentionContext { TenantRetentionDays = 90 });
|
||||
|
||||
// Expect: 15 cascade + 2 inventory + 1 tombstone = 18 steps.
|
||||
Assert.Equal(18, manifest.Steps.Count);
|
||||
// Expect: 16 cascade + 2 inventory + 1 tombstone = 19 steps.
|
||||
Assert.Equal(19, manifest.Steps.Count);
|
||||
|
||||
var aggregate = manifest.Steps.Single(s => s.Class == DeletionStepClass.Aggregate);
|
||||
Assert.Equal(DeletionStepNames.SoftwareInventoryDecrement, aggregate.Step);
|
||||
@@ -640,6 +642,7 @@ public class DeletionManifestBuilderTests
|
||||
private TableEntity? _vulnerabilityReport;
|
||||
private TableEntity? _deviceSnapshot;
|
||||
private TableEntity? _eventSessionIndex;
|
||||
private TableEntity? _sessionTimeBreakdown;
|
||||
private TableEntity? _sessionInventoryContributions;
|
||||
private string _sessionsIndexRowKey = "INDEX_" + SessionId;
|
||||
|
||||
@@ -719,6 +722,16 @@ public class DeletionManifestBuilderTests
|
||||
return this;
|
||||
}
|
||||
|
||||
public ReaderSeed WithSessionTimeBreakdown(string propValue = "tab")
|
||||
{
|
||||
_sessionTimeBreakdown = new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["Marker"] = propValue,
|
||||
};
|
||||
_sessionTimeBreakdown.ETag = new ETag("0xTAB");
|
||||
return this;
|
||||
}
|
||||
|
||||
public ReaderSeed WithEventTypeIndex(int n, string propValue = "eti")
|
||||
{
|
||||
// PK = {tenantId}_{eventType}, RK ends with _{sessionId}.
|
||||
@@ -796,6 +809,8 @@ public class DeletionManifestBuilderTests
|
||||
.ReturnsAsync(_deviceSnapshot);
|
||||
reader.Setup(r => r.GetEntityOrNullAsync(Constants.TableNames.EventSessionIndex, TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(_eventSessionIndex);
|
||||
reader.Setup(r => r.GetEntityOrNullAsync(Constants.TableNames.SessionTimeBreakdowns, TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(_sessionTimeBreakdown);
|
||||
reader.Setup(r => r.GetEntityOrNullAsync(Constants.TableNames.SessionInventoryContributions, TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(_sessionInventoryContributions);
|
||||
|
||||
|
||||
@@ -0,0 +1,809 @@
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// F2 PR4 (insights spec §F2): device-key normalization (junk serials), terminal-only chain
|
||||
/// maintenance (upsert, cap, deletion-ref pruning), journey grouping (first-success end,
|
||||
/// 30-day gap, Incomplete attempts, WG-Pending stays open), the pure daily FTR aggregation
|
||||
/// core (<see cref="MaintenanceService.BuildDeviceJourneyAggregates"/>) and the persistence
|
||||
/// round-trips (table-storage-serialization rule — every property must survive Store→Map).
|
||||
/// </summary>
|
||||
public class DeviceJourneyAndFtrTests
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 7, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
private const string TenantA = "aaaaaaaa-0000-0000-0000-000000000001";
|
||||
private const string TenantB = "bbbbbbbb-0000-0000-0000-000000000002";
|
||||
|
||||
private static SessionSummary Session(
|
||||
string sessionId,
|
||||
SessionStatus status = SessionStatus.Succeeded,
|
||||
DateTime? startedAt = null,
|
||||
DateTime? completedAt = null,
|
||||
int? durationSeconds = 1800,
|
||||
string? serial = "PF4X1ABC",
|
||||
string tenantId = TenantA,
|
||||
bool wg = false,
|
||||
string enrollmentType = "v1",
|
||||
string? adminMarkedAction = null)
|
||||
=> new()
|
||||
{
|
||||
TenantId = tenantId,
|
||||
SessionId = sessionId,
|
||||
SerialNumber = serial!,
|
||||
Status = status,
|
||||
StartedAt = startedAt ?? T0,
|
||||
CompletedAt = completedAt ?? (startedAt ?? T0).AddSeconds(durationSeconds ?? 0),
|
||||
DurationSeconds = durationSeconds,
|
||||
IsPreProvisioned = wg,
|
||||
EnrollmentType = enrollmentType,
|
||||
AdminMarkedAction = adminMarkedAction,
|
||||
Manufacturer = "Contoso",
|
||||
Model = "Laptop 5",
|
||||
};
|
||||
|
||||
private static DeviceSessionRef Ref(
|
||||
string sessionId, SessionStatus status, DateTime startedAt, DateTime? completedAt = null)
|
||||
=> new()
|
||||
{
|
||||
SessionId = sessionId,
|
||||
Status = status.ToString(),
|
||||
StartedAt = startedAt,
|
||||
CompletedAt = completedAt ?? startedAt.AddMinutes(30),
|
||||
};
|
||||
|
||||
// ── serial normalization / junk exclusion ───────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("abc")] // shorter than 4 chars
|
||||
[InlineData("0")]
|
||||
[InlineData("None")]
|
||||
[InlineData("INVALID")]
|
||||
[InlineData("Unknown")] // the agent's WMI-failure sentinel (audit Q5)
|
||||
[InlineData("unknown")]
|
||||
[InlineData("System Serial Number")]
|
||||
[InlineData("TO BE FILLED BY O.E.M.")]
|
||||
[InlineData("Default String")]
|
||||
[InlineData(" Default string ")] // junk match happens after trim + case-fold
|
||||
public void NormalizeSerial_JunkAndPlaceholders_YieldNull(string? raw)
|
||||
{
|
||||
Assert.Null(DeviceJourneyCalculator.NormalizeSerial(raw));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("PF4X1ABC", "pf4x1abc")]
|
||||
[InlineData(" PF4X1ABC ", "pf4x1abc")] // trim
|
||||
[InlineData("pf4x1abc", "pf4x1abc")]
|
||||
[InlineData("1234", "1234")] // exactly 4 chars is a valid identity
|
||||
public void NormalizeSerial_RealSerials_TrimAndCaseFold(string raw, string expected)
|
||||
{
|
||||
Assert.Equal(expected, DeviceJourneyCalculator.NormalizeSerial(raw));
|
||||
}
|
||||
|
||||
// ── terminal set / ref building ─────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData(SessionStatus.Succeeded, true)]
|
||||
[InlineData(SessionStatus.Failed, true)]
|
||||
[InlineData(SessionStatus.Incomplete, true)]
|
||||
[InlineData(SessionStatus.InProgress, false)]
|
||||
[InlineData(SessionStatus.Pending, false)] // WG part 1 sealed — an OPEN session
|
||||
[InlineData(SessionStatus.Stalled, false)]
|
||||
[InlineData(SessionStatus.AwaitingUser, false)]
|
||||
[InlineData(SessionStatus.Unknown, false)]
|
||||
public void IsTerminal_MatchesTheF2TerminalSet(SessionStatus status, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, DeviceJourneyCalculator.IsTerminal(status));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildSessionRef_MapsAllFields_DurationIsAuthoritativeDurationSeconds()
|
||||
{
|
||||
// CompletedAt deliberately diverges from StartedAt + DurationSeconds (25 % of terminal
|
||||
// sessions in production) — the ref must carry DurationSeconds verbatim.
|
||||
var session = Session("s-1", startedAt: T0, completedAt: T0.AddHours(5), durationSeconds: 1800,
|
||||
wg: true, enrollmentType: "v1", adminMarkedAction: "Succeeded");
|
||||
|
||||
var reference = DeviceJourneyCalculator.BuildSessionRef(session)!;
|
||||
|
||||
Assert.Equal("s-1", reference.SessionId);
|
||||
Assert.Equal(T0, reference.StartedAt);
|
||||
Assert.Equal(T0.AddHours(5), reference.CompletedAt);
|
||||
Assert.Equal("Succeeded", reference.Status);
|
||||
Assert.Equal("v1", reference.EnrollmentType);
|
||||
Assert.True(reference.IsPreProvisioned);
|
||||
Assert.Equal(1800, reference.DurationSeconds);
|
||||
Assert.True(reference.AdminMarked);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildSessionRef_WhiteGlovePending_YieldsNull_NeverAnAttempt()
|
||||
{
|
||||
var pending = Session("s-1", status: SessionStatus.Pending, wg: true, durationSeconds: null);
|
||||
Assert.Null(DeviceJourneyCalculator.BuildSessionRef(pending));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildSessionRef_Incomplete_KeepsNullDuration()
|
||||
{
|
||||
var incomplete = Session("s-1", status: SessionStatus.Incomplete, durationSeconds: null);
|
||||
var reference = DeviceJourneyCalculator.BuildSessionRef(incomplete)!;
|
||||
Assert.Equal("Incomplete", reference.Status);
|
||||
Assert.Null(reference.DurationSeconds);
|
||||
Assert.False(reference.AdminMarked);
|
||||
}
|
||||
|
||||
// ── chain maintenance ───────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void MergeChain_UpsertsBySessionId_ReclassificationReplacesTheEntry()
|
||||
{
|
||||
var existing = new List<DeviceSessionRef> { Ref("s-1", SessionStatus.Incomplete, T0) };
|
||||
var update = Ref("s-1", SessionStatus.Succeeded, T0);
|
||||
|
||||
var merged = DeviceJourneyCalculator.MergeChain(existing, new[] { update });
|
||||
|
||||
var entry = Assert.Single(merged);
|
||||
Assert.Equal("Succeeded", entry.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeChain_OrdersByStartedAt_TiebreakSessionId_AndCapsToMostRecent()
|
||||
{
|
||||
var refs = new List<DeviceSessionRef>();
|
||||
for (var i = 0; i < 25; i++)
|
||||
refs.Add(Ref($"s-{i:D2}", SessionStatus.Failed, T0.AddDays(i)));
|
||||
// Same StartedAt as s-24 — the ordinal sessionId tiebreak must keep the order stable.
|
||||
refs.Add(Ref("s-00-twin", SessionStatus.Failed, T0.AddDays(24)));
|
||||
|
||||
var merged = DeviceJourneyCalculator.MergeChain(null, refs);
|
||||
|
||||
Assert.Equal(20, merged.Count); // cap 20 most recent
|
||||
Assert.Equal("s-06", merged[0].SessionId); // s-00..s-05 dropped (oldest)
|
||||
Assert.Equal(T0.AddDays(24), merged[^1].StartedAt);
|
||||
Assert.Equal("s-24", merged[^1].SessionId); // "s-00-twin" < "s-24" ordinal → sorts first
|
||||
Assert.Equal("s-00-twin", merged[^2].SessionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MergeChain_DropsNonTerminalRefs_Defensively()
|
||||
{
|
||||
var corrupt = new List<DeviceSessionRef>
|
||||
{
|
||||
Ref("s-1", SessionStatus.Succeeded, T0),
|
||||
new() { SessionId = "s-2", Status = "Pending", StartedAt = T0.AddDays(1) },
|
||||
new() { SessionId = "s-3", Status = "garbage", StartedAt = T0.AddDays(2) },
|
||||
};
|
||||
|
||||
var merged = DeviceJourneyCalculator.MergeChain(corrupt, Array.Empty<DeviceSessionRef>());
|
||||
|
||||
Assert.Equal("s-1", Assert.Single(merged).SessionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveSessionRefs_DropsDeletedSessions_EmptyChainSignalsRowDeletion()
|
||||
{
|
||||
var chain = new List<DeviceSessionRef>
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0),
|
||||
Ref("s-2", SessionStatus.Succeeded, T0.AddDays(1)),
|
||||
};
|
||||
|
||||
var partial = DeviceJourneyCalculator.RemoveSessionRefs(
|
||||
chain, new HashSet<string>(StringComparer.Ordinal) { "s-1" });
|
||||
Assert.Equal("s-2", Assert.Single(partial).SessionId);
|
||||
|
||||
var empty = DeviceJourneyCalculator.RemoveSessionRefs(
|
||||
chain, new HashSet<string>(StringComparer.Ordinal) { "s-1", "s-2" });
|
||||
Assert.Empty(empty);
|
||||
}
|
||||
|
||||
// ── journey grouping ────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_SingleSuccess_IsOneCompletedFirstTimeRightJourney()
|
||||
{
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(
|
||||
new[] { Ref("s-1", SessionStatus.Succeeded, T0) });
|
||||
|
||||
var journey = Assert.Single(journeys);
|
||||
Assert.True(journey.Completed);
|
||||
Assert.Single(journey.Attempts);
|
||||
Assert.Equal("s-1", journey.CompletingRef!.SessionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_FailAndIncompleteAreAttempts_JourneyEndsWithFirstSuccess()
|
||||
{
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0),
|
||||
Ref("s-2", SessionStatus.Incomplete, T0.AddHours(2)),
|
||||
Ref("s-3", SessionStatus.Succeeded, T0.AddHours(4)),
|
||||
});
|
||||
|
||||
var journey = Assert.Single(journeys);
|
||||
Assert.True(journey.Completed);
|
||||
Assert.Equal(3, journey.Attempts.Count); // Incomplete is a non-successful attempt
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_SessionAfterSuccess_StartsANewJourney_Redeployment()
|
||||
{
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Succeeded, T0),
|
||||
Ref("s-2", SessionStatus.Failed, T0.AddDays(2)),
|
||||
});
|
||||
|
||||
Assert.Equal(2, journeys.Count);
|
||||
Assert.True(journeys[0].Completed);
|
||||
Assert.False(journeys[1].Completed); // open — no failed verdict, just not done yet
|
||||
Assert.Null(journeys[1].CompletingRef);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_FailuresOnly_JourneyStaysOpen_NeverCompleted()
|
||||
{
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0),
|
||||
Ref("s-2", SessionStatus.Failed, T0.AddDays(1)),
|
||||
});
|
||||
|
||||
var journey = Assert.Single(journeys);
|
||||
Assert.False(journey.Completed);
|
||||
Assert.Equal(2, journey.Attempts.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_GapOver30Days_StartsANewJourney_AbandonedStaysUncompleted()
|
||||
{
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0, completedAt: T0.AddHours(1)),
|
||||
Ref("s-2", SessionStatus.Failed, T0.AddHours(1).AddDays(31)),
|
||||
Ref("s-3", SessionStatus.Succeeded, T0.AddHours(1).AddDays(31).AddHours(3)),
|
||||
});
|
||||
|
||||
Assert.Equal(2, journeys.Count);
|
||||
Assert.False(journeys[0].Completed); // shelved/abandoned — never counts for FTR
|
||||
Assert.Single(journeys[0].Attempts);
|
||||
Assert.True(journeys[1].Completed);
|
||||
Assert.Equal(2, journeys[1].Attempts.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_GapBoundary_Exactly30DaysStaysOneJourney_StrictlyMoreSplits()
|
||||
{
|
||||
var prevEnd = T0.AddHours(1);
|
||||
|
||||
var exactly30 = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0, completedAt: prevEnd),
|
||||
Ref("s-2", SessionStatus.Failed, prevEnd.AddDays(30)),
|
||||
});
|
||||
Assert.Single(exactly30); // spec: "> 30 days" — the boundary itself does not split
|
||||
|
||||
var over30 = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0, completedAt: prevEnd),
|
||||
Ref("s-2", SessionStatus.Failed, prevEnd.AddDays(30).AddHours(1)),
|
||||
});
|
||||
Assert.Equal(2, over30.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_GapMeasuredFromCompletedAt_NotStartedAt()
|
||||
{
|
||||
// Previous attempt STARTED 40 days before the next one but went terminal only 10 days
|
||||
// before it — the gap is 10 days, one journey. StartedAt-based measuring would split.
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0.AddDays(-40), completedAt: T0.AddDays(-10)),
|
||||
Ref("s-2", SessionStatus.Succeeded, T0),
|
||||
});
|
||||
|
||||
var journey = Assert.Single(journeys);
|
||||
Assert.Equal(2, journey.Attempts.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GroupJourneys_WhiteGloveIsOneSessionRow_OneAttempt_NoStitching()
|
||||
{
|
||||
// WG part 1 + part 2 share one session row (second-sweep finding): a completed WG
|
||||
// enrollment is exactly one ref and one first-time-right attempt.
|
||||
var wg = Session("s-1", wg: true, durationSeconds: 5400, completedAt: T0.AddDays(6));
|
||||
var reference = DeviceJourneyCalculator.BuildSessionRef(wg)!;
|
||||
|
||||
var journeys = DeviceJourneyCalculator.GroupJourneys(new[] { reference });
|
||||
|
||||
var journey = Assert.Single(journeys);
|
||||
Assert.True(journey.Completed);
|
||||
Assert.Single(journey.Attempts);
|
||||
Assert.Equal(5400, journey.Attempts[0].DurationSeconds); // pause-free combined duration, verbatim
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Derive_CurrentJourneyAttempts_IsTheLastJourneys_OpenOrCompleted()
|
||||
{
|
||||
var (journeyCount, currentAttempts) = DeviceJourneyCalculator.Derive(new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Succeeded, T0),
|
||||
Ref("s-2", SessionStatus.Failed, T0.AddDays(1)),
|
||||
Ref("s-3", SessionStatus.Failed, T0.AddDays(2)),
|
||||
});
|
||||
|
||||
Assert.Equal(2, journeyCount);
|
||||
Assert.Equal(2, currentAttempts); // the open redeployment journey has 2 attempts so far
|
||||
|
||||
Assert.Equal((0, 0), DeviceJourneyCalculator.Derive(Array.Empty<DeviceSessionRef>()));
|
||||
}
|
||||
|
||||
// ── history row assembly (display precedence) ───────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void BuildDeviceHistoryRow_DisplayFields_FollowTheNewestChainEntry()
|
||||
{
|
||||
var newest = Session("s-2", serial: " PF4X1ABC ", startedAt: T0.AddDays(1));
|
||||
var chain = new List<DeviceSessionRef>
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0),
|
||||
Ref("s-2", SessionStatus.Succeeded, T0.AddDays(1)),
|
||||
};
|
||||
|
||||
var row = TableStorageService.BuildDeviceHistoryRow(TenantA, "pf4x1abc", chain, existing: null, newest);
|
||||
|
||||
Assert.Equal("PF4X1ABC", row.SerialNumber); // trimmed original casing
|
||||
Assert.Equal("Contoso", row.Manufacturer);
|
||||
Assert.Equal("Laptop 5", row.Model);
|
||||
Assert.Equal(2, row.CurrentJourneyAttempts);
|
||||
Assert.Equal(1, row.JourneyCount);
|
||||
Assert.Equal(DeviceJourneyCalculator.CurrentVersion, row.JourneyVersion);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildDeviceHistoryRow_BackfillingAnOlderSession_KeepsExistingDisplayFields()
|
||||
{
|
||||
var older = Session("s-1", serial: "pf4x1abc", startedAt: T0);
|
||||
older.Model = "Laptop 4"; // stale model from the older enrollment
|
||||
var chain = new List<DeviceSessionRef>
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0),
|
||||
Ref("s-2", SessionStatus.Succeeded, T0.AddDays(1)),
|
||||
};
|
||||
var existing = new DeviceHistory
|
||||
{
|
||||
TenantId = TenantA, SerialKey = "pf4x1abc",
|
||||
SerialNumber = "PF4X1ABC", Manufacturer = "Contoso", Model = "Laptop 5",
|
||||
};
|
||||
|
||||
var row = TableStorageService.BuildDeviceHistoryRow(TenantA, "pf4x1abc", chain, existing, older);
|
||||
|
||||
Assert.Equal("PF4X1ABC", row.SerialNumber); // not regressed to the older session's data
|
||||
Assert.Equal("Laptop 5", row.Model);
|
||||
}
|
||||
|
||||
// ── persistence round-trips ─────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void DeviceHistoryEntity_RoundTripsAllFields_IncludingChainJson()
|
||||
{
|
||||
var history = new DeviceHistory
|
||||
{
|
||||
TenantId = TenantA,
|
||||
SerialKey = "pf4x1abc",
|
||||
SerialNumber = "PF4X1ABC",
|
||||
Manufacturer = "Contoso",
|
||||
Model = "Laptop 5",
|
||||
CurrentJourneyAttempts = 2,
|
||||
JourneyCount = 3,
|
||||
JourneyVersion = 1,
|
||||
LastUpdated = T0,
|
||||
Chain = new List<DeviceSessionRef>
|
||||
{
|
||||
new()
|
||||
{
|
||||
SessionId = "s-1", StartedAt = T0, CompletedAt = T0.AddHours(5),
|
||||
Status = "Succeeded", EnrollmentType = "v2", IsPreProvisioned = true,
|
||||
DurationSeconds = 1800, AdminMarked = true,
|
||||
},
|
||||
new()
|
||||
{
|
||||
SessionId = "s-2", StartedAt = T0.AddDays(1), CompletedAt = null,
|
||||
Status = "Incomplete", EnrollmentType = "v1", DurationSeconds = null,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var mapped = TableStorageService.MapToDeviceHistory(
|
||||
TableStorageService.BuildDeviceHistoryEntity(history));
|
||||
|
||||
Assert.Equal(TenantA, mapped.TenantId);
|
||||
Assert.Equal("pf4x1abc", mapped.SerialKey);
|
||||
Assert.Equal("PF4X1ABC", mapped.SerialNumber);
|
||||
Assert.Equal("Contoso", mapped.Manufacturer);
|
||||
Assert.Equal("Laptop 5", mapped.Model);
|
||||
Assert.Equal(2, mapped.CurrentJourneyAttempts);
|
||||
Assert.Equal(3, mapped.JourneyCount);
|
||||
Assert.Equal(1, mapped.JourneyVersion);
|
||||
Assert.Equal(T0, mapped.LastUpdated);
|
||||
|
||||
Assert.Equal(2, mapped.Chain.Count);
|
||||
var first = mapped.Chain[0];
|
||||
Assert.Equal("s-1", first.SessionId);
|
||||
Assert.Equal(T0, first.StartedAt);
|
||||
Assert.Equal(T0.AddHours(5), first.CompletedAt);
|
||||
Assert.Equal("Succeeded", first.Status);
|
||||
Assert.Equal("v2", first.EnrollmentType);
|
||||
Assert.True(first.IsPreProvisioned);
|
||||
Assert.Equal(1800, first.DurationSeconds);
|
||||
Assert.True(first.AdminMarked);
|
||||
var second = mapped.Chain[1];
|
||||
Assert.Null(second.CompletedAt); // Incomplete: no terminal timestamp requirement
|
||||
Assert.Null(second.DurationSeconds); // Incomplete: deliberately no duration — stays null
|
||||
Assert.False(second.AdminMarked);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeviceHistoryEntity_RowKeyEncoding_MakesForbiddenKeyCharsSafe_AndRoundTrips()
|
||||
{
|
||||
// '/', '\', '#', '?' are forbidden in Table Storage keys — a defensive encoding keeps a
|
||||
// pathological serial writable, and the mapper restores the exact normalized serial.
|
||||
var weird = "ab/cd\\ef#1?x";
|
||||
var history = new DeviceHistory { TenantId = TenantA, SerialKey = weird };
|
||||
|
||||
var entity = TableStorageService.BuildDeviceHistoryEntity(history);
|
||||
Assert.All(new[] { "/", "\\", "#", "?" }, c => Assert.DoesNotContain(c, entity.RowKey));
|
||||
|
||||
Assert.Equal(weird, TableStorageService.MapToDeviceHistory(entity).SerialKey);
|
||||
|
||||
// Ordinary serials stay human-readable.
|
||||
Assert.Equal("pf4x1abc", TableStorageService.SerialRowKey("pf4x1abc"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeviceJourneyAggregateEntity_RoundTripsAllFields_RowKeyIsTheDate()
|
||||
{
|
||||
var aggregate = new DeviceJourneyDailyAggregate
|
||||
{
|
||||
TenantId = TenantA,
|
||||
Date = "2026-07-20",
|
||||
JourneyVersion = 1,
|
||||
CompletedJourneyCount = 23,
|
||||
FirstTimeRightCount = 19,
|
||||
ExcludedSessionCount = 2,
|
||||
ComputedAt = T0,
|
||||
AttemptHistogram = new List<DeviceJourneyAttemptBucket>
|
||||
{
|
||||
new() { Attempts = 1, JourneyCount = 19 },
|
||||
new() { Attempts = 2, JourneyCount = 3 },
|
||||
new() { Attempts = 4, JourneyCount = 1 },
|
||||
},
|
||||
};
|
||||
|
||||
var entity = TableStorageService.BuildDeviceJourneyAggregateEntity(aggregate);
|
||||
Assert.Equal("2026-07-20", entity.RowKey);
|
||||
|
||||
var mapped = TableStorageService.MapToDeviceJourneyAggregate(entity);
|
||||
Assert.Equal(TenantA, mapped.TenantId);
|
||||
Assert.Equal("2026-07-20", mapped.Date);
|
||||
Assert.Equal(1, mapped.JourneyVersion);
|
||||
Assert.Equal(23, mapped.CompletedJourneyCount);
|
||||
Assert.Equal(19, mapped.FirstTimeRightCount);
|
||||
Assert.Equal(2, mapped.ExcludedSessionCount);
|
||||
Assert.Equal(T0, mapped.ComputedAt);
|
||||
Assert.Equal(3, mapped.AttemptHistogram.Count);
|
||||
Assert.Equal(1, mapped.AttemptHistogram[0].Attempts);
|
||||
Assert.Equal(19, mapped.AttemptHistogram[0].JourneyCount);
|
||||
Assert.Equal(4, mapped.AttemptHistogram[2].Attempts);
|
||||
}
|
||||
|
||||
// ── daily FTR aggregation core ──────────────────────────────────────────
|
||||
|
||||
private static readonly DateTime WindowStart = T0.Date.AddDays(-30);
|
||||
private static readonly DateTime WindowEnd = T0.Date.AddDays(1);
|
||||
|
||||
private static DeviceHistory History(string tenantId, params DeviceSessionRef[] chain)
|
||||
=> new() { TenantId = tenantId, SerialKey = Guid.NewGuid().ToString("N"), Chain = chain.ToList() };
|
||||
|
||||
[Fact]
|
||||
public void FtrAggregates_BucketByCompletingSessionsStartDate_AndMirrorGlobal()
|
||||
{
|
||||
var histories = new List<DeviceHistory>
|
||||
{
|
||||
// Tenant A device 1: fail → success (2 attempts, completes on T0 date).
|
||||
History(TenantA,
|
||||
Ref("a1-1", SessionStatus.Failed, T0.AddDays(-1)),
|
||||
Ref("a1-2", SessionStatus.Succeeded, T0)),
|
||||
// Tenant A device 2: first-time right on the same date.
|
||||
History(TenantA, Ref("a2-1", SessionStatus.Succeeded, T0.AddHours(2))),
|
||||
// Tenant B device: first-time right on a different date.
|
||||
History(TenantB, Ref("b1-1", SessionStatus.Succeeded, T0.AddDays(-3))),
|
||||
};
|
||||
|
||||
var rows = MaintenanceService.BuildDeviceJourneyAggregates(
|
||||
histories, Array.Empty<SessionSummary>(), WindowStart, WindowEnd, T0);
|
||||
|
||||
// Tenant A (1 date) + Tenant B (1 date) + global (2 dates) = 4 rows.
|
||||
Assert.Equal(4, rows.Count);
|
||||
|
||||
var tenantARow = rows.Single(r => r.TenantId == TenantA);
|
||||
Assert.Equal(T0.ToString("yyyy-MM-dd"), tenantARow.Date);
|
||||
Assert.Equal(2, tenantARow.CompletedJourneyCount);
|
||||
Assert.Equal(1, tenantARow.FirstTimeRightCount);
|
||||
Assert.Equal(DeviceJourneyCalculator.CurrentVersion, tenantARow.JourneyVersion);
|
||||
Assert.Equal(2, tenantARow.AttemptHistogram.Count);
|
||||
Assert.Equal(1, tenantARow.AttemptHistogram[0].Attempts);
|
||||
Assert.Equal(1, tenantARow.AttemptHistogram[0].JourneyCount);
|
||||
Assert.Equal(2, tenantARow.AttemptHistogram[1].Attempts);
|
||||
|
||||
var globalOnT0 = rows.Single(r => r.TenantId == "global" && r.Date == T0.ToString("yyyy-MM-dd"));
|
||||
Assert.Equal(2, globalOnT0.CompletedJourneyCount); // tenant B's journey sits on its own date
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FtrAggregates_OpenAndAbandonedJourneys_NeverCount()
|
||||
{
|
||||
var histories = new List<DeviceHistory>
|
||||
{
|
||||
// Open journey: failures only (e.g. a WG device still waiting for its user session
|
||||
// has, at most, prior failed attempts in the chain — the Pending row itself never
|
||||
// becomes a ref).
|
||||
History(TenantA, Ref("a1-1", SessionStatus.Failed, T0)),
|
||||
// Abandoned journey (31d gap, no success) followed by a completed journey.
|
||||
History(TenantA,
|
||||
Ref("a2-1", SessionStatus.Failed, T0.AddDays(-20), completedAt: T0.AddDays(-20).AddHours(1)),
|
||||
Ref("a2-2", SessionStatus.Succeeded, T0.AddDays(11).AddHours(2))),
|
||||
};
|
||||
|
||||
var rows = MaintenanceService.BuildDeviceJourneyAggregates(
|
||||
histories, Array.Empty<SessionSummary>(), WindowStart, T0.Date.AddDays(12), T0);
|
||||
|
||||
// Only the completed journey produced rows: tenant + global on one date.
|
||||
Assert.Equal(2, rows.Count);
|
||||
var tenantRow = rows.Single(r => r.TenantId == TenantA);
|
||||
Assert.Equal(1, tenantRow.CompletedJourneyCount);
|
||||
Assert.Equal(1, tenantRow.FirstTimeRightCount); // the abandoned attempt belongs to a DIFFERENT journey
|
||||
Assert.Equal(T0.AddDays(11).ToString("yyyy-MM-dd"), tenantRow.Date);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FtrAggregates_JourneysCompletedOutsideTheWindow_AreSkipped()
|
||||
{
|
||||
var histories = new List<DeviceHistory>
|
||||
{
|
||||
History(TenantA, Ref("a1-1", SessionStatus.Succeeded, WindowStart.AddDays(-1))),
|
||||
};
|
||||
|
||||
var rows = MaintenanceService.BuildDeviceJourneyAggregates(
|
||||
histories, Array.Empty<SessionSummary>(), WindowStart, WindowEnd, T0);
|
||||
|
||||
Assert.Empty(rows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FtrAggregates_JunkSerialExclusions_ProduceDisclosureRows()
|
||||
{
|
||||
var excluded = new List<SessionSummary>
|
||||
{
|
||||
Session("x-1", serial: "Unknown", startedAt: T0),
|
||||
Session("x-2", serial: "Unknown", startedAt: T0.AddHours(1)),
|
||||
Session("x-3", serial: "Unknown", startedAt: WindowStart.AddDays(-2)), // outside window — ignored
|
||||
};
|
||||
|
||||
var rows = MaintenanceService.BuildDeviceJourneyAggregates(
|
||||
Array.Empty<DeviceHistory>(), excluded, WindowStart, WindowEnd, T0);
|
||||
|
||||
Assert.Equal(2, rows.Count); // tenant + global disclosure rows, no journey claim
|
||||
var tenantRow = rows.Single(r => r.TenantId == TenantA);
|
||||
Assert.Equal(0, tenantRow.CompletedJourneyCount);
|
||||
Assert.Equal(0, tenantRow.FirstTimeRightCount);
|
||||
Assert.Empty(tenantRow.AttemptHistogram);
|
||||
Assert.Equal(2, tenantRow.ExcludedSessionCount);
|
||||
Assert.Equal(2, rows.Single(r => r.TenantId == "global").ExcludedSessionCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FtrAggregates_TwoCompletedJourneysOfOneDevice_SameDay_BothCount()
|
||||
{
|
||||
// Redeploy on the same day: success closes journey 1, the next success closes journey 2.
|
||||
var histories = new List<DeviceHistory>
|
||||
{
|
||||
History(TenantA,
|
||||
Ref("a1-1", SessionStatus.Succeeded, T0),
|
||||
Ref("a1-2", SessionStatus.Succeeded, T0.AddHours(3))),
|
||||
};
|
||||
|
||||
var rows = MaintenanceService.BuildDeviceJourneyAggregates(
|
||||
histories, Array.Empty<SessionSummary>(), WindowStart, WindowEnd, T0);
|
||||
|
||||
var tenantRow = rows.Single(r => r.TenantId == TenantA);
|
||||
Assert.Equal(2, tenantRow.CompletedJourneyCount);
|
||||
Assert.Equal(2, tenantRow.FirstTimeRightCount);
|
||||
}
|
||||
|
||||
// ── attempt number (session banner, PR5) ────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void ComputeAttemptNumber_TerminalSessionInChain_ReturnsPositionWithinItsJourney()
|
||||
{
|
||||
var chain = new[]
|
||||
{
|
||||
Ref("s-1", SessionStatus.Failed, T0),
|
||||
Ref("s-2", SessionStatus.Succeeded, T0.AddHours(2)),
|
||||
Ref("s-3", SessionStatus.Failed, T0.AddDays(5)), // redeployment → new journey
|
||||
};
|
||||
|
||||
Assert.Equal(1, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-1", T0));
|
||||
Assert.Equal(2, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-2", T0.AddHours(2)));
|
||||
Assert.Equal(1, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-3", T0.AddDays(5)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeAttemptNumber_LiveSession_ContinuesTheOpenJourney()
|
||||
{
|
||||
var chain = new[] { Ref("s-1", SessionStatus.Failed, T0) };
|
||||
// s-2 is still InProgress — not a chain ref; the virtual-attempt rule places it.
|
||||
Assert.Equal(2, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-2", T0.AddDays(1)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeAttemptNumber_LiveSession_AfterCompletedJourney_IsAttemptOne()
|
||||
{
|
||||
var chain = new[] { Ref("s-1", SessionStatus.Succeeded, T0) };
|
||||
Assert.Equal(1, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-2", T0.AddDays(2)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeAttemptNumber_LiveSession_AfterThirtyDayGap_IsAttemptOne()
|
||||
{
|
||||
var prevEnd = T0.AddHours(1);
|
||||
var chain = new[] { Ref("s-1", SessionStatus.Failed, T0, completedAt: prevEnd) };
|
||||
Assert.Equal(1, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-2", prevEnd.AddDays(31)));
|
||||
// Inside the gap boundary the open journey continues instead.
|
||||
Assert.Equal(2, DeviceJourneyCalculator.ComputeAttemptNumber(chain, "s-3", prevEnd.AddDays(29)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeAttemptNumber_EmptyChain_YieldsNull_NoGuessedPosition()
|
||||
{
|
||||
Assert.Null(DeviceJourneyCalculator.ComputeAttemptNumber(
|
||||
Array.Empty<DeviceSessionRef>(), "s-1", T0));
|
||||
}
|
||||
|
||||
// ── fleet response helpers (PR5) ────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void SumAggregates_SumsAdditiveCounts_AndMergesHistograms()
|
||||
{
|
||||
var daily = new List<DeviceJourneyDailyAggregate>
|
||||
{
|
||||
new()
|
||||
{
|
||||
CompletedJourneyCount = 2, FirstTimeRightCount = 1, ExcludedSessionCount = 1,
|
||||
AttemptHistogram = new List<DeviceJourneyAttemptBucket>
|
||||
{
|
||||
new() { Attempts = 1, JourneyCount = 1 },
|
||||
new() { Attempts = 2, JourneyCount = 1 },
|
||||
},
|
||||
},
|
||||
new()
|
||||
{
|
||||
CompletedJourneyCount = 1, FirstTimeRightCount = 1, ExcludedSessionCount = 0,
|
||||
AttemptHistogram = new List<DeviceJourneyAttemptBucket> { new() { Attempts = 1, JourneyCount = 1 } },
|
||||
},
|
||||
};
|
||||
|
||||
var totals = DeviceJourneyMetricsResponse.SumAggregates(daily);
|
||||
|
||||
Assert.Equal(3, totals.CompletedJourneys);
|
||||
Assert.Equal(2, totals.FirstTimeRight);
|
||||
Assert.Equal(66.7, totals.FtrRatePct);
|
||||
Assert.Equal(1, totals.ExcludedSessions);
|
||||
Assert.Equal(2, totals.AttemptHistogram.Count);
|
||||
Assert.Equal(2, totals.AttemptHistogram.Single(b => b.Attempts == 1).JourneyCount);
|
||||
Assert.Equal(1, totals.AttemptHistogram.Single(b => b.Attempts == 2).JourneyCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SumAggregates_NoCompletedJourneys_RateIsNull_NeverZero()
|
||||
{
|
||||
var totals = DeviceJourneyMetricsResponse.SumAggregates(new List<DeviceJourneyDailyAggregate>
|
||||
{
|
||||
new() { CompletedJourneyCount = 0, FirstTimeRightCount = 0, ExcludedSessionCount = 3 },
|
||||
});
|
||||
|
||||
Assert.Equal(0, totals.CompletedJourneys);
|
||||
Assert.Null(totals.FtrRatePct);
|
||||
Assert.Equal(3, totals.ExcludedSessions);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, 30)] // default window
|
||||
[InlineData("garbage", 30)]
|
||||
[InlineData("7", 7)]
|
||||
[InlineData("0", 1)] // clamped low
|
||||
[InlineData("999", 180)] // clamped to aggregate retention
|
||||
public void ClampDays_DefaultsAndClamps(string? raw, int expected)
|
||||
{
|
||||
Assert.Equal(expected, DeviceJourneyMetricsResponse.ClampDays(raw));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InclusiveWindowStart_YieldsExactlyNCalendarDays()
|
||||
{
|
||||
// Codex review: both range ends are inclusive, so "today - days" returned N+1 keys —
|
||||
// days=1 summed yesterday AND today.
|
||||
var today = new DateTime(2026, 7, 27);
|
||||
Assert.Equal(today, DeviceJourneyMetricsResponse.InclusiveWindowStart(today, 1));
|
||||
Assert.Equal(today.AddDays(-29), DeviceJourneyMetricsResponse.InclusiveWindowStart(today, 30));
|
||||
}
|
||||
|
||||
private static DeviceHistory RepeaterHistory(
|
||||
string serial, int attempts, params DeviceSessionRef[] chain)
|
||||
=> new()
|
||||
{
|
||||
TenantId = TenantA,
|
||||
SerialKey = serial,
|
||||
SerialNumber = serial.ToUpperInvariant(),
|
||||
Chain = chain.ToList(),
|
||||
CurrentJourneyAttempts = attempts,
|
||||
JourneyCount = 1,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void SelectRepeatDevices_FiltersGate_WindowsOnNewestRef_AndPicksNewestFailure()
|
||||
{
|
||||
var histories = new List<DeviceHistory>
|
||||
{
|
||||
// Single attempt — not a repeat device.
|
||||
RepeaterHistory("clean-1", attempts: 1, Ref("c1", SessionStatus.Succeeded, T0)),
|
||||
// Repeat device, active in window: newest failed ref carries the reason lookup.
|
||||
RepeaterHistory("hot-1", attempts: 3,
|
||||
Ref("h1", SessionStatus.Failed, T0.AddDays(-2)),
|
||||
Ref("h2", SessionStatus.Failed, T0.AddDays(-1)),
|
||||
Ref("h3", SessionStatus.Incomplete, T0)),
|
||||
// Repeat device, but its pain is OLDER than the window — current lists only.
|
||||
RepeaterHistory("old-1", attempts: 4,
|
||||
Ref("o1", SessionStatus.Failed, T0.AddDays(-40))),
|
||||
// Lower attempt count → sorts after hot-1.
|
||||
RepeaterHistory("warm-1", attempts: 2,
|
||||
Ref("w1", SessionStatus.Failed, T0.AddDays(-3)),
|
||||
Ref("w2", SessionStatus.Succeeded, T0.AddDays(-2))),
|
||||
};
|
||||
|
||||
var selected = DeviceJourneyMetricsResponse.SelectRepeatDevices(histories, T0.AddDays(-30));
|
||||
|
||||
Assert.Equal(2, selected.Count);
|
||||
Assert.Equal("hot-1", selected[0].History.SerialKey); // attempts desc
|
||||
Assert.Equal("h3", selected[0].Newest.SessionId);
|
||||
Assert.Equal("h2", selected[0].NewestFailed!.SessionId); // last FAILED, not last ref
|
||||
Assert.Equal("warm-1", selected[1].History.SerialKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelectRepeatDevices_CapsAtTen_ByAttemptsThenRecency()
|
||||
{
|
||||
var histories = new List<DeviceHistory>();
|
||||
for (var i = 0; i < 12; i++)
|
||||
{
|
||||
histories.Add(RepeaterHistory($"dev-{i:D2}", attempts: 2 + i,
|
||||
Ref($"r-{i}", SessionStatus.Failed, T0.AddDays(-i))));
|
||||
}
|
||||
|
||||
var selected = DeviceJourneyMetricsResponse.SelectRepeatDevices(histories, T0.AddDays(-30));
|
||||
|
||||
Assert.Equal(10, selected.Count);
|
||||
Assert.Equal("dev-11", selected[0].History.SerialKey); // highest attempts first
|
||||
Assert.DoesNotContain(selected, c => c.History.SerialKey == "dev-00");
|
||||
Assert.DoesNotContain(selected, c => c.History.SerialKey == "dev-01");
|
||||
}
|
||||
}
|
||||
@@ -154,6 +154,80 @@ public class DiagnosticsBlobCascadeDeleterTests
|
||||
Assert.Equal(0, deleter.HostedDeleteCalls);
|
||||
}
|
||||
|
||||
// ── REAL DeleteAsync, Hosted branch — prefix sweep + belt-and-braces name delete ──
|
||||
// The RecordingDeleter above replaces DeleteAsync wholesale, so these run the REAL
|
||||
// base implementation against a fake storage service to pin the sweep behaviour.
|
||||
|
||||
private const string SessionGuid = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
|
||||
|
||||
[Fact]
|
||||
public async Task RealDeleteAsync_Hosted_SweepsSessionPrefix_AndDeletesManifestBlob()
|
||||
{
|
||||
var storage = new RecordingHostedStorage();
|
||||
var deleter = RealDeleter(storage);
|
||||
var manifest = ManifestWith(blobName: $"{TenantA}/{Filename}", destination: "Hosted");
|
||||
manifest.SessionId = SessionGuid;
|
||||
|
||||
var outcome = await deleter.DeleteAsync(manifest);
|
||||
|
||||
Assert.Equal(DiagnosticsBlobDeleteOutcome.HostedDeleted, outcome);
|
||||
Assert.Equal((TenantA, SessionGuid), storage.SweptSession);
|
||||
Assert.Equal($"{TenantA}/{Filename}", storage.DeletedBlobPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RealDeleteAsync_Hosted_NonGuidSessionId_SkipsSweep_StillDeletesManifestBlob()
|
||||
{
|
||||
// Legacy manifests may carry a malformed/empty SessionId — the sweep must degrade
|
||||
// to the single-blob delete, never throw into the cascade poison path.
|
||||
var storage = new RecordingHostedStorage();
|
||||
var deleter = RealDeleter(storage);
|
||||
var manifest = ManifestWith(blobName: $"{TenantA}/{Filename}", destination: "Hosted");
|
||||
manifest.SessionId = "not-a-guid";
|
||||
|
||||
var outcome = await deleter.DeleteAsync(manifest);
|
||||
|
||||
Assert.Equal(DiagnosticsBlobDeleteOutcome.HostedDeleted, outcome);
|
||||
Assert.Null(storage.SweptSession);
|
||||
Assert.Equal($"{TenantA}/{Filename}", storage.DeletedBlobPath);
|
||||
}
|
||||
|
||||
private static DiagnosticsBlobCascadeDeleter RealDeleter(RecordingHostedStorage storage)
|
||||
// TenantConfigurationService is only consulted on the CustomerSas branch; the
|
||||
// Hosted-branch tests never reach it, so null keeps the harness Azure-free.
|
||||
=> new(
|
||||
storage,
|
||||
tenantConfig: null!,
|
||||
Microsoft.Extensions.Logging.Abstractions.NullLogger<DiagnosticsBlobCascadeDeleter>.Instance);
|
||||
|
||||
private sealed class RecordingHostedStorage : HostedDiagnosticsBlobService
|
||||
{
|
||||
public (string TenantId, string SessionId)? SweptSession { get; private set; }
|
||||
public string? DeletedBlobPath { get; private set; }
|
||||
|
||||
public RecordingHostedStorage()
|
||||
: base(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
Microsoft.Extensions.Logging.Abstractions.NullLogger<HostedDiagnosticsBlobService>.Instance,
|
||||
usesManagedIdentity: false)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task<int> DeleteBySessionPrefixAsync(
|
||||
string tenantId, string sessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
SweptSession = (tenantId, sessionId);
|
||||
return Task.FromResult(1);
|
||||
}
|
||||
|
||||
public override Task<bool> DeleteIfExistsAsync(
|
||||
string blobPath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
DeletedBlobPath = blobPath;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
// ── BuildBlobUrl helper (CustomerSas URL construction) ─────────────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
+4
-4
@@ -160,9 +160,9 @@ public class EndpointPolicyCatalogCompletenessTests
|
||||
|
||||
Assert.NotNull(getConfig);
|
||||
Assert.NotNull(putConfig);
|
||||
// GET config read is admin-tier read (own-tenant Admin / GA / read-only GlobalReader);
|
||||
// PUT config write stays Admin/GA only.
|
||||
Assert.Equal(EndpointPolicy.TenantAdminOrGlobalReader, getConfig.Policy);
|
||||
// GET config read is member-tier (Operators/Viewers get the redacted read-only Settings
|
||||
// view; the handler redacts for every non-admin caller); PUT config write stays Admin/GA only.
|
||||
Assert.Equal(EndpointPolicy.MemberRead, getConfig.Policy);
|
||||
Assert.Equal(EndpointPolicy.TenantAdminOrGA, putConfig.Policy);
|
||||
|
||||
var getRules = EndpointAccessPolicyCatalog.FindPolicy("GET", "/api/rules/gather");
|
||||
@@ -338,7 +338,7 @@ public class EndpointPolicyCatalogCompletenessTests
|
||||
[InlineData("POST", "/api/agent/telemetry", true)] // DeviceOrBootstrapAuth
|
||||
[InlineData("GET", "/api/bootstrap/validate/ABC123", true)] // PublicAnonymous (param route)
|
||||
[InlineData("GET", "/api/sessions", false)] // MemberRead
|
||||
[InlineData("GET", "/api/config/00000000-0000-0000-0000-000000000001", false)] // TenantAdminOrGlobalReader
|
||||
[InlineData("GET", "/api/config/00000000-0000-0000-0000-000000000001", false)] // MemberRead
|
||||
[InlineData("GET", "/api/global/sessions", false)] // GlobalReadOrAdmin
|
||||
[InlineData("GET", "/api/nonexistent", false)] // unregistered → fail-closed
|
||||
public void SkipsJwtValidation_MatchesPolicyTier(string method, string path, bool expectedExempt)
|
||||
|
||||
@@ -10,9 +10,10 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// Covers the server-side Fleet Health aggregation in
|
||||
/// <see cref="MetricsMath.BuildFleetHealthPayload"/>, the single source of truth shared by the
|
||||
/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. These assert
|
||||
/// the semantics ported verbatim from the previous client-side aggregation — notably success rate
|
||||
/// over ALL sessions and average duration over non-in-progress sessions (failures included).
|
||||
/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. Success and
|
||||
/// failure rates follow the SLA convention — over finished enrollments (Succeeded + Failed) only,
|
||||
/// so in-flight sessions never dilute them. Average duration stays over non-in-progress sessions
|
||||
/// that carry a duration (failures included).
|
||||
/// </summary>
|
||||
public class FleetHealthPayloadTests
|
||||
{
|
||||
@@ -55,7 +56,7 @@ public class FleetHealthPayloadTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsOverAllSessions_NotJustTerminal()
|
||||
public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
@@ -63,16 +64,34 @@ public class FleetHealthPayloadTests
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Failed),
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate
|
||||
};
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
|
||||
|
||||
Assert.Equal(4, payload.Stats.Total);
|
||||
Assert.Equal(6, payload.Stats.Total);
|
||||
Assert.Equal(2, payload.Stats.Succeeded);
|
||||
Assert.Equal(1, payload.Stats.Failed);
|
||||
Assert.Equal(1, payload.Stats.InProgress);
|
||||
// 2 / 4 = 50% (over total), NOT 2 / 3 = 66.7% (over terminal).
|
||||
Assert.Equal(50d, payload.Stats.SuccessRate);
|
||||
Assert.Equal(1, payload.Stats.Incomplete);
|
||||
// 2 / (2 + 1) = 66.7% (over finished), NOT 2 / 6 = 33.3% (over total).
|
||||
Assert.Equal(66.7d, payload.Stats.SuccessRate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsZero_WhenNothingFinishedYet()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
};
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
|
||||
|
||||
// No finished enrollments → no rate; clients render "—" off Succeeded + Failed == 0.
|
||||
Assert.Equal(0d, payload.Stats.SuccessRate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -111,6 +130,27 @@ public class FleetHealthPayloadTests
|
||||
Assert.DoesNotContain(payload.ModelHealth, m => m.Model == "Contoso Model1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModelHealth_CountsFailedSeparately_InFlightInTotalOnly()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.Succeeded, model: "X"),
|
||||
S(SessionStatus.Failed, model: "X"),
|
||||
S(SessionStatus.InProgress, model: "X"),
|
||||
};
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
|
||||
|
||||
var m = Assert.Single(payload.ModelHealth);
|
||||
// Total keeps the full device count; the client derives the rate from
|
||||
// Succeeded / (Succeeded + Failed), so the in-flight session must land
|
||||
// in Total but in neither outcome bucket.
|
||||
Assert.Equal(3, m.Total);
|
||||
Assert.Equal(1, m.Succeeded);
|
||||
Assert.Equal(1, m.Failed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModelKey_FallsBackToUnknown_WhenManufacturerAndModelBlank()
|
||||
{
|
||||
@@ -147,10 +187,13 @@ public class FleetHealthPayloadTests
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// Model A: 1 failed of 2 → 50%
|
||||
// Model A: 1 failed / 2 finished → 50%; the 2 in-flight sessions widen Total
|
||||
// but must not dilute the rate (over-total it would read 25%).
|
||||
S(SessionStatus.Failed, model: "A"),
|
||||
S(SessionStatus.Succeeded, model: "A"),
|
||||
// Model B: 2 failed of 2 → 100%
|
||||
S(SessionStatus.InProgress, model: "A"),
|
||||
S(SessionStatus.Pending, model: "A"),
|
||||
// Model B: 2 failed of 2 finished → 100%
|
||||
S(SessionStatus.Failed, model: "B"),
|
||||
S(SessionStatus.Failed, model: "B"),
|
||||
// Model C: no failures → excluded entirely
|
||||
@@ -164,35 +207,37 @@ public class FleetHealthPayloadTests
|
||||
Assert.Equal(2, payload.TopFailingModels[0].Failed);
|
||||
Assert.Equal(100, payload.TopFailingModels[0].FailureRate);
|
||||
Assert.Equal("Contoso A", payload.TopFailingModels[1].Model);
|
||||
Assert.Equal(4, payload.TopFailingModels[1].Total);
|
||||
Assert.Equal(50, payload.TopFailingModels[1].FailureRate);
|
||||
Assert.DoesNotContain(payload.TopFailingModels, m => m.Model == "Contoso C");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailureReasons_TruncateLongReasons_AndTakeTopFive()
|
||||
public void FailureReasons_GroupByPrefix_KeepFullestText_AndTakeTopFive()
|
||||
{
|
||||
var sessions = new List<SessionSummary>();
|
||||
// 6 distinct reasons with descending frequency 6..1 → only top 5 kept.
|
||||
for (int freq = 6; freq >= 1; freq--)
|
||||
// 5 distinct short reasons with descending frequency 5..1.
|
||||
for (int freq = 5; freq >= 1; freq--)
|
||||
for (int i = 0; i < freq; i++)
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: $"Reason {freq}"));
|
||||
// A null reason folds into "Unknown".
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: null));
|
||||
// An over-long reason is collapsed to a 50-char prefix + ellipsis.
|
||||
var longReason = new string('x', 60);
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: longReason));
|
||||
|
||||
// Two variants that share the first 50 characters collapse into one group;
|
||||
// the longest variant is kept for display so the UI can show the full text.
|
||||
var prefix = new string('x', 50);
|
||||
var shortVariant = prefix + " short";
|
||||
var longVariant = prefix + " a much longer tail that must survive intact";
|
||||
for (int i = 0; i < 6; i++)
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: shortVariant));
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: longVariant));
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 30);
|
||||
|
||||
Assert.Equal(5, payload.FailureReasons.Count);
|
||||
Assert.Equal("Reason 6", payload.FailureReasons[0].Reason);
|
||||
Assert.Equal(6, payload.FailureReasons[0].Count);
|
||||
// The collapsed long reason exists in the tally (50 chars + "...").
|
||||
var collapsed = new string('x', 50) + "...";
|
||||
Assert.True(
|
||||
sessions.Count(s => s.FailureReason == longReason) == 1,
|
||||
"sanity: exactly one long-reason session");
|
||||
Assert.Equal(53, collapsed.Length);
|
||||
// The grouped cluster (6 + 1 = 7) outranks every "Reason N".
|
||||
Assert.Equal(7, payload.FailureReasons[0].Count);
|
||||
// The full, longest text survives end-to-end — no 50-char truncation.
|
||||
Assert.Equal(longVariant, payload.FailureReasons[0].Reason);
|
||||
Assert.DoesNotContain("...", payload.FailureReasons[0].Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -50,6 +50,8 @@ public class GeoMetricsProjectionEquivalenceTests
|
||||
["GeoCity"] = "Frankfurt",
|
||||
["GeoLoc"] = "50.11,8.68",
|
||||
["IsPreProvisioned"] = false,
|
||||
["AvgApiLatencyMs"] = 217.4,
|
||||
["ApiRequestCount"] = 270,
|
||||
// Noise the geo aggregation never reads.
|
||||
["SerialNumber"] = "SN-FULL",
|
||||
["Manufacturer"] = "Contoso",
|
||||
@@ -76,6 +78,8 @@ public class GeoMetricsProjectionEquivalenceTests
|
||||
Assert.Equal(fromFull.GeoRegion, fromProjected.GeoRegion);
|
||||
Assert.Equal(fromFull.GeoCity, fromProjected.GeoCity);
|
||||
Assert.Equal(fromFull.GeoLoc, fromProjected.GeoLoc);
|
||||
Assert.Equal(fromFull.AvgApiLatencyMs, fromProjected.AvgApiLatencyMs);
|
||||
Assert.Equal(fromFull.ApiRequestCount, fromProjected.ApiRequestCount);
|
||||
|
||||
// The grouping key must be identical too — it is composed from the Geo* fields.
|
||||
Assert.Equal(
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the API-latency aggregation of
|
||||
/// <see cref="GetGeographicMetricsFunction.ComputeGeographicMetrics"/> (2026-07-28): both the
|
||||
/// per-location and the global figure are REQUEST-WEIGHTED averages —
|
||||
/// Σ(sessionAvg·requests)/Σ(requests) — not averages of session averages, so a long session
|
||||
/// with many uploads counts proportionally. Sessions without latency data (agents predating
|
||||
/// <c>AvgApiLatencyMs</c>) drop out of the weighting entirely.
|
||||
/// </summary>
|
||||
public class GeographicMetricsApiLatencyTests
|
||||
{
|
||||
private static SessionSummary S(string country, string city, double? avgLatencyMs, int? requestCount)
|
||||
=> new()
|
||||
{
|
||||
TenantId = "00000000-0000-0000-0000-000000000fe0",
|
||||
SessionId = Guid.NewGuid().ToString(),
|
||||
Status = SessionStatus.Succeeded,
|
||||
StartedAt = DateTime.UtcNow.AddHours(-2),
|
||||
GeoCountry = country,
|
||||
GeoRegion = country,
|
||||
GeoCity = city,
|
||||
GeoLoc = "0,0",
|
||||
AvgApiLatencyMs = avgLatencyMs,
|
||||
ApiRequestCount = requestCount,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void LocationLatency_IsRequestWeighted_NotAverageOfAverages()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S("ID", "Jakarta", 1000, 300),
|
||||
S("ID", "Jakarta", 500, 100),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "country");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
// (1000·300 + 500·100) / 400 = 875 — an unweighted mean would say 750.
|
||||
Assert.Equal(875d, loc.AvgApiLatencyMs);
|
||||
Assert.Equal(2, loc.ApiLatencySessionCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SessionsWithoutLatencyData_DropOutOfWeighting_ButStayInSessionCount()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S("DE", "Frankfurt", 200, 100),
|
||||
S("DE", "Frankfurt", avgLatencyMs: null, requestCount: null), // pre-feature agent
|
||||
S("DE", "Frankfurt", avgLatencyMs: 0, requestCount: 50), // zero = no signal
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "country");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
Assert.Equal(3, loc.SessionCount);
|
||||
Assert.Equal(1, loc.ApiLatencySessionCount);
|
||||
Assert.Equal(200d, loc.AvgApiLatencyMs);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NoLatencyDataAnywhere_YieldsZeroes_NotDivisionByZero()
|
||||
{
|
||||
var sessions = new List<SessionSummary> { S("DE", "Frankfurt", null, null) };
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "country");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
Assert.Equal(0d, loc.AvgApiLatencyMs);
|
||||
Assert.Equal(0, loc.ApiLatencySessionCount);
|
||||
Assert.Equal(0d, loc.ApiLatencyVsGlobalPct);
|
||||
Assert.Equal(0d, result.GlobalAverages.AvgApiLatencyMs);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GlobalLatency_WeightsAcrossLocations_AndVsGlobalPctCompares()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S("DE", "Frankfurt", 200, 100),
|
||||
S("ID", "Jakarta", 800, 100),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "country");
|
||||
|
||||
// Equal request weights → global = (200 + 800) / 2 = 500.
|
||||
Assert.Equal(500d, result.GlobalAverages.AvgApiLatencyMs);
|
||||
|
||||
var de = result.Locations.Single(l => l.Country == "DE");
|
||||
var id = result.Locations.Single(l => l.Country == "ID");
|
||||
Assert.Equal(-60d, de.ApiLatencyVsGlobalPct); // 200 vs 500
|
||||
Assert.Equal(60d, id.ApiLatencyVsGlobalPct); // 800 vs 500
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LocationWithoutData_KeepsZeroPct_EvenWhenGlobalHasData()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S("DE", "Frankfurt", 200, 100),
|
||||
S("FR", "Paris", null, null),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "country");
|
||||
|
||||
var fr = result.Locations.Single(l => l.Country == "FR");
|
||||
Assert.Equal(0d, fr.AvgApiLatencyMs);
|
||||
Assert.Equal(0d, fr.ApiLatencyVsGlobalPct); // no data ≠ "0 ms fast"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the success-rate semantics of the geographic aggregation
|
||||
/// (<see cref="GetGeographicMetricsFunction.ComputeGeographicMetrics"/>): the rate is an outcome
|
||||
/// quota over finished enrollments (Succeeded + Failed) following the SLA convention. In-flight
|
||||
/// sessions (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal, non-failure) sit
|
||||
/// in SessionCount but never dilute the rate — the customer-visible symptom this pins against was
|
||||
/// a site mid-rollout showing a poor rate despite zero actual failures.
|
||||
/// </summary>
|
||||
public class GeographicMetricsSuccessRateTests
|
||||
{
|
||||
private static SessionSummary S(SessionStatus status, string city = "Frankfurt")
|
||||
=> new()
|
||||
{
|
||||
TenantId = "00000000-0000-0000-0000-000000000fe0",
|
||||
SessionId = Guid.NewGuid().ToString(),
|
||||
Status = status,
|
||||
StartedAt = DateTime.UtcNow.AddHours(-2),
|
||||
GeoCountry = "DE",
|
||||
GeoRegion = "Hessen",
|
||||
GeoCity = city,
|
||||
GeoLoc = "50.11,8.68",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Failed),
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
S(SessionStatus.AwaitingUser),
|
||||
S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "city");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
Assert.Equal(8, loc.SessionCount);
|
||||
Assert.Equal(3, loc.Succeeded);
|
||||
Assert.Equal(1, loc.Failed);
|
||||
// 3 / (3 + 1) = 75% (over finished), NOT 3 / 8 = 37.5% (over all sessions).
|
||||
Assert.Equal(75d, loc.SuccessRate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsZero_WhenNothingFinishedYet()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "city");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
// No finished enrollments → no rate; the frontend renders "—" off Succeeded + Failed == 0.
|
||||
Assert.Equal(0d, loc.SuccessRate);
|
||||
Assert.Equal(0, loc.Succeeded + loc.Failed);
|
||||
Assert.Equal(2, loc.SessionCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsPerLocation_NotGlobal()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// Frankfurt: 1 / 1 finished → 100% despite the in-flight session.
|
||||
S(SessionStatus.Succeeded, city: "Frankfurt"),
|
||||
S(SessionStatus.InProgress, city: "Frankfurt"),
|
||||
// Berlin: 1 / 2 finished → 50%.
|
||||
S(SessionStatus.Succeeded, city: "Berlin"),
|
||||
S(SessionStatus.Failed, city: "Berlin"),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "city");
|
||||
|
||||
var frankfurt = result.Locations.Single(l => l.City == "Frankfurt");
|
||||
var berlin = result.Locations.Single(l => l.City == "Berlin");
|
||||
Assert.Equal(100d, frankfurt.SuccessRate);
|
||||
Assert.Equal(50d, berlin.SuccessRate);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Guards the cross-tenant redaction on GET metrics/ime-versions.
|
||||
///
|
||||
/// The route is <c>MemberRead</c> (EndpointAccessPolicyCatalog) and the archive itself is a
|
||||
/// single global partition — every tenant member reads the SAME platform-wide rows. The only
|
||||
/// thing standing between an ordinary tenant user and another tenant's identifiers is the
|
||||
/// projection in <see cref="GetImeVersionHistoryFunction.BuildResponsePayload"/>. That made it
|
||||
/// worth pinning: the redaction had no test at all, and a boundary nobody asserts is one a
|
||||
/// later refactor can open without a single failure.
|
||||
///
|
||||
/// The JSON tests are the ones that matter — a property dropped from an anonymous type is only
|
||||
/// a real redaction once it is also absent from the wire, and the wire is what the customer's
|
||||
/// MCP client receives. camelCase mirrors Program.cs's PropertyNamingPolicy.
|
||||
/// </summary>
|
||||
public class GetImeVersionHistoryRedactionTests
|
||||
{
|
||||
private static readonly DateTime First = new(2026, 7, 8, 3, 18, 13, DateTimeKind.Utc);
|
||||
private static readonly DateTime Last = new(2026, 7, 27, 16, 59, 34, DateTimeKind.Utc);
|
||||
|
||||
private static readonly JsonSerializerOptions CamelCase =
|
||||
new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
|
||||
|
||||
// =========================================================================
|
||||
// Non-global caller (ordinary tenant member) — the redaction
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void NonGlobalCaller_DropsForeignTenantAndSessionIds()
|
||||
{
|
||||
var json = SerializeFor(hasGlobalScope: false, MakeEntry());
|
||||
|
||||
Assert.DoesNotContain("firstSeenTenantId", json, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("firstSeenSessionId", json, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The values, not just the property names: a rename of the model property must not let the
|
||||
/// identifier survive under a different key. Asserting on the GUIDs catches that.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void NonGlobalCaller_ForeignIdentifierValuesAppearNowhereInThePayload()
|
||||
{
|
||||
var json = SerializeFor(hasGlobalScope: false, MakeEntry());
|
||||
|
||||
Assert.DoesNotContain("57f34dd1-6a42-47f9-9bc0-b921fa6caa30", json, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("baae0453-6d19-47b5-aa58-8bed5a572fc8", json, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonGlobalCaller_ExposesExactlyTheFourAllowedProperties()
|
||||
{
|
||||
using var doc = JsonDocument.Parse(SerializeFor(hasGlobalScope: false, MakeEntry()));
|
||||
|
||||
var keys = doc.RootElement[0].EnumerateObject().Select(p => p.Name).OrderBy(n => n).ToArray();
|
||||
|
||||
Assert.Equal(new[] { "firstSeenAt", "lastSeenAt", "sessionCount", "version" }, keys);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SessionCount stays on purpose — it is a platform-wide total, and the route exists so any
|
||||
/// tenant member can track Microsoft's IME rollout. Pinned so that "the counts are global"
|
||||
/// stays a decision someone made rather than something that quietly drifts either way.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void NonGlobalCaller_KeepsVersionDatesAndGlobalSessionCount()
|
||||
{
|
||||
using var doc = JsonDocument.Parse(SerializeFor(hasGlobalScope: false, MakeEntry()));
|
||||
|
||||
var row = doc.RootElement[0];
|
||||
Assert.Equal("1.103.101.0", row.GetProperty("version").GetString());
|
||||
Assert.Equal(First, row.GetProperty("firstSeenAt").GetDateTime());
|
||||
Assert.Equal(Last, row.GetProperty("lastSeenAt").GetDateTime());
|
||||
Assert.Equal(3109, row.GetProperty("sessionCount").GetInt32());
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Global caller (Global Admin or Global Reader) — full archive
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void GlobalCaller_KeepsTenantAndSessionIds()
|
||||
{
|
||||
using var doc = JsonDocument.Parse(SerializeFor(hasGlobalScope: true, MakeEntry()));
|
||||
|
||||
var row = doc.RootElement[0];
|
||||
Assert.Equal("57f34dd1-6a42-47f9-9bc0-b921fa6caa30", row.GetProperty("firstSeenTenantId").GetString());
|
||||
Assert.Equal("baae0453-6d19-47b5-aa58-8bed5a572fc8", row.GetProperty("firstSeenSessionId").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GlobalCaller_ExposesTheFullEntry()
|
||||
{
|
||||
using var doc = JsonDocument.Parse(SerializeFor(hasGlobalScope: true, MakeEntry()));
|
||||
|
||||
var keys = doc.RootElement[0].EnumerateObject().Select(p => p.Name).OrderBy(n => n).ToArray();
|
||||
|
||||
Assert.Equal(
|
||||
new[] { "firstSeenAt", "firstSeenSessionId", "firstSeenTenantId", "lastSeenAt", "sessionCount", "version" },
|
||||
keys);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Shape-preserving properties of the projection
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void BothScopes_EmitAJsonArray_EvenForASingleRow()
|
||||
{
|
||||
foreach (var scope in new[] { true, false })
|
||||
{
|
||||
using var doc = JsonDocument.Parse(SerializeFor(scope, MakeEntry()));
|
||||
Assert.Equal(JsonValueKind.Array, doc.RootElement.ValueKind);
|
||||
Assert.Equal(1, doc.RootElement.GetArrayLength());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BothScopes_EmptyArchive_SerializesAsEmptyArray()
|
||||
{
|
||||
foreach (var scope in new[] { true, false })
|
||||
{
|
||||
using var doc = JsonDocument.Parse(SerializeFor(scope));
|
||||
Assert.Equal(JsonValueKind.Array, doc.RootElement.ValueKind);
|
||||
Assert.Equal(0, doc.RootElement.GetArrayLength());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The repository orders by FirstSeenAt descending; the projection must not reshuffle, or a
|
||||
/// redacted caller would see a different sequence than a Global Admin for the same archive.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void NonGlobalCaller_PreservesRowOrderAndCount()
|
||||
{
|
||||
var newest = MakeEntry("1.103.101.0", First);
|
||||
var older = MakeEntry("1.101.111.0", First.AddDays(-48));
|
||||
var oldest = MakeEntry("1.99.101.0", First.AddDays(-131));
|
||||
|
||||
using var doc = JsonDocument.Parse(SerializeFor(false, newest, older, oldest));
|
||||
|
||||
Assert.Equal(3, doc.RootElement.GetArrayLength());
|
||||
Assert.Equal("1.103.101.0", doc.RootElement[0].GetProperty("version").GetString());
|
||||
Assert.Equal("1.101.111.0", doc.RootElement[1].GetProperty("version").GetString());
|
||||
Assert.Equal("1.99.101.0", doc.RootElement[2].GetProperty("version").GetString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BuildResponsePayload returns `object`; the branch that keeps everything hands back the
|
||||
/// entries themselves. System.Text.Json resolves an `object` declared type via the runtime
|
||||
/// type, which is what keeps the wire format identical to the pre-extraction inline code —
|
||||
/// assert it rather than trust it, since a regression here would silently emit "{}" rows.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GlobalPayload_SerializedAsObject_StillEmitsPopulatedRows()
|
||||
{
|
||||
var payload = GetImeVersionHistoryFunction.BuildResponsePayload([MakeEntry()], hasGlobalScope: true);
|
||||
|
||||
var json = JsonSerializer.Serialize(payload, CamelCase);
|
||||
|
||||
Assert.Contains("1.103.101.0", json, StringComparison.Ordinal);
|
||||
Assert.NotEqual("[{}]", json);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Helpers
|
||||
// =========================================================================
|
||||
|
||||
private static string SerializeFor(bool hasGlobalScope, params ImeVersionHistoryEntry[] versions) =>
|
||||
JsonSerializer.Serialize(
|
||||
GetImeVersionHistoryFunction.BuildResponsePayload(versions, hasGlobalScope),
|
||||
CamelCase);
|
||||
|
||||
private static ImeVersionHistoryEntry MakeEntry(string version = "1.103.101.0", DateTime? firstSeenAt = null) =>
|
||||
new()
|
||||
{
|
||||
Version = version,
|
||||
FirstSeenAt = firstSeenAt ?? First,
|
||||
FirstSeenSessionId = "baae0453-6d19-47b5-aa58-8bed5a572fc8",
|
||||
FirstSeenTenantId = "57f34dd1-6a42-47f9-9bc0-b921fa6caa30",
|
||||
LastSeenAt = Last,
|
||||
SessionCount = 3109,
|
||||
};
|
||||
}
|
||||
@@ -35,6 +35,7 @@ public class GetTenantFeatureFlagsPayloadTests
|
||||
Assert.Equal(new[]
|
||||
{
|
||||
"bootstrapTokenEnabled",
|
||||
"diagnosticsUploadConfigured",
|
||||
"edition",
|
||||
"enableIntegrityBypassAnalyzer",
|
||||
"enableSoftwareInventoryAnalyzer",
|
||||
@@ -114,6 +115,35 @@ public class GetTenantFeatureFlagsPayloadTests
|
||||
Assert.DoesNotContain("webhookUrl", json, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
// ── diagnosticsUploadConfigured (drives the session-detail "Collect Logs" button) ──
|
||||
|
||||
[Theory]
|
||||
// mode Off (or unset) → never configured, regardless of destination
|
||||
[InlineData(null, null, null, false)]
|
||||
[InlineData("Off", "https://x.blob.core.windows.net/c?sv=1", "CustomerSas", false)]
|
||||
[InlineData("Off", null, "Hosted", false)]
|
||||
// mode active + Hosted destination → configured without any SAS
|
||||
[InlineData("OnFailure", null, "Hosted", true)]
|
||||
[InlineData("Always", null, "Hosted", true)]
|
||||
// mode active + CustomerSas → configured only when a SAS URL is present
|
||||
[InlineData("OnFailure", "https://x.blob.core.windows.net/c?sv=1", "CustomerSas", true)]
|
||||
[InlineData("OnFailure", null, "CustomerSas", false)]
|
||||
[InlineData("Always", "", null, false)]
|
||||
public void Payload_DiagnosticsUploadConfigured_ModeAndDestinationMatrix(
|
||||
string? mode, string? sasUrl, string? destination, bool expected)
|
||||
{
|
||||
// Null-forgiving: the model declares these non-nullable with defaults, but table
|
||||
// round-trips can produce null at runtime — the payload must tolerate that shape.
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
DiagnosticsUploadMode = mode!,
|
||||
DiagnosticsBlobSasUrl = sasUrl!,
|
||||
DiagnosticsUploadDestination = destination!,
|
||||
});
|
||||
|
||||
Assert.Equal(expected, element.GetProperty("diagnosticsUploadConfigured").GetBoolean());
|
||||
}
|
||||
|
||||
// ── Edition / trial surface ─────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
using AutopilotMonitor.Functions.Functions.Reports;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the TPM-PSS-unsupported aggregation logic extracted from
|
||||
/// GetTpmPssUnsupportedFunction. All tests target the pure static BuildAggregatedResult
|
||||
/// method — no HTTP or DI needed. Aggregation is by serial number (the TPM incompatibility
|
||||
/// is a per-device condition, not a per-model one — firmware levels differ within a model).
|
||||
/// </summary>
|
||||
public class GetTpmPssUnsupportedFunctionTests
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// =========================================================================
|
||||
// Empty / no matching reports
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_EmptyList_ReturnsEmpty()
|
||||
{
|
||||
var (aggregated, totalRawReports) = GetTpmPssUnsupportedFunction.BuildAggregatedResult([]);
|
||||
|
||||
Assert.Empty(aggregated);
|
||||
Assert.Equal(0, totalRawReports);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_FiltersTpmPssUnsupportedOnly()
|
||||
{
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", T0),
|
||||
MakeEntry("DeviceNotRegistered", "Lenovo", "ThinkPad T14", "SN2", T0.AddMinutes(1)),
|
||||
MakeEntry("AuthCertificateRejected", "HP", "EliteBook", "SN3", T0.AddMinutes(2)),
|
||||
};
|
||||
|
||||
var (aggregated, totalRawReports) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
Assert.Single(aggregated);
|
||||
Assert.Equal(1, totalRawReports);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Aggregation by serial number
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_AggregatesBySerialCaseInsensitive()
|
||||
{
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "S4SQ8685", T0),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "s4sq8685", T0.AddMinutes(1)),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", " S4SQ8685 ", T0.AddMinutes(2)),
|
||||
};
|
||||
|
||||
var (aggregated, totalRawReports) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
Assert.Single(aggregated);
|
||||
Assert.Equal(3, totalRawReports);
|
||||
var item = ToDynamic(aggregated[0]);
|
||||
Assert.Equal(3, (int)item.attemptCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_DistinctSerials_ProduceDistinctRows()
|
||||
{
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", T0),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN2", T0.AddMinutes(1)),
|
||||
MakeEntry("TpmPssUnsupported", "Microsoft", "Surface Book", "SN3", T0.AddMinutes(2)),
|
||||
};
|
||||
|
||||
var (aggregated, totalRawReports) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
Assert.Equal(3, aggregated.Count);
|
||||
Assert.Equal(3, totalRawReports);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_UsesMostRecentManufacturerModel()
|
||||
{
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "OldMfr", "OldModel", "SN1", T0),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", T0.AddMinutes(5)),
|
||||
};
|
||||
|
||||
var (aggregated, _) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
var item = ToDynamic(aggregated[0]);
|
||||
Assert.Equal("Lenovo", (string)item.manufacturer);
|
||||
Assert.Equal("ThinkCentre M720q", (string)item.model);
|
||||
Assert.Equal("SN1", (string)item.serialNumber);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Timestamps
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_FirstSeenLastSeen_Correct()
|
||||
{
|
||||
var t1 = T0;
|
||||
var t2 = T0.AddHours(1);
|
||||
var t3 = T0.AddHours(2);
|
||||
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", t2),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", t1),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", t3),
|
||||
};
|
||||
|
||||
var (aggregated, _) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
var item = ToDynamic(aggregated[0]);
|
||||
Assert.Equal(t1, (DateTime)item.firstSeen);
|
||||
Assert.Equal(t3, (DateTime)item.lastSeen);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_OrderedByLastSeenDescending()
|
||||
{
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "SN1", T0),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkPad X13", "SN2", T0.AddHours(2)),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720s", "SN3", T0.AddHours(1)),
|
||||
};
|
||||
|
||||
var (aggregated, _) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
Assert.Equal(3, aggregated.Count);
|
||||
var first = ToDynamic(aggregated[0]);
|
||||
var second = ToDynamic(aggregated[1]);
|
||||
var third = ToDynamic(aggregated[2]);
|
||||
|
||||
Assert.True((DateTime)first.lastSeen > (DateTime)second.lastSeen);
|
||||
Assert.True((DateTime)second.lastSeen > (DateTime)third.lastSeen);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Null / empty serial number
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void BuildAggregatedResult_NullOrEmptySerials_GroupedIntoSingleBucket()
|
||||
{
|
||||
var reports = new List<DistressReportEntry>
|
||||
{
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", null, T0),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", "", T0.AddMinutes(1)),
|
||||
MakeEntry("TpmPssUnsupported", "Lenovo", "ThinkCentre M720q", " ", T0.AddMinutes(2)),
|
||||
};
|
||||
|
||||
var (aggregated, _) = GetTpmPssUnsupportedFunction.BuildAggregatedResult(reports);
|
||||
|
||||
Assert.Single(aggregated);
|
||||
var item = ToDynamic(aggregated[0]);
|
||||
Assert.Equal(3, (int)item.attemptCount);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Helpers
|
||||
// =========================================================================
|
||||
|
||||
private static DistressReportEntry MakeEntry(
|
||||
string errorType, string? manufacturer, string? model, string? serial, DateTime ingestedAt) =>
|
||||
new()
|
||||
{
|
||||
TenantId = "test-tenant",
|
||||
ErrorType = errorType,
|
||||
Manufacturer = manufacturer,
|
||||
Model = model,
|
||||
SerialNumber = serial,
|
||||
IngestedAt = ingestedAt,
|
||||
AgentTimestamp = ingestedAt,
|
||||
};
|
||||
|
||||
private static dynamic ToDynamic(object obj)
|
||||
{
|
||||
var type = obj.GetType();
|
||||
var dict = new System.Dynamic.ExpandoObject() as IDictionary<string, object?>;
|
||||
foreach (var prop in type.GetProperties())
|
||||
{
|
||||
dict[prop.Name] = prop.GetValue(obj);
|
||||
}
|
||||
return dict;
|
||||
}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Behavioural coverage for <see cref="TableHardwareRejectionNotificationTracker"/> — the dedup
|
||||
/// store behind the two distress-driven bell notifications (hardware rejection per model,
|
||||
/// TPM-PSS incompatibility per serial). <see cref="HardwareRejectionNotificationTrackerKeyTests"/>
|
||||
/// pins the RowKey shape; this file pins what the tracker DOES with those keys.
|
||||
///
|
||||
/// CORRECTNESS GUARD: the "one bell ever" promise rests entirely on the 409 branch of
|
||||
/// <c>AddEntityAsync</c> — a rewrite to Upsert or a get-then-add would silently start ringing
|
||||
/// the bell on every distress report (a device with an incompatible TPM retries forever, so
|
||||
/// "every report" means a notification storm, not a second bell).
|
||||
///
|
||||
/// The retention sweep is covered here too because <c>DeleteOlderThanAsync</c> prunes BOTH key
|
||||
/// spaces from the shared table: pruning a <c>tpmpss|</c> row re-arms that device's bell.
|
||||
/// </summary>
|
||||
public class HardwareRejectionNotificationTrackerDedupTests
|
||||
{
|
||||
private const string TenantId = "77777777-7777-7777-7777-777777777777";
|
||||
private const string OtherTenantId = "88888888-8888-8888-8888-888888888888";
|
||||
private const string Serial = "S4SQ8685";
|
||||
|
||||
// ── TPM PSS: first bell, then never again ──────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TpmPss_FirstReport_RegistersRowAndReturnsTrue()
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
var isFirst = await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial);
|
||||
|
||||
Assert.True(isFirst);
|
||||
var entity = Assert.Single(harness.Store.Values);
|
||||
Assert.Equal(TenantId.ToLowerInvariant(), entity.PartitionKey);
|
||||
Assert.Equal("tpmpss|s4sq8685", entity.RowKey);
|
||||
// Serialization guard: the row must carry the fields the cleanup sweep and any future
|
||||
// operator query rely on — losing FirstNotifiedAt would make the row immortal.
|
||||
Assert.Equal(TenantId, entity.GetString("TenantId"));
|
||||
Assert.Equal(Serial, entity.GetString("SerialNumber"));
|
||||
Assert.NotNull(entity.GetDateTime("FirstNotifiedAt"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TpmPss_SecondReport_SameSerial_ReturnsFalse()
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.False(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.Single(harness.Store);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("s4sq8685")]
|
||||
[InlineData(" S4SQ8685 ")]
|
||||
[InlineData("S4Sq8685")]
|
||||
public async Task TpmPss_SecondReport_CasingOrWhitespaceVariant_StillDeduped(string variant)
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.False(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, variant));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TpmPss_DifferentSerial_OrDifferentTenant_RingsAgain()
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, "OTHER123"));
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(OtherTenantId, Serial));
|
||||
Assert.Equal(3, harness.Store.Count);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("", Serial)]
|
||||
[InlineData(" ", Serial)]
|
||||
[InlineData(TenantId, "")]
|
||||
[InlineData(TenantId, " ")]
|
||||
public async Task TpmPss_BlankInput_ReturnsFalse_WithoutTouchingTheTable(string tenantId, string serial)
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
Assert.False(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(tenantId, serial));
|
||||
Assert.Empty(harness.Store);
|
||||
harness.Table.Verify(
|
||||
c => c.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TpmPss_UnexpectedTableFailure_ReturnsFalse()
|
||||
{
|
||||
// Fail-soft in the SUPPRESSING direction: the write may or may not have landed, so the
|
||||
// caller must not fire a bell it might fire again on the next report.
|
||||
var harness = new Harness { AddFailure = new RequestFailedException(503, "ServerBusy") };
|
||||
|
||||
Assert.False(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
}
|
||||
|
||||
// ── Hardware rejection: same contract, disjoint key space ──────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task HardwareRejection_SecondReport_SameModel_ReturnsFalse()
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
Assert.True(await harness.Sut.TryRegisterFirstNotificationAsync(TenantId, "Lenovo", "ThinkPad X1"));
|
||||
Assert.False(await harness.Sut.TryRegisterFirstNotificationAsync(TenantId, "LENOVO", " thinkpad x1 "));
|
||||
Assert.Single(harness.Store);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HardwareRejection_AndTpmPss_DoNotDedupEachOtherInTheSharedTable()
|
||||
{
|
||||
var harness = new Harness();
|
||||
|
||||
Assert.True(await harness.Sut.TryRegisterFirstNotificationAsync(TenantId, "Lenovo", Serial));
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.Equal(2, harness.Store.Count);
|
||||
}
|
||||
|
||||
// ── F3 rule regression: one bell per episode, keyspaces stay independent ──────
|
||||
|
||||
[Fact]
|
||||
public async Task RuleRegression_SecondRegister_SameRule_ReturnsFalse_OneBellPerEpisode()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var alert = new AutopilotMonitor.Shared.Models.RuleRegressionAlert
|
||||
{
|
||||
TenantId = TenantId, RuleId = "ANALYZE-NET-001",
|
||||
FirstNotifiedAt = DateTime.UtcNow, LastEvaluatedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
Assert.True(await harness.Sut.TryRegisterRuleRegressionAsync(TenantId, alert));
|
||||
Assert.False(await harness.Sut.TryRegisterRuleRegressionAsync(TenantId, alert));
|
||||
|
||||
var entity = Assert.Single(harness.Store.Values);
|
||||
Assert.Equal("ruleregression|analyze-net-001", entity.RowKey);
|
||||
Assert.NotNull(entity.GetDateTime("FirstNotifiedAt")); // the retention sweep depends on it
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RuleRegression_DeleteReArms_AndDoesNotTouchOtherKeyspaces()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var alert = new AutopilotMonitor.Shared.Models.RuleRegressionAlert
|
||||
{
|
||||
TenantId = TenantId, RuleId = "ANALYZE-NET-001",
|
||||
FirstNotifiedAt = DateTime.UtcNow, LastEvaluatedAt = DateTime.UtcNow,
|
||||
};
|
||||
Assert.True(await harness.Sut.TryRegisterRuleRegressionAsync(TenantId, alert));
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
|
||||
await harness.Sut.DeleteRuleRegressionAsync(TenantId, "ANALYZE-NET-001");
|
||||
|
||||
// The episode re-arms; the unrelated tpmpss row survives.
|
||||
Assert.Equal("tpmpss|s4sq8685", Assert.Single(harness.Store.Values).RowKey);
|
||||
Assert.True(await harness.Sut.TryRegisterRuleRegressionAsync(TenantId, alert));
|
||||
}
|
||||
|
||||
// ── Retention sweep ───────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteOlderThanAsync_PrunesRuleRegressionRows_ReArmingLongBurningEpisodes()
|
||||
{
|
||||
var harness = new Harness();
|
||||
Assert.True(await harness.Sut.TryRegisterRuleRegressionAsync(TenantId,
|
||||
new AutopilotMonitor.Shared.Models.RuleRegressionAlert
|
||||
{
|
||||
TenantId = TenantId, RuleId = "ANALYZE-NET-001",
|
||||
FirstNotifiedAt = DateTime.UtcNow, LastEvaluatedAt = DateTime.UtcNow,
|
||||
}));
|
||||
harness.AgeRows(TimeSpan.FromDays(31));
|
||||
|
||||
// Spec §F3: retention cleanup re-arms — a month-old still-burning regression rings again.
|
||||
Assert.Equal(1, await harness.Sut.DeleteOlderThanAsync(DateTime.UtcNow.AddDays(-30)));
|
||||
Assert.Empty(harness.Store);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteOlderThanAsync_PrunesTpmPssRows_AndReArmsThatDevicesBell()
|
||||
{
|
||||
var harness = new Harness();
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
harness.AgeRows(TimeSpan.FromDays(31));
|
||||
|
||||
var deleted = await harness.Sut.DeleteOlderThanAsync(DateTime.UtcNow.AddDays(-30));
|
||||
|
||||
Assert.Equal(1, deleted);
|
||||
Assert.Empty(harness.Store);
|
||||
// A device whose TPM was never fixed keeps reporting; after the retention window it is
|
||||
// worth one more bell rather than permanent silence.
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteOlderThanAsync_KeepsRowsInsideTheWindow_DedupSurvives()
|
||||
{
|
||||
var harness = new Harness();
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.True(await harness.Sut.TryRegisterFirstNotificationAsync(TenantId, "Lenovo", "ThinkPad X1"));
|
||||
|
||||
var deleted = await harness.Sut.DeleteOlderThanAsync(DateTime.UtcNow.AddDays(-30));
|
||||
|
||||
Assert.Equal(0, deleted);
|
||||
Assert.Equal(2, harness.Store.Count);
|
||||
Assert.False(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.False(await harness.Sut.TryRegisterFirstNotificationAsync(TenantId, "Lenovo", "ThinkPad X1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteOlderThanAsync_FiltersOnFirstNotifiedAt_NotTimestamp()
|
||||
{
|
||||
// Rows are insert-once, so FirstNotifiedAt is the creation time. Timestamp would look
|
||||
// equivalent but is rewritten by a table restore/copy, which would silently re-arm every
|
||||
// bell in the tenant at once.
|
||||
var harness = new Harness();
|
||||
var cutoff = new DateTime(2026, 7, 25, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await harness.Sut.DeleteOlderThanAsync(cutoff);
|
||||
|
||||
var filter = Assert.Single(harness.QueryFilters);
|
||||
Assert.Equal("FirstNotifiedAt lt datetime'2026-07-25T12:00:00Z'", filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteOlderThanAsync_ContinuesAfterASingleRowFailure()
|
||||
{
|
||||
var harness = new Harness();
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, Serial));
|
||||
Assert.True(await harness.Sut.TryRegisterFirstTpmPssNotificationAsync(TenantId, "OTHER123"));
|
||||
harness.AgeRows(TimeSpan.FromDays(31));
|
||||
harness.FailDeleteForRowKeys.Add("tpmpss|s4sq8685");
|
||||
|
||||
var deleted = await harness.Sut.DeleteOlderThanAsync(DateTime.UtcNow.AddDays(-30));
|
||||
|
||||
Assert.Equal(1, deleted);
|
||||
Assert.Equal("tpmpss|s4sq8685", Assert.Single(harness.Store.Values).RowKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteOlderThanAsync_QueryFailure_ReturnsZero()
|
||||
{
|
||||
var harness = new Harness { QueryFailure = new RequestFailedException(500, "Boom") };
|
||||
|
||||
Assert.Equal(0, await harness.Sut.DeleteOlderThanAsync(DateTime.UtcNow.AddDays(-30)));
|
||||
}
|
||||
|
||||
// ── Harness ───────────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
private static readonly Regex CutoffFilter =
|
||||
new(@"^FirstNotifiedAt lt datetime'(?<cutoff>[^']+)'$", RegexOptions.Compiled);
|
||||
|
||||
public TableHardwareRejectionNotificationTracker Sut { get; }
|
||||
public Mock<TableClient> Table { get; }
|
||||
public Dictionary<(string Pk, string Rk), TableEntity> Store { get; } = new();
|
||||
public List<string> QueryFilters { get; } = new();
|
||||
public HashSet<string> FailDeleteForRowKeys { get; } = new();
|
||||
|
||||
/// <summary>Non-409 failure injected into AddEntityAsync (fail-soft path).</summary>
|
||||
public Exception? AddFailure { get; init; }
|
||||
|
||||
/// <summary>Failure injected into QueryAsync enumeration (sweep fail-soft path).</summary>
|
||||
public Exception? QueryFailure { get; init; }
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Table = new Mock<TableClient>();
|
||||
|
||||
Table.Setup(c => c.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<TableEntity, CancellationToken>((e, _) =>
|
||||
{
|
||||
if (AddFailure != null) throw AddFailure;
|
||||
if (Store.ContainsKey((e.PartitionKey, e.RowKey)))
|
||||
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
|
||||
Store[(e.PartitionKey, e.RowKey)] = e;
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
Table.Setup(c => c.QueryAsync<TableEntity>(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<int?>(),
|
||||
It.IsAny<IEnumerable<string>>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<string, int?, IEnumerable<string>, CancellationToken>((filter, _, _, _) =>
|
||||
{
|
||||
if (QueryFailure != null) throw QueryFailure;
|
||||
QueryFilters.Add(filter);
|
||||
var rows = Store.Values.Where(e => MatchesCutoff(filter, e)).ToList();
|
||||
return AsyncPageable<TableEntity>.FromPages(new[]
|
||||
{
|
||||
Page<TableEntity>.FromValues(rows, null, new Mock<Response>().Object),
|
||||
});
|
||||
});
|
||||
|
||||
Table.Setup(c => c.DeleteEntityAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<ETag>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, ETag, CancellationToken>((pk, rk, _, _) =>
|
||||
{
|
||||
if (FailDeleteForRowKeys.Contains(rk))
|
||||
throw new RequestFailedException(500, "DeleteFailed");
|
||||
Store.Remove((pk, rk));
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
var serviceClient = new Mock<TableServiceClient>();
|
||||
serviceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(Table.Object);
|
||||
|
||||
Sut = new TableHardwareRejectionNotificationTracker(
|
||||
new TableStorageService(serviceClient.Object, NullLogger<TableStorageService>.Instance),
|
||||
NullLogger<TableHardwareRejectionNotificationTracker>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>Backdates every stored row so a retention sweep sees it as expired.</summary>
|
||||
public void AgeRows(TimeSpan age)
|
||||
{
|
||||
foreach (var entity in Store.Values)
|
||||
entity["FirstNotifiedAt"] = DateTime.UtcNow - age;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal server-side emulation of the OData filter the tracker builds. An unrecognised
|
||||
/// filter matches nothing, so a rewrite that no longer selects expired rows fails loudly
|
||||
/// here instead of leaving dead rows in production.
|
||||
/// </summary>
|
||||
private static bool MatchesCutoff(string filter, TableEntity entity)
|
||||
{
|
||||
var match = CutoffFilter.Match(filter ?? string.Empty);
|
||||
if (!match.Success) return false;
|
||||
|
||||
var cutoff = DateTime.Parse(
|
||||
match.Groups["cutoff"].Value,
|
||||
null,
|
||||
System.Globalization.DateTimeStyles.AdjustToUniversal | System.Globalization.DateTimeStyles.AssumeUniversal);
|
||||
var firstNotified = entity.GetDateTime("FirstNotifiedAt");
|
||||
return firstNotified.HasValue && firstNotified.Value.ToUniversalTime() < cutoff;
|
||||
}
|
||||
}
|
||||
}
|
||||
+31
@@ -59,4 +59,35 @@ public class HardwareRejectionNotificationTrackerKeyTests
|
||||
var key = TableHardwareRejectionNotificationTracker.BuildRowKey("", "");
|
||||
Assert.Equal("|", key);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// TPM PSS row keys (same table, "tpmpss|" prefix — dedup per serial)
|
||||
// =========================================================================
|
||||
|
||||
[Fact]
|
||||
public void BuildTpmPssRowKey_LowercasesTrimsAndPrefixes()
|
||||
{
|
||||
var key = TableHardwareRejectionNotificationTracker.BuildTpmPssRowKey(" S4SQ8685 ");
|
||||
Assert.Equal("tpmpss|s4sq8685", key);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("S4SQ8685", "s4sq8685")]
|
||||
[InlineData("s4Sq8685", "S4SQ8685")]
|
||||
public void BuildTpmPssRowKey_IsCaseInsensitive(string serialA, string serialB)
|
||||
{
|
||||
Assert.Equal(
|
||||
TableHardwareRejectionNotificationTracker.BuildTpmPssRowKey(serialA),
|
||||
TableHardwareRejectionNotificationTracker.BuildTpmPssRowKey(serialB));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTpmPssRowKey_DoesNotCollideWithHardwareKeyOfSameText()
|
||||
{
|
||||
// A hardware key is "{mfr}|{model}"; the TPM key's fixed "tpmpss|" prefix keeps the
|
||||
// two key spaces disjoint within the shared table.
|
||||
var hardwareKey = TableHardwareRejectionNotificationTracker.BuildRowKey("Lenovo", "S4SQ8685");
|
||||
var tpmKey = TableHardwareRejectionNotificationTracker.BuildTpmPssRowKey("S4SQ8685");
|
||||
Assert.NotEqual(hardwareKey, tpmKey);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,9 +4,12 @@ using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Diagnostics;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Azure.Storage.Sas;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
@@ -172,6 +175,119 @@ public class HostedDiagnosticsBlobServiceTests
|
||||
Assert.StartsWith(TenantB + "/", b.BlobPath);
|
||||
}
|
||||
|
||||
// ── DeleteBySessionPrefixAsync — cascade sweep for multi-package sessions ──────
|
||||
|
||||
private const string SessionA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid", SessionA)]
|
||||
[InlineData(TenantA, "not-a-guid")]
|
||||
public async Task DeleteBySessionPrefix_RejectsNonGuidInputs(string tenantId, string sessionId)
|
||||
{
|
||||
var svc = new PrefixDeleteFake(containerExists: true);
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => svc.DeleteBySessionPrefixAsync(tenantId, sessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteBySessionPrefix_MissingContainer_IsNoOp()
|
||||
{
|
||||
var svc = new PrefixDeleteFake(containerExists: false);
|
||||
var deleted = await svc.DeleteBySessionPrefixAsync(TenantA, SessionA);
|
||||
Assert.Equal(0, deleted);
|
||||
Assert.Null(svc.CapturedPrefix);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteBySessionPrefix_UsesSessionScopedPrefix_AndDeletesEveryMatch()
|
||||
{
|
||||
// Two packages for the session: an on-demand (server-requested) one and the terminal
|
||||
// one. The sweep must issue a delete for BOTH even though the Sessions row (and thus
|
||||
// the deletion manifest) only references the last.
|
||||
var svc = new PrefixDeleteFake(
|
||||
containerExists: true,
|
||||
blobNames: new[]
|
||||
{
|
||||
$"{TenantA}/AgentDiagnostics-{SessionA}-20260724T100000-server-requested.zip",
|
||||
$"{TenantA}/AgentDiagnostics-{SessionA}-20260724T113000.zip",
|
||||
});
|
||||
|
||||
var deleted = await svc.DeleteBySessionPrefixAsync(TenantA, SessionA);
|
||||
|
||||
Assert.Equal(2, deleted);
|
||||
Assert.Equal($"{TenantA}/AgentDiagnostics-{SessionA}-", svc.CapturedPrefix);
|
||||
Assert.Equal(2, svc.DeletedBlobNames.Count);
|
||||
Assert.All(svc.DeletedBlobNames, n => Assert.StartsWith(svc.CapturedPrefix!, n));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteBySessionPrefix_NoMatches_ReturnsZero()
|
||||
{
|
||||
var svc = new PrefixDeleteFake(containerExists: true, blobNames: Array.Empty<string>());
|
||||
var deleted = await svc.DeleteBySessionPrefixAsync(TenantA, SessionA);
|
||||
Assert.Equal(0, deleted);
|
||||
Assert.Equal($"{TenantA}/AgentDiagnostics-{SessionA}-", svc.CapturedPrefix);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Overrides <c>GetContainerClient</c> with a Moq'd <see cref="BlobContainerClient"/> whose
|
||||
/// enumeration returns a canned page for the CAPTURED prefix. Prefix filtering itself is
|
||||
/// the Azure SDK's contract — these tests pin what prefix we ask for and that every
|
||||
/// returned item is deleted.
|
||||
/// </summary>
|
||||
private sealed class PrefixDeleteFake : HostedDiagnosticsBlobService
|
||||
{
|
||||
private readonly Mock<BlobContainerClient> _container;
|
||||
|
||||
public string? CapturedPrefix { get; private set; }
|
||||
public List<string> DeletedBlobNames { get; } = new();
|
||||
|
||||
public PrefixDeleteFake(bool containerExists, string[]? blobNames = null)
|
||||
: base(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<HostedDiagnosticsBlobService>.Instance,
|
||||
usesManagedIdentity: false)
|
||||
{
|
||||
_container = new Mock<BlobContainerClient>();
|
||||
_container
|
||||
.Setup(c => c.ExistsAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Response.FromValue(containerExists, Mock.Of<Response>()));
|
||||
|
||||
_container
|
||||
.Setup(c => c.GetBlobsAsync(
|
||||
It.IsAny<BlobTraits>(), It.IsAny<BlobStates>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((BlobTraits _, BlobStates _, string? prefix, CancellationToken _) =>
|
||||
{
|
||||
CapturedPrefix = prefix;
|
||||
var items = (blobNames ?? Array.Empty<string>())
|
||||
.Select(n => BlobsModelFactory.BlobItem(name: n))
|
||||
.ToList();
|
||||
return AsyncPageable<BlobItem>.FromPages(new[]
|
||||
{
|
||||
Page<BlobItem>.FromValues(items, continuationToken: null, Mock.Of<Response>()),
|
||||
});
|
||||
});
|
||||
|
||||
_container
|
||||
.Setup(c => c.GetBlobClient(It.IsAny<string>()))
|
||||
.Returns((string name) =>
|
||||
{
|
||||
var blob = new Mock<BlobClient>();
|
||||
blob
|
||||
.Setup(b => b.DeleteIfExistsAsync(
|
||||
It.IsAny<DeleteSnapshotsOption>(), It.IsAny<BlobRequestConditions>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() =>
|
||||
{
|
||||
DeletedBlobNames.Add(name);
|
||||
return Response.FromValue(true, Mock.Of<Response>());
|
||||
});
|
||||
return blob.Object;
|
||||
});
|
||||
}
|
||||
|
||||
protected override BlobContainerClient GetContainerClient() => _container.Object;
|
||||
}
|
||||
|
||||
// ── Constants ───────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the per-session API-latency projection (2026-07-28): the V2 agent emits cumulative
|
||||
/// counters <c>net_total_latency_ms</c> / <c>net_total_requests</c> in every
|
||||
/// <c>agent_metrics_snapshot</c>; ingest derives the session-wide average from the LAST
|
||||
/// snapshot of the batch and persists it via <c>UpdateSessionNetworkLatencyAsync</c>
|
||||
/// (last-write-wins on cumulative counters → replay-idempotent).
|
||||
/// Wire-format note: the ingest body is parsed with Newtonsoft into
|
||||
/// <c>Dictionary<string, object></c>, so integer JSON numbers arrive as boxed
|
||||
/// <c>long</c> — fixtures mirror that, not the agent's in-memory types.
|
||||
/// </summary>
|
||||
public class IngestApiLatencyProjectionTests
|
||||
{
|
||||
private static EnrollmentEvent Snapshot(object? totalLatencyMs, object? totalRequests)
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = "agent_metrics_snapshot",
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
["agent_cpu_percent"] = 3.62,
|
||||
["net_requests"] = 50L,
|
||||
["net_avg_latency_ms"] = 513.0,
|
||||
},
|
||||
};
|
||||
if (totalLatencyMs != null) evt.Data["net_total_latency_ms"] = totalLatencyMs;
|
||||
if (totalRequests != null) evt.Data["net_total_requests"] = totalRequests;
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CumulativeCounters_YieldSessionAverage()
|
||||
{
|
||||
var events = new List<EnrollmentEvent> { Snapshot(51_300L, 100L) };
|
||||
|
||||
Assert.True(EventIngestProcessor.TryComputeSessionApiLatency(events, out var avg, out var count));
|
||||
Assert.Equal(513.0, avg);
|
||||
Assert.Equal(100, count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AverageIsRounded_ToOneDecimal()
|
||||
{
|
||||
var events = new List<EnrollmentEvent> { Snapshot(1_000L, 3L) };
|
||||
|
||||
Assert.True(EventIngestProcessor.TryComputeSessionApiLatency(events, out var avg, out _));
|
||||
Assert.Equal(333.3, avg);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LastSnapshotOfBatch_Wins()
|
||||
{
|
||||
// Cumulative counters grow monotonically — the last snapshot carries the most
|
||||
// complete session average.
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Snapshot(51_300L, 100L),
|
||||
new EnrollmentEvent { EventType = "download_progress" },
|
||||
Snapshot(120_000L, 200L),
|
||||
};
|
||||
|
||||
Assert.True(EventIngestProcessor.TryComputeSessionApiLatency(events, out var avg, out var count));
|
||||
Assert.Equal(600.0, avg);
|
||||
Assert.Equal(200, count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OldAgentSnapshot_WithoutTotalLatencyField_DoesNotMatch()
|
||||
{
|
||||
// Agents predating net_total_latency_ms emit net_total_requests only.
|
||||
var events = new List<EnrollmentEvent> { Snapshot(totalLatencyMs: null, totalRequests: 100L) };
|
||||
|
||||
Assert.False(EventIngestProcessor.TryComputeSessionApiLatency(events, out _, out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SnapshotWithLatencyButMissingRequests_DoesNotMatch()
|
||||
{
|
||||
var events = new List<EnrollmentEvent> { Snapshot(51_300L, totalRequests: null) };
|
||||
|
||||
Assert.False(EventIngestProcessor.TryComputeSessionApiLatency(events, out _, out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroRequests_DoesNotMatch()
|
||||
{
|
||||
// No requests yet (first snapshot before the first upload completes) — no division.
|
||||
var events = new List<EnrollmentEvent> { Snapshot(0L, 0L) };
|
||||
|
||||
Assert.False(EventIngestProcessor.TryComputeSessionApiLatency(events, out _, out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MatchingLastSnapshot_SkipsOlderAgentSnapshotAfterIt()
|
||||
{
|
||||
// A field-less (old-schema) snapshot later in the batch must not shadow an earlier
|
||||
// one that carries the counters — the predicate selects the last MATCHING snapshot.
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Snapshot(51_300L, 100L),
|
||||
Snapshot(totalLatencyMs: null, totalRequests: 150L),
|
||||
};
|
||||
|
||||
Assert.True(EventIngestProcessor.TryComputeSessionApiLatency(events, out var avg, out var count));
|
||||
Assert.Equal(513.0, avg);
|
||||
Assert.Equal(100, count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BatchWithoutSnapshots_DoesNotMatch()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
new EnrollmentEvent { EventType = "esp_phase_changed" },
|
||||
new EnrollmentEvent { EventType = "app_install_started" },
|
||||
};
|
||||
|
||||
Assert.False(EventIngestProcessor.TryComputeSessionApiLatency(events, out _, out _));
|
||||
}
|
||||
|
||||
// -------- TryGetDouble coercion ladder (mirrors PlatformMetricsService.GetDouble) --------
|
||||
|
||||
[Theory]
|
||||
[InlineData(513.5)] // double — decimal JSON number
|
||||
[InlineData(513)] // int — in-memory constructed events (tests, V1 paths)
|
||||
[InlineData(513L)] // long — Newtonsoft wire form for integer JSON numbers
|
||||
[InlineData(513f)] // float
|
||||
public void TryGetDouble_CoercesBoxedNumericPrimitives(object boxed)
|
||||
{
|
||||
var data = new Dictionary<string, object> { ["k"] = boxed };
|
||||
|
||||
Assert.True(EventIngestProcessor.TryGetDouble(data, "k", out var value));
|
||||
Assert.Equal(Convert.ToDouble(boxed), value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryGetDouble_ParsesStrings_InvariantCulture()
|
||||
{
|
||||
var data = new Dictionary<string, object> { ["k"] = "513.5" };
|
||||
|
||||
Assert.True(EventIngestProcessor.TryGetDouble(data, "k", out var value));
|
||||
Assert.Equal(513.5, value);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-number")]
|
||||
[InlineData("")]
|
||||
public void TryGetDouble_RejectsUnparseableStrings(string raw)
|
||||
{
|
||||
var data = new Dictionary<string, object> { ["k"] = raw };
|
||||
|
||||
Assert.False(EventIngestProcessor.TryGetDouble(data, "k", out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryGetDouble_MissingKey_ReturnsFalse()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.TryGetDouble(new Dictionary<string, object>(), "k", out _));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Session eaf3d8c4 (2026-07-23): a previous enrollment's IME log surviving on disk made the
|
||||
/// agent replay a week-old script history — 156 replayed <c>script_completed</c> events
|
||||
/// inflated the session's RemediationScriptCount. Newer agents suppress the replay at the
|
||||
/// source; this predicate covers the fleet of not-yet-updated agents by skipping the script
|
||||
/// counter increment for events whose <c>rejectedSourceTimestamp</c> (the source log line's
|
||||
/// own time, preserved by the agent's staleness clamp) is more than 24 h older than the
|
||||
/// event's clock-clamped stamp.
|
||||
/// </summary>
|
||||
public class IngestHistoricImeReplayTests
|
||||
{
|
||||
private static readonly DateTime EventStamp = new(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
private static EnrollmentEvent ScriptEvent(object? rejectedSourceTimestamp, bool includeKey = true)
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = "script_completed",
|
||||
Timestamp = EventStamp,
|
||||
Data = new Dictionary<string, object> { ["scriptType"] = "remediation" },
|
||||
};
|
||||
if (includeKey)
|
||||
evt.Data["rejectedSourceTimestamp"] = rejectedSourceTimestamp!;
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleRejectedSourceTimestamp_IsHistoricReplay()
|
||||
{
|
||||
var evt = ScriptEvent(EventStamp.AddDays(-7).ToString("o"));
|
||||
Assert.True(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleAppInstallEvent_IsHistoricReplay()
|
||||
{
|
||||
// The predicate is event-type-agnostic — it also guards AggregateAppInstallEvent
|
||||
// (AppInstallSummaries would otherwise be overwritten with week-old runs).
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = "app_install_completed",
|
||||
Timestamp = EventStamp,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
["appName"] = "Contoso App",
|
||||
["rejectedSourceTimestamp"] = EventStamp.AddDays(-7).ToString("o"),
|
||||
},
|
||||
};
|
||||
Assert.True(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JustOver24h_IsHistoricReplay()
|
||||
{
|
||||
var evt = ScriptEvent(EventStamp.AddHours(-25).ToString("o"));
|
||||
Assert.True(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Within24h_IsNotHistoricReplay()
|
||||
{
|
||||
var evt = ScriptEvent(EventStamp.AddHours(-23).ToString("o"));
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FutureSkewRejection_IsNotHistoricReplay()
|
||||
{
|
||||
// A rejected source timestamp AHEAD of the event stamp is a mid-enrollment clock
|
||||
// jump (WhiteGlove +1h CMTrace skew), not replayed history.
|
||||
var evt = ScriptEvent(EventStamp.AddHours(2).ToString("o"));
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingKey_IsNotHistoricReplay()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(ScriptEvent(null, includeKey: false)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullData_IsNotHistoricReplay()
|
||||
{
|
||||
var evt = new EnrollmentEvent { EventType = "script_completed", Timestamp = EventStamp, Data = null! };
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("not-a-date")]
|
||||
public void UnparseableValue_IsNotHistoricReplay(string raw)
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(ScriptEvent(raw)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullEvent_IsNotHistoricReplay()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(null));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Session 8e4cc4ae (2026-07-24): two on-demand ("Collect Logs") uploads landed in hosted
|
||||
/// storage, but the portal never flipped to Download — the agent's ServerActionDispatcher
|
||||
/// confirms the upload via <c>server_action_executed(actionType=request_diagnostics,
|
||||
/// blobName=…)</c> and emits NO <c>diagnostics_uploaded</c> (only the terminal path does),
|
||||
/// so the Sessions row was never stamped. These tests pin the classification predicate and
|
||||
/// the destination inference that close that gap backend-side (no agent release needed).
|
||||
/// </summary>
|
||||
public class IngestOnDemandDiagnosticsTests
|
||||
{
|
||||
private const string TenantId = "b54dc1af-5320-4f60-b5d4-821e0cf2a359";
|
||||
private const string HostedBlob =
|
||||
TenantId + "/AgentDiagnostics-8e4cc8ae-1111-2222-3333-444455556666-20260724170902-server-requested.zip";
|
||||
|
||||
private static EnrollmentEvent Evt(
|
||||
string eventType = "server_action_executed",
|
||||
string? actionType = "request_diagnostics",
|
||||
string? blobName = HostedBlob)
|
||||
{
|
||||
var data = new Dictionary<string, object>();
|
||||
if (actionType != null) data["actionType"] = actionType;
|
||||
if (blobName != null) data["blobName"] = blobName;
|
||||
return new EnrollmentEvent { EventType = eventType, Data = data };
|
||||
}
|
||||
|
||||
// -------- IsOnDemandDiagnosticsUploadConfirmation --------
|
||||
|
||||
[Fact]
|
||||
public void ExecutedRequestDiagnosticsWithBlobName_IsConfirmation()
|
||||
{
|
||||
Assert.True(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(Evt()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActionTypeIsCaseInsensitive()
|
||||
{
|
||||
Assert.True(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(
|
||||
Evt(actionType: "Request_Diagnostics")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("server_action_received")]
|
||||
[InlineData("server_action_failed")]
|
||||
[InlineData("diagnostics_uploaded")]
|
||||
public void OtherEventTypes_AreNotConfirmations(string eventType)
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(Evt(eventType: eventType)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("rotate_config")]
|
||||
[InlineData("terminate_session")]
|
||||
[InlineData(null)]
|
||||
public void OtherActionTypes_AreNotConfirmations(string? actionType)
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(Evt(actionType: actionType)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
public void MissingOrEmptyBlobName_IsNotAConfirmation(string? blobName)
|
||||
{
|
||||
// A failed upload never reaches server_action_executed, but an executed event without
|
||||
// blobName (defensive) must not wipe/stamp the row either.
|
||||
Assert.False(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(Evt(blobName: blobName)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullEventOrData_IsNotAConfirmation()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(null));
|
||||
// Data is non-nullable on the model but the JSON deserializer can still hand us null
|
||||
// for a payload without "data" — null! pins the guard the predicate actually has.
|
||||
Assert.False(EventIngestProcessor.IsOnDemandDiagnosticsUploadConfirmation(
|
||||
new EnrollmentEvent { EventType = "server_action_executed", Data = null! }));
|
||||
}
|
||||
|
||||
// -------- InferDiagnosticsDestination --------
|
||||
|
||||
[Fact]
|
||||
public void ExplicitDestination_AlwaysWins()
|
||||
{
|
||||
Assert.Equal("CustomerSas",
|
||||
EventIngestProcessor.InferDiagnosticsDestination("CustomerSas", HostedBlob, TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TenantPrefixedBlobName_InfersHosted()
|
||||
{
|
||||
Assert.Equal("Hosted",
|
||||
EventIngestProcessor.InferDiagnosticsDestination(null, HostedBlob, TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TenantPrefixMatch_IsCaseInsensitive()
|
||||
{
|
||||
Assert.Equal("Hosted",
|
||||
EventIngestProcessor.InferDiagnosticsDestination(null, HostedBlob.ToUpperInvariant(), TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BareFilename_InfersNothing()
|
||||
{
|
||||
// CustomerSas uploads persist the bare zip filename — repo must leave the
|
||||
// destination column unchanged (read-time default CustomerSas).
|
||||
Assert.Null(EventIngestProcessor.InferDiagnosticsDestination(
|
||||
null, "AgentDiagnostics-x-20260724.zip", TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForeignTenantPrefix_InfersNothing()
|
||||
{
|
||||
Assert.Null(EventIngestProcessor.InferDiagnosticsDestination(
|
||||
null, "99999999-9999-9999-9999-999999999999/x.zip", TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullBlobName_InfersNothing()
|
||||
{
|
||||
Assert.Null(EventIngestProcessor.InferDiagnosticsDestination(null, null, TenantId));
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,138 @@ public class MetricsMathAppPayloadTests
|
||||
Assert.Equal(2, topCode.GetProperty("count").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailureRate_IsOverFinishedInstalls_InProgressExcluded()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "App A", Status = "Failed" },
|
||||
new() { AppName = "App A", Status = "Succeeded", DurationSeconds = 10 },
|
||||
new() { AppName = "App A", Status = "InProgress" },
|
||||
new() { AppName = "App A", Status = "InProgress" },
|
||||
};
|
||||
|
||||
var app = Build(summaries).GetProperty("topFailingApps")[0];
|
||||
|
||||
Assert.Equal(4, app.GetProperty("totalInstalls").GetInt32());
|
||||
// 1 / (1 + 1) = 50% (over finished), NOT 1 / 4 = 25% (over all installs).
|
||||
Assert.Equal(50d, app.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
// ── PR0 (2026-07-26): skip/unmeasured classification ────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Skips_LeaveTheFailureRate_AndAreReportedSeparately()
|
||||
{
|
||||
// 1 Failed + 1 real install + 3 skips ("Update for X" not applicable).
|
||||
// Rate must be 1/(1+1) = 50%, NOT 1/(1+4) = 20% — skips are not attempts.
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "App A", Status = "Failed", TerminalState = "Error" },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 30 },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Skipped" },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Skipped" },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Postponed" },
|
||||
};
|
||||
|
||||
var root = Build(summaries);
|
||||
var app = root.GetProperty("topFailingApps")[0];
|
||||
|
||||
Assert.Equal(50d, app.GetProperty("failureRate").GetDouble());
|
||||
Assert.Equal(1, app.GetProperty("succeeded").GetInt32());
|
||||
Assert.Equal(3, app.GetProperty("skipped").GetInt32());
|
||||
Assert.Equal(0, app.GetProperty("unmeasured").GetInt32());
|
||||
Assert.Equal(3, root.GetProperty("totalSkipped").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroDurations_AreExcludedFromAverages_AndCountedAsUnmeasured()
|
||||
{
|
||||
// Two measured installs (100s, 200s) + one whose start was never observed (audit §0.5:
|
||||
// agent attach window — duration UNKNOWN). avg must be 150, not (100+200+0)/3 = 100.
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 100 },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 200 },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 0 },
|
||||
};
|
||||
|
||||
var root = Build(summaries);
|
||||
Assert.Equal(1, root.GetProperty("totalUnmeasured").GetInt32());
|
||||
|
||||
// Rebuild with a third measured row to clear the sample floor and read the average.
|
||||
summaries.Add(new AppInstallSummary { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 150 });
|
||||
var slowest = Build(summaries).GetProperty("slowestApps");
|
||||
Assert.Equal(1, slowest.GetArrayLength());
|
||||
Assert.Equal(150d, slowest[0].GetProperty("avgDurationSeconds").GetDouble());
|
||||
Assert.Equal(1, slowest[0].GetProperty("unmeasured").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyZeroDurationRows_StayInRate_ButNeverInDurations()
|
||||
{
|
||||
// Rows written before TerminalState existed: cannot be classified as skip vs unmeasured.
|
||||
// They keep the legacy rate behavior (counted as succeeded) and are excluded from
|
||||
// durations (counted as unmeasured — duration genuinely not measured).
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "App A", Status = "Failed" },
|
||||
new() { AppName = "App A", Status = "Succeeded", DurationSeconds = 0 }, // legacy, no TerminalState
|
||||
new() { AppName = "App A", Status = "Succeeded", DurationSeconds = 60, TerminalState = "Installed" },
|
||||
};
|
||||
|
||||
var app = Build(summaries).GetProperty("topFailingApps")[0];
|
||||
|
||||
Assert.Equal(2, app.GetProperty("succeeded").GetInt32()); // legacy row stays in the rate
|
||||
Assert.Equal(33.3d, app.GetProperty("failureRate").GetDouble());
|
||||
Assert.Equal(1, app.GetProperty("unmeasured").GetInt32());
|
||||
Assert.Equal(0, app.GetProperty("skipped").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ImplausibleDurations_AreExcludedFromAverages_AndCountedAsUnmeasured()
|
||||
{
|
||||
// A duration beyond the agent's max observation lifetime is a back-stamped emission
|
||||
// artifact (CompletedAt pinned at session end — production: 60 175 s app rows inside a
|
||||
// 2 697 s session), not a slow install. It must fall out of the averages exactly like
|
||||
// an unobserved start. Boundary: exactly 6 h is still plausible/measured.
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 100 },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 200 },
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed",
|
||||
DurationSeconds = MetricsMath.MaxPlausibleInstallDurationSeconds }, // boundary: measured
|
||||
new() { AppName = "App A", Status = "Succeeded", TerminalState = "Installed",
|
||||
DurationSeconds = MetricsMath.MaxPlausibleInstallDurationSeconds + 1 }, // artifact: unmeasured
|
||||
};
|
||||
|
||||
var root = Build(summaries);
|
||||
Assert.Equal(1, root.GetProperty("totalUnmeasured").GetInt32());
|
||||
|
||||
var slowest = root.GetProperty("slowestApps");
|
||||
Assert.Equal(1, slowest.GetArrayLength());
|
||||
Assert.Equal(3, slowest[0].GetProperty("measuredInstalls").GetInt32());
|
||||
// avg over {100, 200, 21600} — the 21601 artifact contributes nothing.
|
||||
Assert.Equal(7300d, slowest[0].GetProperty("avgDurationSeconds").GetDouble());
|
||||
Assert.Equal(MetricsMath.MaxPlausibleInstallDurationSeconds, slowest[0].GetProperty("maxDurationSeconds").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SlowestApps_GateOnMeasuredInstalls_NotOnSucceededCount()
|
||||
{
|
||||
// 3 succeeded but only 2 measured → below the floor; an app must not rank "fast"
|
||||
// (or at all) on a handful of unobserved zeros.
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "Mostly Unmeasured", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 5 },
|
||||
new() { AppName = "Mostly Unmeasured", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 5 },
|
||||
new() { AppName = "Mostly Unmeasured", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 0 },
|
||||
};
|
||||
|
||||
var slowest = Build(summaries).GetProperty("slowestApps");
|
||||
Assert.Equal(0, slowest.GetArrayLength());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SlowestApps_DropsAppsBelowSampleFloor()
|
||||
{
|
||||
|
||||
@@ -125,5 +125,6 @@ public class OffboardingSessionEnumeratorTests
|
||||
public Task<bool> IsSessionIndexEmptyAsync() => throw new NotImplementedException();
|
||||
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException();
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync() => throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Security.Claims;
|
||||
using AutopilotMonitor.Functions.Functions.Config;
|
||||
using AutopilotMonitor.Functions.Middleware;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
@@ -205,8 +206,101 @@ public class PolicyEnforcementMiddlewareTests
|
||||
Assert.Equal(403, result.StatusCode);
|
||||
}
|
||||
|
||||
// ── config GET (TenantAdminOrGlobalReader): own-tenant admin view vs reader view ──
|
||||
// GetTenantConfigurationFunction redacts only when IsGlobalReader && !(IsTenantAdmin && Target==Tenant).
|
||||
// ── TenantAdminOrOperator (POST sessions/{id}/actions — "Collect Logs") ────────
|
||||
|
||||
private const string SessionA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
|
||||
|
||||
private static void AsTenantMemberRole(Harness h, string tenantId, string upn, string role) =>
|
||||
h.Repo.Setup(r => r.GetTenantMemberAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(new TenantMember { Upn = upn, TenantId = tenantId, Role = role, IsEnabled = true });
|
||||
|
||||
[Fact]
|
||||
public async Task SessionActions_TenantOperator_IsAllowed_WithOperatorRoleOnContext()
|
||||
{
|
||||
const string upn = "operator@contoso.com";
|
||||
var h = BuildHarness();
|
||||
AsTenantMemberRole(h, TenantA, upn, Constants.TenantRoles.Operator);
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"POST", $"/api/sessions/{SessionA}/actions", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
// The function's per-type gate depends on these exact flags: an Operator must arrive
|
||||
// with IsTenantAdmin=false + UserRole=Operator so terminate_session/rotate_config 403.
|
||||
Assert.False(result.Context!.IsTenantAdmin);
|
||||
Assert.Equal(Constants.TenantRoles.Operator, result.Context!.UserRole);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionActions_TenantAdmin_IsAllowed_WithAdminFlag()
|
||||
{
|
||||
const string upn = "admin@contoso.com";
|
||||
var h = BuildHarness();
|
||||
h.AsTenantAdmin(TenantA, upn);
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"POST", $"/api/sessions/{SessionA}/actions", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.True(result.Context!.IsTenantAdmin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionActions_TenantViewer_IsForbidden()
|
||||
{
|
||||
const string upn = "viewer@contoso.com";
|
||||
var h = BuildHarness();
|
||||
AsTenantMemberRole(h, TenantA, upn, Constants.TenantRoles.Viewer);
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"POST", $"/api/sessions/{SessionA}/actions", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.False(result.Allowed);
|
||||
Assert.Equal(403, result.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionActions_NonMember_IsForbidden()
|
||||
{
|
||||
var h = BuildHarness(); // defaults: no membership, no global role
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"POST", $"/api/sessions/{SessionA}/actions", null, AuthedPrincipal(TenantA, "outsider@contoso.com"));
|
||||
|
||||
Assert.False(result.Allowed);
|
||||
Assert.Equal(403, result.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionActions_GlobalAdmin_IsAllowed()
|
||||
{
|
||||
const string upn = "ga@platform.com";
|
||||
var h = BuildHarness();
|
||||
h.AsGlobalRole(Constants.GlobalRoles.GlobalAdmin);
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"POST", $"/api/sessions/{SessionA}/actions", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.True(result.Context!.IsGlobalAdmin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionActions_PureGlobalReader_IsForbidden_WriteTier()
|
||||
{
|
||||
const string upn = "reader@platform.com";
|
||||
var h = BuildHarness();
|
||||
h.AsGlobalRole(Constants.GlobalRoles.GlobalReader);
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"POST", $"/api/sessions/{SessionA}/actions", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.False(result.Allowed);
|
||||
Assert.Equal(403, result.StatusCode);
|
||||
}
|
||||
|
||||
// ── config GET (MemberRead): own-tenant admin view vs redacted reader/member view ──
|
||||
// GetTenantConfigurationFunction redacts unless GA or own-tenant Admin (CanViewSecrets).
|
||||
// These lock in the RequestContext that drives that decision.
|
||||
|
||||
[Fact]
|
||||
@@ -243,6 +337,38 @@ public class PolicyEnforcementMiddlewareTests
|
||||
Assert.Equal(TenantB, rc.TargetTenantId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConfigGet_OwnTenantOperator_IsAllowed_RedactedMemberView()
|
||||
{
|
||||
// Read-only Settings view for troubleshooting staff: an own-tenant Operator is admitted
|
||||
// (MemberRead) but arrives with IsTenantAdmin=false, so CanViewSecrets ⇒ redacted copy.
|
||||
const string upn = "operator@contoso.com";
|
||||
var h = BuildHarness();
|
||||
AsTenantMemberRole(h, TenantA, upn, Constants.TenantRoles.Operator);
|
||||
|
||||
var result = await h.Middleware.DecideAsync("GET", $"/api/config/{TenantA}", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
var rc = result.Context!;
|
||||
Assert.False(rc.IsTenantAdmin);
|
||||
Assert.False(GetTenantConfigurationFunction.CanViewSecrets(rc));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConfigGet_CrossTenantOperator_IsForbidden()
|
||||
{
|
||||
// Membership is resolved against the caller's JWT tenant — an Operator of A naming
|
||||
// tenant B in the route is a cross-tenant read and must stay 403.
|
||||
const string upn = "operator@contoso.com";
|
||||
var h = BuildHarness();
|
||||
AsTenantMemberRole(h, TenantA, upn, Constants.TenantRoles.Operator);
|
||||
|
||||
var result = await h.Middleware.DecideAsync("GET", $"/api/config/{TenantB}", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.False(result.Allowed);
|
||||
Assert.Equal(403, result.StatusCode);
|
||||
}
|
||||
|
||||
// ── Cross-tenant read: GlobalReader crosses tenants (HasGlobalScope gate) ───────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// F3 PR6 (insights spec §F3): the Wilson primitive (hand-computed vectors), the radar's
|
||||
/// detection gates and EVERY suppression branch (table-driven), the re-arm rule, the
|
||||
/// dimension-concentration gates, the tracker keyspace round-trip (tri-states included)
|
||||
/// and the alert wording contracts ("correlated, not necessarily causal").
|
||||
/// </summary>
|
||||
public class RuleRegressionRadarTests
|
||||
{
|
||||
private static readonly DateTime Target = new(2026, 7, 26, 0, 0, 0, DateTimeKind.Utc);
|
||||
private const string TenantA = "aaaaaaaa-0000-0000-0000-000000000001";
|
||||
private const string RuleX = "ANALYZE-NET-001";
|
||||
|
||||
// ── Wilson interval (MetricsMath) — hand-computed 95 % vectors ──────────
|
||||
|
||||
[Theory]
|
||||
[InlineData(10, 20, 0.2993, 0.7007)] // symmetric mid-range
|
||||
[InlineData(0, 20, 0.0000, 0.1611)] // zero successes: lower pinned at 0
|
||||
[InlineData(5, 25, 0.0886, 0.3913)]
|
||||
public void WilsonInterval_MatchesHandComputedVectors(int successes, int trials, double lower, double upper)
|
||||
{
|
||||
var (l, u) = MetricsMath.WilsonInterval(successes, trials);
|
||||
Assert.Equal(lower, l, 3);
|
||||
Assert.Equal(upper, u, 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WilsonInterval_ZeroTrials_IsUninformative_NeverClaimsSeparation()
|
||||
{
|
||||
Assert.Equal((0.0, 1.0), MetricsMath.WilsonInterval(0, 0));
|
||||
// A zero-denominator window can never be "separated" from anything.
|
||||
Assert.False(MetricsMath.RateIncreaseSeparated(0, 0, 10, 500));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RateIncreaseSeparated_DisjointIntervals_True_OverlappingFalse()
|
||||
{
|
||||
// 15 % on n=100 (lower ≈ 0.093) vs 2 % baseline on n=500 (upper ≈ 0.036) → separated.
|
||||
Assert.True(MetricsMath.RateIncreaseSeparated(15, 100, 10, 500));
|
||||
// 5 % on n=100 vs 4 % baseline on n=500 → intervals overlap → noise, not a regression.
|
||||
Assert.False(MetricsMath.RateIncreaseSeparated(5, 100, 20, 500));
|
||||
}
|
||||
|
||||
// ── radar evaluation — gates + suppression branches ─────────────────────
|
||||
|
||||
private static RuleStatsEntry Entry(
|
||||
string date, int fire, int sessions, string ruleId = RuleX, string ruleType = "analyze")
|
||||
=> new()
|
||||
{
|
||||
TenantId = TenantA,
|
||||
Date = date,
|
||||
RuleId = ruleId,
|
||||
RuleType = ruleType,
|
||||
RuleTitle = "Network check",
|
||||
Category = "network",
|
||||
Severity = "warning",
|
||||
FireCount = fire,
|
||||
EvaluationCount = sessions,
|
||||
SessionsEvaluated = sessions,
|
||||
};
|
||||
|
||||
private static string Day(int daysBeforeTarget) => Target.AddDays(-daysBeforeTarget).ToString("yyyy-MM-dd");
|
||||
|
||||
/// <summary>Window fire=15/100 across two days; baseline fire=10/500 across three days — fires all gates.</summary>
|
||||
private static List<RuleStatsEntry> RegressedEntries() => new()
|
||||
{
|
||||
Entry(Day(0), 10, 60),
|
||||
Entry(Day(6), 5, 40), // windowStart boundary — still window
|
||||
Entry(Day(7), 4, 200), // first baseline day
|
||||
Entry(Day(20), 4, 200),
|
||||
Entry(Day(34), 2, 100), // baselineStart boundary — still baseline
|
||||
};
|
||||
|
||||
private static Dictionary<string, RuleRegressionRadar.RuleTimestamps> OldRule(
|
||||
DateTime? createdAt = null, DateTime? updatedAt = null, string ruleId = RuleX)
|
||||
=> new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[ruleId] = new RuleRegressionRadar.RuleTimestamps(
|
||||
createdAt ?? Target.AddDays(-100), updatedAt ?? Target.AddDays(-50)),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_AllGatesPass_FiresWithFullNumbers()
|
||||
{
|
||||
var findings = RuleRegressionRadar.Evaluate(RegressedEntries(), Target, OldRule());
|
||||
|
||||
var finding = Assert.Single(findings);
|
||||
Assert.Equal(RuleX, finding.RuleId);
|
||||
Assert.Equal(15, finding.WindowFireCount);
|
||||
Assert.Equal(100, finding.WindowSessionCount);
|
||||
Assert.Equal(10, finding.BaselineFireCount);
|
||||
Assert.Equal(500, finding.BaselineSessionCount);
|
||||
Assert.Equal(15.0, finding.WindowRatePct);
|
||||
Assert.Equal(2.0, finding.BaselineRatePct);
|
||||
Assert.Equal(7.5, finding.Lift);
|
||||
Assert.Equal(Day(6), finding.WindowStartDate);
|
||||
Assert.Equal(Day(0), finding.WindowEndDate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_ZeroBaselineFires_FiresAsNewSignal_WithNullLift()
|
||||
{
|
||||
var entries = new List<RuleStatsEntry>
|
||||
{
|
||||
Entry(Day(0), 15, 100),
|
||||
Entry(Day(10), 0, 500),
|
||||
};
|
||||
|
||||
var finding = Assert.Single(RuleRegressionRadar.Evaluate(entries, Target, OldRule()));
|
||||
Assert.Null(finding.Lift); // no finite lift is ever invented (truthfulness rule 1)
|
||||
Assert.Equal(0.0, finding.BaselineRatePct);
|
||||
}
|
||||
|
||||
public static TheoryData<string, List<RuleStatsEntry>> SuppressedByCounts() => new()
|
||||
{
|
||||
// Below the ≥5 hit-session gate.
|
||||
{ "below hit gate", new List<RuleStatsEntry> { Entry(Day(0), 4, 100), Entry(Day(10), 1, 500) } },
|
||||
// Below the ≥20 evaluated-session gate.
|
||||
{ "below session gate", new List<RuleStatsEntry> { Entry(Day(0), 5, 19), Entry(Day(10), 1, 500) } },
|
||||
// No baseline denominator at all (rule disabled through the baseline, re-enabled recently).
|
||||
{ "no baseline sessions", new List<RuleStatsEntry> { Entry(Day(0), 15, 100) } },
|
||||
// Lift below 2× (10 % vs 6 %).
|
||||
{ "lift below 2x", new List<RuleStatsEntry> { Entry(Day(0), 10, 100), Entry(Day(10), 30, 500) } },
|
||||
// Lift 2.1× but Wilson intervals overlap (5/20 vs 60/500) — small-n noise never alerts.
|
||||
{ "not separated", new List<RuleStatsEntry> { Entry(Day(0), 5, 20), Entry(Day(10), 60, 500) } },
|
||||
// Gather rows are ignored entirely (audit Q6: per-batch dedup — bad math until fixed).
|
||||
{ "gather ignored", new List<RuleStatsEntry>
|
||||
{ Entry(Day(0), 50, 60, ruleType: "gather"), Entry(Day(10), 1, 500, ruleType: "gather") } },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(SuppressedByCounts))]
|
||||
public void Evaluate_SuppressionBranches_ProduceNoFinding(string branch, List<RuleStatsEntry> entries)
|
||||
{
|
||||
Assert.True(RuleRegressionRadar.Evaluate(entries, Target, OldRule()).Count == 0,
|
||||
$"branch '{branch}' must not fire");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_RuleEntityGates_Suppress()
|
||||
{
|
||||
var entries = RegressedEntries();
|
||||
|
||||
// Deleted rule (no entity) never alerts.
|
||||
Assert.Empty(RuleRegressionRadar.Evaluate(
|
||||
entries, Target, new Dictionary<string, RuleRegressionRadar.RuleTimestamps>()));
|
||||
|
||||
// Grace period: created inside baseline+window (< 34 days before target).
|
||||
Assert.Empty(RuleRegressionRadar.Evaluate(
|
||||
entries, Target, OldRule(createdAt: Target.AddDays(-20))));
|
||||
|
||||
// Edited inside the 7-day window — edits legitimately change hit rates.
|
||||
Assert.Empty(RuleRegressionRadar.Evaluate(
|
||||
entries, Target, OldRule(updatedAt: Target.AddDays(-2))));
|
||||
|
||||
// Edited just BEFORE the window → not suppressed.
|
||||
Assert.Single(RuleRegressionRadar.Evaluate(
|
||||
entries, Target, OldRule(updatedAt: Target.AddDays(-8))));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SumWindows_BoundaryDates_LandInTheRightBucket()
|
||||
{
|
||||
var entries = new List<RuleStatsEntry>
|
||||
{
|
||||
Entry(Target.AddDays(1).ToString("yyyy-MM-dd"), 100, 100), // future — ignored
|
||||
Entry(Day(0), 1, 10),
|
||||
Entry(Day(6), 2, 20), // last window day
|
||||
Entry(Day(7), 4, 40), // first baseline day
|
||||
Entry(Day(34), 8, 80), // last baseline day
|
||||
Entry(Day(35), 100, 100), // beyond horizon — ignored
|
||||
};
|
||||
|
||||
var sums = RuleRegressionRadar.SumWindows(entries, Target);
|
||||
Assert.Equal(3, sums.WindowFire);
|
||||
Assert.Equal(30, sums.WindowSessions);
|
||||
Assert.Equal(12, sums.BaselineFire);
|
||||
Assert.Equal(120, sums.BaselineSessions);
|
||||
}
|
||||
|
||||
// ── re-arm rule ─────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void ShouldReArm_FiresStopped_OrRateBackUnderOnePointFive()
|
||||
{
|
||||
// Nothing fired in the window anymore → re-arm (also covers "no stats rows at all").
|
||||
Assert.True(RuleRegressionRadar.ShouldReArm(new List<RuleStatsEntry>(), Target));
|
||||
Assert.True(RuleRegressionRadar.ShouldReArm(
|
||||
new List<RuleStatsEntry> { Entry(Day(0), 0, 100), Entry(Day(10), 10, 500) }, Target));
|
||||
|
||||
// 2 % window vs 2 % baseline → 1.0× < 1.5 → re-arm.
|
||||
Assert.True(RuleRegressionRadar.ShouldReArm(
|
||||
new List<RuleStatsEntry> { Entry(Day(0), 2, 100), Entry(Day(10), 10, 500) }, Target));
|
||||
|
||||
// 4 % window vs 2 % baseline → 2.0× ≥ 1.5 → still elevated, keep the episode.
|
||||
Assert.False(RuleRegressionRadar.ShouldReArm(
|
||||
new List<RuleStatsEntry> { Entry(Day(0), 4, 100), Entry(Day(10), 10, 500) }, Target));
|
||||
|
||||
// Zero-baseline episode ("new signal"): no rate threshold to fall under — only the
|
||||
// fires-stopped branch re-arms it.
|
||||
Assert.False(RuleRegressionRadar.ShouldReArm(
|
||||
new List<RuleStatsEntry> { Entry(Day(0), 3, 100), Entry(Day(10), 0, 500) }, Target));
|
||||
}
|
||||
|
||||
// ── dimension concentration ─────────────────────────────────────────────
|
||||
|
||||
private static SessionSummary DimSession(string osBuild = "", string model = "", string agentVersion = "")
|
||||
=> new()
|
||||
{
|
||||
TenantId = TenantA,
|
||||
SessionId = Guid.NewGuid().ToString(),
|
||||
SerialNumber = "PF4X1ABC",
|
||||
OsBuild = osBuild,
|
||||
Manufacturer = "Contoso",
|
||||
Model = model,
|
||||
AgentVersion = agentVersion,
|
||||
ImeAgentVersion = "",
|
||||
StartedAt = Target,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void DimensionConcentration_DominantOsBuild_ReportsBothShares()
|
||||
{
|
||||
var all = new List<SessionSummary>();
|
||||
for (var i = 0; i < 20; i++) all.Add(DimSession(osBuild: "26100.4652"));
|
||||
for (var i = 0; i < 80; i++) all.Add(DimSession(osBuild: "22631.3880"));
|
||||
var hits = all.Take(16).Concat(all.Skip(20).Take(4)).ToList(); // 16 on the new build, 4 on the old
|
||||
|
||||
var result = RuleRegressionRadar.ComputeDimensionConcentration(hits, all)!;
|
||||
|
||||
Assert.Equal("osBuild", result.Dimension);
|
||||
Assert.Equal("26100.4652", result.Value);
|
||||
Assert.Equal(16, result.HitCount);
|
||||
Assert.Equal(80.0, result.HitSharePct);
|
||||
Assert.Equal(20.0, result.AllSharePct);
|
||||
Assert.Equal(4.0, result.Lift);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DimensionConcentration_BelowGates_YieldsNull_NeverStretched()
|
||||
{
|
||||
var all = Enumerable.Range(0, 100).Select(_ => DimSession(osBuild: "26100.1")).ToList();
|
||||
|
||||
// Every session shares the build → lift 1.0 < 2 → no claim.
|
||||
Assert.Null(RuleRegressionRadar.ComputeDimensionConcentration(all.Take(10).ToList(), all));
|
||||
|
||||
// Concentrated but only 4 hit sessions (< 5) → no claim.
|
||||
var mixed = Enumerable.Range(0, 96).Select(_ => DimSession(osBuild: "22631.1"))
|
||||
.Concat(Enumerable.Range(0, 4).Select(_ => DimSession(osBuild: "26100.9")))
|
||||
.ToList();
|
||||
var hits4 = mixed.Skip(96).ToList();
|
||||
Assert.Null(RuleRegressionRadar.ComputeDimensionConcentration(hits4, mixed));
|
||||
|
||||
// No hit sessions at all → no claim.
|
||||
Assert.Null(RuleRegressionRadar.ComputeDimensionConcentration(new List<SessionSummary>(), all));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DimensionConcentration_PicksTheHighestLiftAcrossDimensions()
|
||||
{
|
||||
var all = new List<SessionSummary>();
|
||||
for (var i = 0; i < 25; i++) all.Add(DimSession(osBuild: "26100.1", model: "Laptop 5"));
|
||||
for (var i = 0; i < 75; i++) all.Add(DimSession(osBuild: "22631.1", model: i < 25 ? "Laptop 5" : "Laptop 4"));
|
||||
// Hits: 8 of 10 on the new build (lift 80/25 = 3.2); model "Contoso Laptop 5" is 50 % of
|
||||
// fleet and 80 % of hits (lift 1.6 — below gate).
|
||||
var hits = all.Take(8).Concat(all.Skip(25).Take(2)).ToList();
|
||||
|
||||
var result = RuleRegressionRadar.ComputeDimensionConcentration(hits, all)!;
|
||||
Assert.Equal("osBuild", result.Dimension);
|
||||
Assert.Equal(3.2, result.Lift);
|
||||
}
|
||||
|
||||
// ── tracker keyspace (round-trip + key shape) ───────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void RuleRegressionRowKey_PrefixedAndCaseFolded()
|
||||
{
|
||||
Assert.Equal("ruleregression|analyze-net-001",
|
||||
TableHardwareRejectionNotificationTracker.BuildRuleRegressionRowKey(" ANALYZE-NET-001 "));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RuleRegressionEntity_RoundTripsAllFields_IncludingDimension()
|
||||
{
|
||||
var alert = new RuleRegressionAlert
|
||||
{
|
||||
TenantId = TenantA,
|
||||
RuleId = RuleX,
|
||||
RuleTitle = "Network check",
|
||||
WindowFireCount = 15,
|
||||
WindowSessionCount = 100,
|
||||
BaselineFireCount = 10,
|
||||
BaselineSessionCount = 500,
|
||||
WindowRatePct = 15.0,
|
||||
BaselineRatePct = 2.0,
|
||||
Lift = 7.5,
|
||||
WindowStartDate = "2026-07-20",
|
||||
WindowEndDate = "2026-07-26",
|
||||
Dimension = new RuleRegressionDimension
|
||||
{
|
||||
Dimension = "osBuild", Value = "26100.4652",
|
||||
HitCount = 12, HitSharePct = 80.0, AllSharePct = 20.0, Lift = 4.0,
|
||||
},
|
||||
FirstNotifiedAt = Target,
|
||||
LastEvaluatedAt = Target.AddHours(2),
|
||||
};
|
||||
|
||||
var entity = TableHardwareRejectionNotificationTracker.BuildRuleRegressionEntity(TenantA, alert);
|
||||
Assert.Equal(TenantA.ToLowerInvariant(), entity.PartitionKey);
|
||||
Assert.Equal("ruleregression|analyze-net-001", entity.RowKey);
|
||||
|
||||
var mapped = TableHardwareRejectionNotificationTracker.MapToRuleRegressionAlert(entity);
|
||||
Assert.Equal(RuleX, mapped.RuleId);
|
||||
Assert.Equal("Network check", mapped.RuleTitle);
|
||||
Assert.Equal(15, mapped.WindowFireCount);
|
||||
Assert.Equal(100, mapped.WindowSessionCount);
|
||||
Assert.Equal(10, mapped.BaselineFireCount);
|
||||
Assert.Equal(500, mapped.BaselineSessionCount);
|
||||
Assert.Equal(15.0, mapped.WindowRatePct);
|
||||
Assert.Equal(2.0, mapped.BaselineRatePct);
|
||||
Assert.Equal(7.5, mapped.Lift);
|
||||
Assert.Equal("2026-07-20", mapped.WindowStartDate);
|
||||
Assert.Equal("2026-07-26", mapped.WindowEndDate);
|
||||
Assert.Equal(Target, mapped.FirstNotifiedAt);
|
||||
Assert.Equal(Target.AddHours(2), mapped.LastEvaluatedAt);
|
||||
Assert.NotNull(mapped.Dimension);
|
||||
Assert.Equal("osBuild", mapped.Dimension!.Dimension);
|
||||
Assert.Equal(4.0, mapped.Dimension.Lift);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RuleRegressionEntity_TriStates_AbsentColumnsMapToNull_NeverInvented()
|
||||
{
|
||||
var alert = new RuleRegressionAlert
|
||||
{
|
||||
TenantId = TenantA, RuleId = RuleX,
|
||||
Lift = null, // new signal — no finite lift
|
||||
Dimension = null, // no clear concentration
|
||||
FirstNotifiedAt = Target, LastEvaluatedAt = Target,
|
||||
};
|
||||
|
||||
var entity = TableHardwareRejectionNotificationTracker.BuildRuleRegressionEntity(TenantA, alert);
|
||||
Assert.False(entity.ContainsKey("Lift"));
|
||||
Assert.False(entity.ContainsKey("DimensionJson"));
|
||||
|
||||
var mapped = TableHardwareRejectionNotificationTracker.MapToRuleRegressionAlert(entity);
|
||||
Assert.Null(mapped.Lift);
|
||||
Assert.Null(mapped.Dimension);
|
||||
}
|
||||
|
||||
// ── wording contracts ───────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void DescribeDimension_UsesCorrelationWording_NeverCausal()
|
||||
{
|
||||
var text = MaintenanceService.DescribeDimension(new RuleRegressionDimension
|
||||
{
|
||||
Dimension = "osBuild", Value = "26100.4652",
|
||||
HitCount = 12, HitSharePct = 87.0, AllSharePct = 22.0, Lift = 4.0,
|
||||
})!;
|
||||
|
||||
Assert.Contains("87% of affected sessions are on osBuild 26100.4652", text);
|
||||
Assert.Contains("vs 22% of all sessions (lift 4x)", text);
|
||||
Assert.Contains("correlated, not necessarily causal", text);
|
||||
Assert.Null(MaintenanceService.DescribeDimension(null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildRegressionMessage_CarriesFullNumbers_AndHonestNoDimensionFallback()
|
||||
{
|
||||
var finding = new RuleRegressionFinding
|
||||
{
|
||||
TenantId = TenantA, RuleId = RuleX, RuleTitle = "Network check",
|
||||
WindowFireCount = 15, WindowSessionCount = 100, WindowRatePct = 15.0,
|
||||
BaselineFireCount = 10, BaselineSessionCount = 500, BaselineRatePct = 2.0,
|
||||
Lift = 7.5, WindowStartDate = "2026-07-20", WindowEndDate = "2026-07-26",
|
||||
};
|
||||
|
||||
var message = MaintenanceService.BuildRegressionMessage(finding, dimensionSummary: null);
|
||||
Assert.Contains("15 of 100 evaluated sessions", message);
|
||||
Assert.Contains("(15%)", message);
|
||||
Assert.Contains("baseline 2% (10/500 over the prior 28 days)", message);
|
||||
Assert.Contains("lift 7.5x", message);
|
||||
Assert.Contains("No clear dimension concentration.", message);
|
||||
|
||||
var newSignal = MaintenanceService.BuildRegressionMessage(
|
||||
new RuleRegressionFinding
|
||||
{
|
||||
RuleId = RuleX, RuleTitle = "Network check",
|
||||
WindowFireCount = 15, WindowSessionCount = 100, WindowRatePct = 15.0,
|
||||
BaselineFireCount = 0, BaselineSessionCount = 500, BaselineRatePct = 0.0,
|
||||
Lift = null, WindowStartDate = "2026-07-20", WindowEndDate = "2026-07-26",
|
||||
},
|
||||
dimensionSummary: "dim summary");
|
||||
Assert.Contains("new signal", newSignal);
|
||||
Assert.Contains("dim summary.", newSignal);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using AutopilotMonitor.Functions.Functions.Sessions;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
@@ -11,6 +12,35 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// </summary>
|
||||
public class ServerActionQueueTests
|
||||
{
|
||||
// ── Per-type caller gate (route tier is TenantAdminOrOperator; the function must
|
||||
// re-gate admin-only types for Operators) ─────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
// Operator (neither admin flag): only request_diagnostics
|
||||
[InlineData(ServerActionTypes.RequestDiagnostics, false, false, true)]
|
||||
[InlineData(ServerActionTypes.TerminateSession, false, false, false)]
|
||||
[InlineData(ServerActionTypes.RotateConfig, false, false, false)]
|
||||
// Tenant Admin: everything
|
||||
[InlineData(ServerActionTypes.TerminateSession, true, false, true)]
|
||||
[InlineData(ServerActionTypes.RotateConfig, true, false, true)]
|
||||
[InlineData(ServerActionTypes.RequestDiagnostics, true, false, true)]
|
||||
// Global Admin: everything
|
||||
[InlineData(ServerActionTypes.TerminateSession, false, true, true)]
|
||||
[InlineData(ServerActionTypes.RotateConfig, false, true, true)]
|
||||
public void IsTypeAllowedForCaller_Matrix(
|
||||
string actionType, bool isTenantAdmin, bool isGlobalAdmin, bool expected)
|
||||
{
|
||||
Assert.Equal(expected,
|
||||
QueueSessionActionFunction.IsTypeAllowedForCaller(actionType, isTenantAdmin, isGlobalAdmin));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsTypeAllowedForCaller_IsCaseInsensitive_ForOperator()
|
||||
{
|
||||
Assert.True(QueueSessionActionFunction.IsTypeAllowedForCaller(
|
||||
"Request_Diagnostics", isTenantAdmin: false, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeserializePendingActions_NullOrEmpty_ReturnsEmpty()
|
||||
{
|
||||
|
||||
@@ -62,7 +62,7 @@ public class SessionDeletionWorkerTests
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: QueuePollingWorkerBase.DefaultMaxDequeueCount + 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
// Poison queue received the body and main queue deleted the original.
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(
|
||||
@@ -121,7 +121,7 @@ public class SessionDeletionWorkerTests
|
||||
},
|
||||
"etag-mock"));
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.Contains($"\"observedResidualCount\":{capObservation}", poisoned.RawDetails);
|
||||
@@ -168,7 +168,7 @@ public class SessionDeletionWorkerTests
|
||||
},
|
||||
"etag-mock"));
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.Contains("\"observedResidualCount\":3", poisoned.RawDetails);
|
||||
@@ -307,7 +307,7 @@ public class SessionDeletionWorkerTests
|
||||
},
|
||||
"etag-mock"));
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.True(poisoned.RawDetails.Length < 4096,
|
||||
@@ -357,7 +357,7 @@ public class SessionDeletionWorkerTests
|
||||
return Task.FromResult(Response.FromValue(receipt, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var casIdx = callOrder.IndexOf("cas-running-to-poisoned");
|
||||
var poisonIdx = callOrder.IndexOf("poison-queue-send");
|
||||
@@ -402,7 +402,7 @@ public class SessionDeletionWorkerTests
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
// Both CAS attempts fired, and the poison-queue send still completed.
|
||||
harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync(
|
||||
@@ -444,7 +444,7 @@ public class SessionDeletionWorkerTests
|
||||
CurrentState = "Unknown",
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
@@ -457,7 +457,7 @@ public class SessionDeletionWorkerTests
|
||||
var harness = new Harness();
|
||||
harness.EnqueueMessage("{ \"this is not valid JSON for an envelope: garbage", dequeueCount: 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.MainQueueDeleted());
|
||||
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
@@ -491,7 +491,7 @@ public class SessionDeletionWorkerTests
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(350), ct);
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(800));
|
||||
await harness.RunUntilAsync(() => harness.HeartbeatCount() >= 3);
|
||||
|
||||
// Heartbeat called at least 3× while the handler was busy (350ms / 50ms ≈ 7, minus a few
|
||||
// due to scheduling jitter). One call is enough to prove the heartbeat task is wired.
|
||||
@@ -512,7 +512,7 @@ public class SessionDeletionWorkerTests
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.MainQueueDeleted());
|
||||
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
@@ -663,7 +663,9 @@ public class SessionDeletionWorkerTests
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Returns<OpsEventEntry>(e =>
|
||||
{
|
||||
OpsEventCalls.Add(CapturedOpsEvent.From(e));
|
||||
// Locked so RunUntilAsync's condition poll can read Count from the test
|
||||
// thread while the worker thread appends.
|
||||
lock (OpsEventCalls) OpsEventCalls.Add(CapturedOpsEvent.From(e));
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
var alertDispatch = new OpsAlertDispatchService(
|
||||
@@ -721,6 +723,41 @@ public class SessionDeletionWorkerTests
|
||||
try { await Sut.StopAsync(CancellationToken.None); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// De-flake (CI run 30047512186): <see cref="RunForAsync"/>'s fixed wall-clock window
|
||||
/// raced the worker's background poll loop on loaded CI runners — positive assertions
|
||||
/// (delete issued, ops event recorded) could fire before the loop ever processed the
|
||||
/// message. Runs the worker until <paramref name="condition"/> observes the awaited
|
||||
/// side effect (polled every 25 ms); the ceiling only bounds a genuinely broken worker,
|
||||
/// the asserts after this call remain the source of truth. Same pattern as the watchdog
|
||||
/// event-gate de-flake. Only for POSITIVE expectations — Times.Never-style tests keep
|
||||
/// the fixed window (there is no event to wait for).
|
||||
/// </summary>
|
||||
public async Task RunUntilAsync(Func<bool> condition, TimeSpan? timeout = null)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(10));
|
||||
using var cts = new CancellationTokenSource();
|
||||
try { await Sut.StartAsync(cts.Token); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
while (!condition() && DateTime.UtcNow < deadline)
|
||||
await Task.Delay(25);
|
||||
cts.Cancel();
|
||||
try { await Sut.StopAsync(CancellationToken.None); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
}
|
||||
|
||||
// Condition probes for RunUntilAsync. Moq's Invocations list snapshots under its
|
||||
// internal lock, so polling it from the test thread while the worker invokes the
|
||||
// mock is safe. OpsEventCalls is guarded by its own lock (see the repo callback).
|
||||
public bool MainQueueDeleted() =>
|
||||
MainQueue.Invocations.Any(i => i.Method.Name == nameof(QueueClient.DeleteMessageAsync));
|
||||
public int HeartbeatCount() =>
|
||||
MainQueue.Invocations.Count(i => i.Method.Name == nameof(QueueClient.UpdateMessageAsync));
|
||||
public bool OpsEventRecorded()
|
||||
{
|
||||
lock (OpsEventCalls) return OpsEventCalls.Count > 0;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record AuditEntry(
|
||||
|
||||
@@ -13,6 +13,7 @@ public class TenantNotificationAudienceCatalogTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("hardware_rejection", NotificationAudience.Admin)]
|
||||
[InlineData("tpm_pss_unsupported", NotificationAudience.Admin)]
|
||||
[InlineData("sla_breach", NotificationAudience.Member)]
|
||||
[InlineData("sla_consecutive_failures", NotificationAudience.Member)]
|
||||
[InlineData("sla_resolved", NotificationAudience.Member)]
|
||||
|
||||
@@ -0,0 +1,562 @@
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Golden fixtures for <see cref="TimeAttributionCalculator"/> (F1 PR1, insights spec §F1) —
|
||||
/// synthetic sessions only (no customer data), each pinning one semantic decision:
|
||||
/// linear phase partition, clock skew, late start, WhiteGlove two-window split, missing /
|
||||
/// truncated blocking lists, IME retry occupancy, and reboot gaps. EVERY fixture asserts the
|
||||
/// exact-partition invariant: sum(span seconds) + unattributed == wall clock ==
|
||||
/// the session's DurationSeconds (truthfulness rule 2 — never normalized, never interpolated).
|
||||
/// </summary>
|
||||
public class TimeAttributionCalculatorTests
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 7, 26, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const string AppA = "aaaaaaaa-1111-2222-3333-444444444444"; // in blocking list
|
||||
private const string AppB = "bbbbbbbb-1111-2222-3333-444444444444"; // in blocking list, never observed
|
||||
private const string AppC = "cccccccc-1111-2222-3333-444444444444"; // observed, NOT listed
|
||||
|
||||
private static long _seq;
|
||||
|
||||
private static EnrollmentEvent Evt(
|
||||
DateTime ts, string eventType,
|
||||
EnrollmentPhase phase = EnrollmentPhase.Unknown,
|
||||
Dictionary<string, object>? data = null)
|
||||
=> new()
|
||||
{
|
||||
EventType = eventType,
|
||||
Timestamp = ts,
|
||||
Sequence = ++_seq,
|
||||
Phase = phase,
|
||||
Data = data ?? new Dictionary<string, object>(),
|
||||
};
|
||||
|
||||
private static Dictionary<string, object> AppData(string appId, string appName, string? state = null)
|
||||
{
|
||||
var d = new Dictionary<string, object> { ["appId"] = appId, ["appName"] = appName };
|
||||
if (state != null) d["state"] = state;
|
||||
return d;
|
||||
}
|
||||
|
||||
/// <summary>Storage-shape blocking lists (EventDataNormalizer yields List<object>).</summary>
|
||||
private static Dictionary<string, object> EspLists(long? win32Count = null, params string[] win32Ids)
|
||||
=> new()
|
||||
{
|
||||
["espTrackedWin32AppIds"] = new List<object>(win32Ids),
|
||||
["espTrackedUserWin32AppIds"] = new List<object>(),
|
||||
["espTrackedMsiProductCodes"] = new List<object>(),
|
||||
["espTrackedModernAppPfns"] = new List<object>(),
|
||||
["espTrackedWin32Count"] = win32Count ?? win32Ids.Length,
|
||||
["espTrackedMsiCount"] = 0L,
|
||||
["espTrackedModernCount"] = 0L,
|
||||
};
|
||||
|
||||
private static TimeAttributionInput Input(
|
||||
List<EnrollmentEvent> events, DateTime completedAt, int durationSeconds,
|
||||
string status = "Succeeded", bool wg = false, DateTime? resumedAt = null)
|
||||
=> new()
|
||||
{
|
||||
TenantId = "00000000-0000-0000-0000-0000000000t1",
|
||||
SessionId = "00000000-0000-0000-0000-0000000000s1",
|
||||
Status = status,
|
||||
StartedAt = T0,
|
||||
CompletedAt = completedAt,
|
||||
DurationSeconds = durationSeconds,
|
||||
IsPreProvisioned = wg,
|
||||
ResumedAt = resumedAt,
|
||||
Events = events,
|
||||
};
|
||||
|
||||
/// <summary>The invariant every fixture must satisfy exactly (spec §0 rule 2).</summary>
|
||||
private static void AssertExactPartition(SessionTimeBreakdown b)
|
||||
{
|
||||
Assert.Equal(b.WallClockSeconds, b.Segments.Sum(s => s.Seconds) + b.UnattributedSeconds);
|
||||
Assert.True(b.UnattributedSeconds >= 0, "unattributed must never be negative");
|
||||
}
|
||||
|
||||
// ── golden: linear UDE session ──────────────────────────────────────────
|
||||
|
||||
private static List<EnrollmentEvent> LinearSession()
|
||||
=> new()
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(2), "esp_config_detected", data: EspLists(win32Ids: new[] { AppA, AppB })),
|
||||
Evt(T0.AddMinutes(5), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(6), "app_install_started", data: AppData(AppA, "App A")),
|
||||
Evt(T0.AddMinutes(10), "app_install_completed", data: AppData(AppA, "App A", "Installed")),
|
||||
Evt(T0.AddMinutes(11), "app_install_started", data: AppData(AppC, "Untracked App")),
|
||||
Evt(T0.AddMinutes(13), "app_install_completed", data: AppData(AppC, "Untracked App", "Installed")),
|
||||
Evt(T0.AddMinutes(15), "esp_phase_changed", EnrollmentPhase.AccountSetup),
|
||||
Evt(T0.AddMinutes(20), "esp_phase_changed", EnrollmentPhase.AppsUser),
|
||||
Evt(T0.AddMinutes(25), "esp_phase_changed", EnrollmentPhase.FinalizingSetup),
|
||||
Evt(T0.AddMinutes(28), "desktop_arrived"),
|
||||
Evt(T0.AddMinutes(30), "enrollment_complete"),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Linear_Session_PartitionsIntoAllFiveSegments_Exactly()
|
||||
{
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(LinearSession(), completedAt: T0.AddMinutes(30), durationSeconds: 1800))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.Equal(1800, b.WallClockSeconds);
|
||||
Assert.Equal(0, b.UnattributedSeconds);
|
||||
Assert.Equal(TimeAttributionFlags.None, b.QualityFlags);
|
||||
Assert.Equal(TimeAttributionCalculator.CurrentVersion, b.AttributionVersion);
|
||||
|
||||
var totals = b.GetSegmentTotals();
|
||||
Assert.Equal(300, totals[TimeAttributionSegments.DevicePrep]); // T0 → apps entry (+5m)
|
||||
Assert.Equal(600, totals[TimeAttributionSegments.EspApps]); // +5m → +15m
|
||||
Assert.Equal(480, totals[TimeAttributionSegments.IdentityHello]); // AccountSetup 300 + Finalizing 180
|
||||
Assert.Equal(300, totals[TimeAttributionSegments.UserEsp]); // +20m → +25m
|
||||
Assert.Equal(120, totals[TimeAttributionSegments.DesktopHandoff]); // desktop_arrived → terminal
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Linear_Session_BlockingJoin_IsPositiveEvidenceOnly()
|
||||
{
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(LinearSession(), completedAt: T0.AddMinutes(30), durationSeconds: 1800))!;
|
||||
|
||||
// App A is listed → blocking, interval from EVENT timestamps (started → completed).
|
||||
var interval = Assert.Single(b.BlockingApps);
|
||||
Assert.Equal(AppA, interval.AppId);
|
||||
Assert.Equal(240, interval.Seconds); // +6m → +10m
|
||||
// App C installed fine but is NOT in the lists → unknown, NEVER "not blocking":
|
||||
// it appears nowhere (no negative claim), and only App A counts as matched.
|
||||
Assert.Equal(1, b.BlockingAppCount);
|
||||
// Occupancy = App A's interval clipped to the esp_apps span.
|
||||
Assert.Equal(240, b.EspAppsOccupancySeconds);
|
||||
}
|
||||
|
||||
// ── skew: backward anchor is dropped, dust stays honest ─────────────────
|
||||
|
||||
[Fact]
|
||||
public void BackwardPhaseAnchor_IsDropped_WithClockSkewFlag()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(10), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
// Replayed declaration whose timestamp runs BACKWARD in canonical order.
|
||||
Evt(T0.AddMinutes(3), "esp_phase_changed", EnrollmentPhase.AccountSetup),
|
||||
Evt(T0.AddMinutes(30), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(30), durationSeconds: 1800))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.True(b.QualityFlags.HasFlag(TimeAttributionFlags.ClockSkewDropped));
|
||||
var totals = b.GetSegmentTotals();
|
||||
Assert.Equal(600, totals[TimeAttributionSegments.DevicePrep]);
|
||||
Assert.Equal(1200, totals[TimeAttributionSegments.EspApps]); // runs to window end — no phantom AccountSetup
|
||||
Assert.False(totals.ContainsKey(TimeAttributionSegments.IdentityHello));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SubSecondBoundaries_FloorIntoUnattributedDust_NeverInvented()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddSeconds(90.5), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddSeconds(300), durationSeconds: 300))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
var totals = b.GetSegmentTotals();
|
||||
Assert.Equal(90, totals[TimeAttributionSegments.DevicePrep]); // floor(90.5)
|
||||
Assert.Equal(209, totals[TimeAttributionSegments.EspApps]); // floor(209.5)
|
||||
Assert.Equal(1, b.UnattributedSeconds); // the dust, disclosed
|
||||
}
|
||||
|
||||
// ── late start ──────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void AgentLateStart_SetsPartialObservation()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddSeconds(1), "agent_late_start"),
|
||||
Evt(T0.AddMinutes(10), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(10), durationSeconds: 600))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.True(b.QualityFlags.HasFlag(TimeAttributionFlags.PartialObservation));
|
||||
}
|
||||
|
||||
// ── WhiteGlove: two windows in ONE session, pause never attributed ──────
|
||||
|
||||
[Fact]
|
||||
public void WhiteGlove_SplitsIntoTwoWindows_PauseExcluded_WallClockIsDurationSeconds()
|
||||
{
|
||||
var part1End = T0.AddMinutes(20);
|
||||
var resumedAt = T0.AddDays(2); // 2-day pause
|
||||
var completedAt = resumedAt.AddMinutes(10); // part 2 = 600 s
|
||||
// Stored combined duration: part1 (1200 s) + part2 (600 s) — pause excluded by design.
|
||||
const int duration = 1800;
|
||||
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(1), "esp_config_detected", data: EspLists(win32Ids: new[] { AppA })),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(part1End, "whiteglove_part1_complete"),
|
||||
// — pause —
|
||||
Evt(resumedAt.AddSeconds(30), "agent_started", EnrollmentPhase.AccountSetup),
|
||||
Evt(resumedAt.AddMinutes(5), "esp_phase_changed", EnrollmentPhase.FinalizingSetup),
|
||||
Evt(resumedAt.AddMinutes(8), "desktop_arrived"),
|
||||
Evt(completedAt, "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt, duration, wg: true, resumedAt: resumedAt))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
// Wall clock is the stored DurationSeconds — NOT CompletedAt − StartedAt (≈ 2 days).
|
||||
Assert.Equal(1800, b.WallClockSeconds);
|
||||
Assert.Equal(TimeAttributionFlags.None, b.QualityFlags); // part1 anchored on the event
|
||||
|
||||
var totals = b.GetSegmentTotals();
|
||||
Assert.Equal(120, totals[TimeAttributionSegments.DevicePrep]); // W1: T0 → +2m
|
||||
Assert.Equal(1080, totals[TimeAttributionSegments.EspApps]); // W1: +2m → part1End
|
||||
Assert.Equal(450, totals[TimeAttributionSegments.IdentityHello]); // W2: +30s → +8m
|
||||
Assert.Equal(120, totals[TimeAttributionSegments.DesktopHandoff]); // W2: +8m → +10m
|
||||
// W2's 30 s resume gap has no phase context — unattributed, not guessed.
|
||||
Assert.Equal(30, b.UnattributedSeconds);
|
||||
// No span may live inside the pause.
|
||||
Assert.DoesNotContain(b.Segments, s => s.StartUtc > part1End && s.EndUtc < resumedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGlove_WithoutPart1CompleteEvent_FallsBackAndFlags()
|
||||
{
|
||||
var resumedAt = T0.AddDays(1);
|
||||
var completedAt = resumedAt.AddMinutes(10);
|
||||
var lastPart1Event = T0.AddMinutes(20);
|
||||
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(lastPart1Event, "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(resumedAt.AddSeconds(10), "agent_started", EnrollmentPhase.AccountSetup),
|
||||
Evt(completedAt, "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt, durationSeconds: 1800, wg: true, resumedAt: resumedAt))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.True(b.QualityFlags.HasFlag(TimeAttributionFlags.WhiteGloveAnchorsIncomplete));
|
||||
Assert.Equal(1800, b.WallClockSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGlove_Part2LongerThanStoredDuration_DegradesToSingleWindow_Flagged()
|
||||
{
|
||||
// Stored duration (600 s) contradicts the resume stamp (part 2 alone spans 900 s):
|
||||
// no honest two-window split exists — degrade to the single derived window instead
|
||||
// of inventing a negative part 1.
|
||||
var resumedAt = T0.AddDays(1);
|
||||
var completedAt = resumedAt.AddMinutes(15);
|
||||
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(resumedAt.AddSeconds(5), "agent_started", EnrollmentPhase.AccountSetup),
|
||||
Evt(completedAt, "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt, durationSeconds: 600, wg: true, resumedAt: resumedAt))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.Equal(600, b.WallClockSeconds);
|
||||
Assert.True(b.QualityFlags.HasFlag(TimeAttributionFlags.WhiteGloveAnchorsIncomplete));
|
||||
}
|
||||
|
||||
// ── blocking list missing / truncated ───────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void NoBlockingLists_MeansUnknown_NotZero()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(3), "app_install_started", data: AppData(AppA, "App A")),
|
||||
Evt(T0.AddMinutes(8), "app_install_completed", data: AppData(AppA, "App A", "Installed")),
|
||||
Evt(T0.AddMinutes(10), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(10), durationSeconds: 600))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.True(b.QualityFlags.HasFlag(TimeAttributionFlags.BlockingSetUnknown));
|
||||
Assert.Empty(b.BlockingApps);
|
||||
Assert.Equal(0, b.BlockingAppCount);
|
||||
Assert.Null(b.EspAppsOccupancySeconds); // unknown — NOT 0 (rule 1)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspConfigWithoutListKeys_IsNotEvidence_LaterEmissionNeverErasesEarlierLists()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(1), "esp_config_detected", data: EspLists(win32Ids: new[] { AppA })),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(3), "app_install_started", data: AppData(AppA, "App A")),
|
||||
Evt(T0.AddMinutes(8), "app_install_completed", data: AppData(AppA, "App A", "Installed")),
|
||||
// Later emission whose probe found no Diagnostics key — no espTracked* keys at all.
|
||||
Evt(T0.AddMinutes(9), "esp_config_detected",
|
||||
data: new Dictionary<string, object> { ["source"] = "registry_firstsync" }),
|
||||
Evt(T0.AddMinutes(10), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(10), durationSeconds: 600))!;
|
||||
|
||||
Assert.False(b.QualityFlags.HasFlag(TimeAttributionFlags.BlockingSetUnknown));
|
||||
Assert.Equal(AppA, Assert.Single(b.BlockingApps).AppId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TruncatedBlockingList_SetsFlag_AbsentAppsStayUnknown()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
// Uncapped total (60) exceeds the emitted list (1 id) → cap dropped entries.
|
||||
Evt(T0.AddMinutes(1), "esp_config_detected", data: EspLists(win32Count: 60, AppA)),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(3), "app_install_started", data: AppData(AppC, "Maybe Capped App")),
|
||||
Evt(T0.AddMinutes(7), "app_install_completed", data: AppData(AppC, "Maybe Capped App", "Installed")),
|
||||
Evt(T0.AddMinutes(10), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(10), durationSeconds: 600))!;
|
||||
|
||||
Assert.True(b.QualityFlags.HasFlag(TimeAttributionFlags.BlockingSetTruncated));
|
||||
// App C is absent from the (capped) list → it stays UNKNOWN: no interval, no claim.
|
||||
Assert.Empty(b.BlockingApps);
|
||||
Assert.Equal(0, b.BlockingAppCount);
|
||||
}
|
||||
|
||||
// ── retry: occupancy spans to the LAST terminal event (audit Q1) ────────
|
||||
|
||||
[Fact]
|
||||
public void Retry_IntervalRunsFromFirstStart_ToLastTerminalEvent()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(1), "esp_config_detected", data: EspLists(win32Ids: new[] { AppA })),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(6), "app_install_started", data: AppData(AppA, "App A")),
|
||||
Evt(T0.AddMinutes(8), "app_install_failed", data: AppData(AppA, "App A")),
|
||||
Evt(T0.AddMinutes(9), "app_install_started", data: AppData(AppA, "App A")), // IME retry
|
||||
Evt(T0.AddMinutes(12), "app_install_completed", data: AppData(AppA, "App A", "Installed")),
|
||||
Evt(T0.AddMinutes(15), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(15), durationSeconds: 900))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
var interval = Assert.Single(b.BlockingApps);
|
||||
// First started (+6m) → LAST terminal (+12m) = 360 s. The payload timing would have
|
||||
// frozen at the first failure (+8m) — event timestamps are the only valid source.
|
||||
Assert.Equal(360, interval.Seconds);
|
||||
Assert.Equal(360, b.EspAppsOccupancySeconds);
|
||||
}
|
||||
|
||||
// ── reboots: spans from event gaps, never the detection event's stamp ───
|
||||
|
||||
[Fact]
|
||||
public void RebootSpan_ComesFromEventGapAroundLastBootUtc_NotFromDetectionTimestamp()
|
||||
{
|
||||
var boot = T0.AddMinutes(12);
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(5), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(10), "app_install_started", data: AppData(AppA, "App A")), // last pre-reboot
|
||||
Evt(T0.AddMinutes(14), "system_reboot_detected", // next run, detection-time
|
||||
data: new Dictionary<string, object> { ["lastBootUtc"] = boot.ToString("o") }),
|
||||
// Duplicate detection of the SAME boot (batch replay) — must not double-count.
|
||||
Evt(T0.AddMinutes(14.5), "system_reboot_detected",
|
||||
data: new Dictionary<string, object> { ["lastBootUtc"] = boot.ToString("o") }),
|
||||
Evt(T0.AddMinutes(20), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(20), durationSeconds: 1200))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
var span = Assert.Single(b.RebootSpans);
|
||||
// Gap = last event before boot (+10m) → first observation at/after boot (+14m).
|
||||
Assert.Equal(T0.AddMinutes(10), span.StartUtc);
|
||||
Assert.Equal(T0.AddMinutes(14), span.EndUtc);
|
||||
Assert.Equal(240, span.Seconds);
|
||||
Assert.Equal(240, b.RebootSeconds);
|
||||
// The reboot began inside the apps phase — cross-cutting annotation, segment unchanged.
|
||||
Assert.Equal(TimeAttributionSegments.EspApps, span.SegmentKey);
|
||||
Assert.Equal(1200 - 300, b.GetSegmentTotals()[TimeAttributionSegments.EspApps]);
|
||||
}
|
||||
|
||||
// ── WhiteGlove pause vs. intervals/reboots (Codex review P1): the pause lies INSIDE
|
||||
// the windows' hull, so hull-clamping alone would count it — a reboot bracketing the
|
||||
// pause must contribute only its in-window flanks, never the pause itself ─────────────
|
||||
|
||||
[Fact]
|
||||
public void WhiteGlove_RebootGapAcrossPause_CountsOnlyInWindowFlanks()
|
||||
{
|
||||
var part1End = T0.AddMinutes(20);
|
||||
var resumedAt = T0.AddDays(2); // 2-day pause
|
||||
var completedAt = resumedAt.AddMinutes(10); // part 2 = 600 s → part 1 = 1200 s
|
||||
var bootInsidePause = T0.AddDays(1);
|
||||
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(part1End, "whiteglove_part1_complete"),
|
||||
// — pause: device sealed, user boots it a day later —
|
||||
Evt(resumedAt.AddSeconds(30), "agent_started", EnrollmentPhase.AccountSetup),
|
||||
Evt(resumedAt.AddSeconds(40), "system_reboot_detected",
|
||||
data: new Dictionary<string, object> { ["lastBootUtc"] = bootInsidePause.ToString("o") }),
|
||||
Evt(completedAt, "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt, durationSeconds: 1800, wg: true, resumedAt: resumedAt))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
var span = Assert.Single(b.RebootSpans);
|
||||
// Gap = part1End → first post-boot event (+30 s into part 2). In-window seconds are
|
||||
// ONLY the part-2 flank (30 s) — the ~2-day pause never counts (pre-fix: ~172 830 s
|
||||
// of "reboot" in a 30-minute enrollment).
|
||||
Assert.Equal(part1End, span.StartUtc);
|
||||
Assert.Equal(resumedAt.AddSeconds(30), span.EndUtc);
|
||||
Assert.Equal(30, span.Seconds);
|
||||
Assert.Equal(30, b.RebootSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGlove_BlockingIntervalAcrossPause_CountsOnlyInWindowSeconds()
|
||||
{
|
||||
var part1End = T0.AddMinutes(20);
|
||||
var resumedAt = T0.AddDays(2);
|
||||
var completedAt = resumedAt.AddMinutes(10);
|
||||
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(1), "esp_config_detected", data: EspLists(win32Ids: new[] { AppA })),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(T0.AddMinutes(10), "app_install_started", data: AppData(AppA, "App A")),
|
||||
Evt(part1End, "whiteglove_part1_complete"),
|
||||
// — pause —
|
||||
Evt(resumedAt.AddSeconds(30), "agent_started", EnrollmentPhase.AccountSetup),
|
||||
// The app's final terminal lands in part 2 (user-scope finish after resume).
|
||||
Evt(resumedAt.AddMinutes(2), "app_install_completed", data: AppData(AppA, "App A", "Installed")),
|
||||
Evt(completedAt, "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt, durationSeconds: 1800, wg: true, resumedAt: resumedAt))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
var interval = Assert.Single(b.BlockingApps);
|
||||
// Chronological endpoints stay the real event times…
|
||||
Assert.Equal(T0.AddMinutes(10), interval.StartUtc);
|
||||
Assert.Equal(resumedAt.AddMinutes(2), interval.EndUtc);
|
||||
// …but Seconds is the in-window sum: part 1 tail (10 min) + part 2 head (2 min).
|
||||
Assert.Equal(720, interval.Seconds);
|
||||
// Occupancy intersects with the esp_apps span (part 1 only) — pause never leaks in.
|
||||
Assert.Equal(600, b.EspAppsOccupancySeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGlove_IntervalWhollyInsidePause_MakesNoClaim_AndNoSkewFlag()
|
||||
{
|
||||
var part1End = T0.AddMinutes(20);
|
||||
var resumedAt = T0.AddDays(2);
|
||||
var completedAt = resumedAt.AddMinutes(10);
|
||||
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0, "agent_started", EnrollmentPhase.DeviceSetup),
|
||||
Evt(T0.AddMinutes(1), "esp_config_detected", data: EspLists(win32Ids: new[] { AppA })),
|
||||
Evt(T0.AddMinutes(2), "esp_phase_changed", EnrollmentPhase.AppsDevice),
|
||||
Evt(part1End, "whiteglove_part1_complete"),
|
||||
// Both endpoints inside the pause (post-seal tail activity) — observed, but in no window.
|
||||
Evt(part1End.AddHours(1), "app_install_started", data: AppData(AppA, "App A")),
|
||||
Evt(part1End.AddHours(2), "app_install_completed", data: AppData(AppA, "App A", "Installed")),
|
||||
Evt(resumedAt.AddSeconds(30), "agent_started", EnrollmentPhase.AccountSetup),
|
||||
Evt(completedAt, "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt, durationSeconds: 1800, wg: true, resumedAt: resumedAt))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
// No in-window observation → no interval claim; and nothing is wrong with the clocks,
|
||||
// so the skew flag must NOT fire (it would exclude the session from fleet medians).
|
||||
Assert.Empty(b.BlockingApps);
|
||||
Assert.False(b.QualityFlags.HasFlag(TimeAttributionFlags.ClockSkewDropped));
|
||||
}
|
||||
|
||||
// ── non-computable sessions ─────────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("Incomplete")] // terminal but deliberately NO DurationSeconds
|
||||
[InlineData("InProgress")]
|
||||
[InlineData("Pending")]
|
||||
public void NonComputableStatuses_ReturnNull(string status)
|
||||
{
|
||||
var input = Input(LinearSession(), completedAt: T0.AddMinutes(30), durationSeconds: 1800, status: status);
|
||||
Assert.Null(TimeAttributionCalculator.Compute(input));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingDuration_ReturnsNull_NeverGuessesFromCompletedMinusStarted()
|
||||
{
|
||||
var input = Input(LinearSession(), completedAt: T0.AddMinutes(30), durationSeconds: 0);
|
||||
Assert.Null(TimeAttributionCalculator.Compute(input));
|
||||
}
|
||||
|
||||
// ── no phase evidence at all ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void NoPhaseDeclarations_LeaveEverythingUnattributed()
|
||||
{
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
Evt(T0.AddMinutes(1), "network_state_change"),
|
||||
Evt(T0.AddMinutes(9), "enrollment_complete"),
|
||||
};
|
||||
|
||||
var b = TimeAttributionCalculator.Compute(
|
||||
Input(events, completedAt: T0.AddMinutes(10), durationSeconds: 600))!;
|
||||
|
||||
AssertExactPartition(b);
|
||||
Assert.Empty(b.Segments);
|
||||
Assert.Equal(600, b.UnattributedSeconds); // no evidence → no claim (rule 1)
|
||||
}
|
||||
}
|
||||
+458
@@ -0,0 +1,458 @@
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// F1 PR2 (insights spec §F1 "Data & compute changes"): persistence round-trips for the
|
||||
/// SessionTimeBreakdowns / TimeAttributionAggregates rows (table-storage-serialization rule —
|
||||
/// every property must survive Store→Map, tri-states included), the pure daily-aggregation
|
||||
/// core (<see cref="MaintenanceService.BuildTimeAttributionAggregates"/>: clean/flagged/missing
|
||||
/// disclosure, per-class bucketing, fixed segment stack, per-app gates) and the what-if bound.
|
||||
/// </summary>
|
||||
public class TimeAttributionPersistenceAndAggregationTests
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 7, 26, 10, 0, 0, DateTimeKind.Utc);
|
||||
private const string TenantA = "aaaaaaaa-0000-0000-0000-000000000001";
|
||||
private const string TenantB = "bbbbbbbb-0000-0000-0000-000000000002";
|
||||
private const string AppX = "aaaaaaaa-1111-2222-3333-444444444444";
|
||||
private const string AppY = "bbbbbbbb-1111-2222-3333-444444444444";
|
||||
|
||||
// ── breakdown row round-trip ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void BreakdownEntity_RoundTripsAllFields_IncludingJsonLists()
|
||||
{
|
||||
var breakdown = new SessionTimeBreakdown
|
||||
{
|
||||
TenantId = TenantA,
|
||||
SessionId = "s-1",
|
||||
AttributionVersion = 1,
|
||||
EventCountAtCompute = 261,
|
||||
WallClockSeconds = 1800,
|
||||
UnattributedSeconds = 30,
|
||||
RebootSeconds = 240,
|
||||
BlockingAppCount = 3,
|
||||
EspAppsOccupancySeconds = 600,
|
||||
QualityFlags = TimeAttributionFlags.PartialObservation | TimeAttributionFlags.BlockingSetTruncated,
|
||||
Segments = new List<TimeAttributionSpan>
|
||||
{
|
||||
new() { SegmentKey = TimeAttributionSegments.DevicePrep, StartUtc = T0, EndUtc = T0.AddMinutes(5), Seconds = 300 },
|
||||
new() { SegmentKey = TimeAttributionSegments.EspApps, StartUtc = T0.AddMinutes(5), EndUtc = T0.AddMinutes(30), Seconds = 1500 },
|
||||
},
|
||||
RebootSpans = new List<RebootSpan>
|
||||
{
|
||||
new() { StartUtc = T0.AddMinutes(10), EndUtc = T0.AddMinutes(14), Seconds = 240, SegmentKey = TimeAttributionSegments.EspApps },
|
||||
},
|
||||
BlockingApps = new List<BlockingAppInterval>
|
||||
{
|
||||
new() { AppId = AppX, AppName = "App X", StartUtc = T0.AddMinutes(6), EndUtc = T0.AddMinutes(16), Seconds = 600 },
|
||||
},
|
||||
};
|
||||
|
||||
var mapped = TableStorageService.MapToSessionTimeBreakdown(
|
||||
TableStorageService.BuildSessionTimeBreakdownEntity(breakdown));
|
||||
|
||||
Assert.Equal(TenantA, mapped.TenantId);
|
||||
Assert.Equal("s-1", mapped.SessionId);
|
||||
Assert.Equal(1, mapped.AttributionVersion);
|
||||
// Sweep change signal (Codex review: late batches must trigger a recompute) — a lost
|
||||
// column would read as 0 and recompute forever OR mask real drift.
|
||||
Assert.Equal(261, mapped.EventCountAtCompute);
|
||||
Assert.Equal(1800, mapped.WallClockSeconds);
|
||||
Assert.Equal(30, mapped.UnattributedSeconds);
|
||||
Assert.Equal(240, mapped.RebootSeconds);
|
||||
Assert.Equal(3, mapped.BlockingAppCount);
|
||||
Assert.Equal(600, mapped.EspAppsOccupancySeconds);
|
||||
Assert.Equal(TimeAttributionFlags.PartialObservation | TimeAttributionFlags.BlockingSetTruncated, mapped.QualityFlags);
|
||||
|
||||
Assert.Equal(2, mapped.Segments.Count);
|
||||
Assert.Equal(TimeAttributionSegments.DevicePrep, mapped.Segments[0].SegmentKey);
|
||||
Assert.Equal(T0, mapped.Segments[0].StartUtc);
|
||||
Assert.Equal(300, mapped.Segments[0].Seconds);
|
||||
|
||||
var reboot = Assert.Single(mapped.RebootSpans);
|
||||
Assert.Equal(TimeAttributionSegments.EspApps, reboot.SegmentKey);
|
||||
Assert.Equal(240, reboot.Seconds);
|
||||
|
||||
var app = Assert.Single(mapped.BlockingApps);
|
||||
Assert.Equal(AppX, app.AppId);
|
||||
Assert.Equal("App X", app.AppName);
|
||||
Assert.Equal(600, app.Seconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BreakdownEntity_OccupancyTriState_AbsentColumnMapsToNull_NeverZero()
|
||||
{
|
||||
var unknown = new SessionTimeBreakdown
|
||||
{
|
||||
TenantId = TenantA, SessionId = "s-1", WallClockSeconds = 600,
|
||||
EspAppsOccupancySeconds = null, QualityFlags = TimeAttributionFlags.BlockingSetUnknown,
|
||||
};
|
||||
|
||||
var entity = TableStorageService.BuildSessionTimeBreakdownEntity(unknown);
|
||||
Assert.False(entity.ContainsKey("EspAppsOccupancySeconds"));
|
||||
Assert.Null(TableStorageService.MapToSessionTimeBreakdown(entity).EspAppsOccupancySeconds);
|
||||
|
||||
// Zero is a real measured value (blocking set known, nothing occupied) — must survive.
|
||||
var zero = new SessionTimeBreakdown
|
||||
{
|
||||
TenantId = TenantA, SessionId = "s-2", WallClockSeconds = 600, EspAppsOccupancySeconds = 0,
|
||||
};
|
||||
Assert.Equal(0, TableStorageService.MapToSessionTimeBreakdown(
|
||||
TableStorageService.BuildSessionTimeBreakdownEntity(zero)).EspAppsOccupancySeconds);
|
||||
}
|
||||
|
||||
// ── aggregate row round-trip ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void AggregateEntity_RoundTripsAllFields()
|
||||
{
|
||||
var aggregate = new TimeAttributionDailyAggregate
|
||||
{
|
||||
TenantId = TenantA,
|
||||
Date = "2026-07-26",
|
||||
EnrollmentClass = "user_driven",
|
||||
AttributionVersion = 1,
|
||||
CleanSessionCount = 23,
|
||||
FlaggedExcludedCount = 4,
|
||||
MissingBreakdownCount = 1,
|
||||
ComputedAt = T0,
|
||||
SegmentStats = new List<TimeAttributionSegmentStat>
|
||||
{
|
||||
new() { SegmentKey = TimeAttributionSegments.EspApps, MedianSeconds = 600, P75Seconds = 700, P90Seconds = 900 },
|
||||
},
|
||||
TopBlockingApps = new List<TimeAttributionBlockingAppStat>
|
||||
{
|
||||
new()
|
||||
{
|
||||
AppId = AppX, AppName = "App X", SessionCount = 12,
|
||||
MedianSeconds = 300, P75Seconds = 400, MedianSavingSeconds = 120, P75SavingSeconds = 200,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var entity = TableStorageService.BuildTimeAttributionAggregateEntity(aggregate);
|
||||
Assert.Equal("2026-07-26|user_driven", entity.RowKey);
|
||||
|
||||
var mapped = TableStorageService.MapToTimeAttributionAggregate(entity);
|
||||
Assert.Equal(TenantA, mapped.TenantId);
|
||||
Assert.Equal("2026-07-26", mapped.Date);
|
||||
Assert.Equal("user_driven", mapped.EnrollmentClass);
|
||||
Assert.Equal(1, mapped.AttributionVersion);
|
||||
Assert.Equal(23, mapped.CleanSessionCount);
|
||||
Assert.Equal(4, mapped.FlaggedExcludedCount);
|
||||
Assert.Equal(1, mapped.MissingBreakdownCount);
|
||||
|
||||
var segment = Assert.Single(mapped.SegmentStats);
|
||||
Assert.Equal(TimeAttributionSegments.EspApps, segment.SegmentKey);
|
||||
Assert.Equal(600, segment.MedianSeconds);
|
||||
Assert.Equal(900, segment.P90Seconds);
|
||||
|
||||
var app = Assert.Single(mapped.TopBlockingApps);
|
||||
Assert.Equal(AppX, app.AppId);
|
||||
Assert.Equal(12, app.SessionCount);
|
||||
Assert.Equal(120, app.MedianSavingSeconds);
|
||||
}
|
||||
|
||||
// ── enrollment class ────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void EnrollmentClass_Precedence_WhiteGlove_Wdp_SelfDeploying_UserDriven()
|
||||
{
|
||||
Assert.Equal("whiteglove", TimeAttributionCalculator.GetEnrollmentClass(
|
||||
new SessionSummary { IsPreProvisioned = true, EnrollmentType = "v2", IsSelfDeployingProfile = true }));
|
||||
Assert.Equal("device_preparation", TimeAttributionCalculator.GetEnrollmentClass(
|
||||
new SessionSummary { EnrollmentType = "v2", IsSelfDeployingProfile = true }));
|
||||
Assert.Equal("self_deploying", TimeAttributionCalculator.GetEnrollmentClass(
|
||||
new SessionSummary { IsSelfDeployingProfile = true }));
|
||||
Assert.Equal("user_driven", TimeAttributionCalculator.GetEnrollmentClass(new SessionSummary()));
|
||||
}
|
||||
|
||||
// ── what-if bound ───────────────────────────────────────────────────────
|
||||
|
||||
private static BlockingAppInterval Interval(string appId, DateTime start, DateTime end)
|
||||
=> new() { AppId = appId, AppName = appId, StartUtc = start, EndUtc = end, Seconds = (int)(end - start).TotalSeconds };
|
||||
|
||||
[Fact]
|
||||
public void WhatIf_LastEndingApp_SavesTailToPreviousEnd_OthersSaveNothing()
|
||||
{
|
||||
var intervals = new List<BlockingAppInterval>
|
||||
{
|
||||
Interval(AppX, T0, T0.AddMinutes(10)), // ends 10:10
|
||||
Interval(AppY, T0.AddMinutes(5), T0.AddMinutes(20)), // ends 10:20 — critical-path end
|
||||
};
|
||||
|
||||
// Removing Y moves the path end 10:20 → 10:10.
|
||||
Assert.Equal(600, TimeAttributionCalculator.WhatIfSavingSeconds(intervals, AppY));
|
||||
// Removing X changes nothing — Y still ends at 10:20.
|
||||
Assert.Equal(0, TimeAttributionCalculator.WhatIfSavingSeconds(intervals, AppX));
|
||||
// An app without a measured interval yields no claim.
|
||||
Assert.Equal(0, TimeAttributionCalculator.WhatIfSavingSeconds(intervals, "cccccccc-0000-0000-0000-000000000000"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhatIf_OnlyBlockingApp_FallsBackToOwnStart()
|
||||
{
|
||||
var intervals = new List<BlockingAppInterval> { Interval(AppX, T0.AddMinutes(5), T0.AddMinutes(20)) };
|
||||
Assert.Equal(900, TimeAttributionCalculator.WhatIfSavingSeconds(intervals, AppX));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhatIf_IdleGapBeforeLastApp_CountsIntoTheBound()
|
||||
{
|
||||
// A ends 10:00; X runs 10:05 → 10:20. Removing X exposes 10:00 as the path end —
|
||||
// the ESP waited for X to START too, so the bound is 20 min, not 15 (spec formula).
|
||||
var intervals = new List<BlockingAppInterval>
|
||||
{
|
||||
Interval(AppY, T0.AddMinutes(-30), T0),
|
||||
Interval(AppX, T0.AddMinutes(5), T0.AddMinutes(20)),
|
||||
};
|
||||
Assert.Equal(1200, TimeAttributionCalculator.WhatIfSavingSeconds(intervals, AppX));
|
||||
}
|
||||
|
||||
// ── daily aggregation core ──────────────────────────────────────────────
|
||||
|
||||
private static SessionSummary Session(string tenantId, DateTime startedAt, bool wg = false)
|
||||
=> new()
|
||||
{
|
||||
TenantId = tenantId,
|
||||
SessionId = Guid.NewGuid().ToString(),
|
||||
StartedAt = startedAt,
|
||||
Status = SessionStatus.Succeeded,
|
||||
IsPreProvisioned = wg,
|
||||
};
|
||||
|
||||
private static SessionTimeBreakdown Breakdown(
|
||||
TimeAttributionFlags flags = TimeAttributionFlags.None,
|
||||
int espAppsSeconds = 600,
|
||||
int unattributed = 0,
|
||||
List<BlockingAppInterval>? apps = null)
|
||||
=> new()
|
||||
{
|
||||
AttributionVersion = TimeAttributionCalculator.CurrentVersion,
|
||||
WallClockSeconds = espAppsSeconds + 300 + unattributed,
|
||||
UnattributedSeconds = unattributed,
|
||||
QualityFlags = flags,
|
||||
Segments = new List<TimeAttributionSpan>
|
||||
{
|
||||
new() { SegmentKey = TimeAttributionSegments.DevicePrep, StartUtc = T0, EndUtc = T0.AddSeconds(300), Seconds = 300 },
|
||||
new() { SegmentKey = TimeAttributionSegments.EspApps, StartUtc = T0.AddSeconds(300), EndUtc = T0.AddSeconds(300 + espAppsSeconds), Seconds = espAppsSeconds },
|
||||
},
|
||||
BlockingApps = apps ?? new List<BlockingAppInterval>(),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_BucketPerTenantClassDate_AndMirrorGlobal()
|
||||
{
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>
|
||||
{
|
||||
(Session(TenantA, T0), Breakdown(espAppsSeconds: 100)),
|
||||
(Session(TenantA, T0), Breakdown(espAppsSeconds: 300)),
|
||||
(Session(TenantA, T0, wg: true), Breakdown(espAppsSeconds: 900)), // separate class
|
||||
(Session(TenantB, T0), Breakdown(espAppsSeconds: 500)), // separate tenant
|
||||
};
|
||||
|
||||
var aggregates = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0);
|
||||
|
||||
// Tenant A: user_driven + whiteglove; Tenant B: user_driven; global: user_driven + whiteglove.
|
||||
Assert.Equal(5, aggregates.Count);
|
||||
|
||||
var tenantAUde = aggregates.Single(a => a.TenantId == TenantA && a.EnrollmentClass == "user_driven");
|
||||
Assert.Equal("2026-07-26", tenantAUde.Date);
|
||||
Assert.Equal(2, tenantAUde.CleanSessionCount);
|
||||
Assert.Equal(TimeAttributionCalculator.CurrentVersion, tenantAUde.AttributionVersion);
|
||||
// Median of {100, 300} = 100 (nearest-rank: ceil(0.5·2)−1 = index 0 — repo convention).
|
||||
Assert.Equal(100, tenantAUde.SegmentStats.Single(s => s.SegmentKey == TimeAttributionSegments.EspApps).MedianSeconds);
|
||||
|
||||
var wg = aggregates.Single(a => a.TenantId == TenantA && a.EnrollmentClass == "whiteglove");
|
||||
Assert.Equal(1, wg.CleanSessionCount); // classes never mixed
|
||||
|
||||
var globalUde = aggregates.Single(a => a.TenantId == "global" && a.EnrollmentClass == "user_driven");
|
||||
Assert.Equal(3, globalUde.CleanSessionCount); // A(2) + B(1)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_FlaggedSessions_AreExcludedFromStats_ButDisclosed()
|
||||
{
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>
|
||||
{
|
||||
(Session(TenantA, T0), Breakdown(espAppsSeconds: 100)),
|
||||
(Session(TenantA, T0), Breakdown(flags: TimeAttributionFlags.PartialObservation, espAppsSeconds: 99999)),
|
||||
};
|
||||
|
||||
var aggregate = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0)
|
||||
.Single(a => a.TenantId == TenantA);
|
||||
|
||||
Assert.Equal(1, aggregate.CleanSessionCount);
|
||||
Assert.Equal(1, aggregate.FlaggedExcludedCount);
|
||||
// The flagged 99999s outlier must not shape the median.
|
||||
Assert.Equal(100, aggregate.SegmentStats.Single(s => s.SegmentKey == TimeAttributionSegments.EspApps).MedianSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_AllFlaggedDay_YieldsDisclosureRow_WithoutStats()
|
||||
{
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>
|
||||
{
|
||||
(Session(TenantA, T0), Breakdown(flags: TimeAttributionFlags.ClockSkewDropped)),
|
||||
};
|
||||
|
||||
var aggregate = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0)
|
||||
.Single(a => a.TenantId == TenantA);
|
||||
|
||||
Assert.Equal(0, aggregate.CleanSessionCount);
|
||||
Assert.Equal(1, aggregate.FlaggedExcludedCount);
|
||||
Assert.Empty(aggregate.SegmentStats); // no clean sessions → no statistical claim
|
||||
Assert.Empty(aggregate.TopBlockingApps);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_BlockingOnlyFlags_StayInSegmentStats()
|
||||
{
|
||||
// BlockingSetUnknown/Truncated limit per-app blocking EVIDENCE, not the measured spans.
|
||||
// Gating on any flag starved the fleet aggregates in production (903/1000 breakdowns
|
||||
// carried BlockingSetUnknown, 2026-07-27) — blocking-only flagged sessions must count
|
||||
// as clean; only duration-critical flags exclude.
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>
|
||||
{
|
||||
(Session(TenantA, T0), Breakdown(flags: TimeAttributionFlags.BlockingSetUnknown, espAppsSeconds: 100)),
|
||||
(Session(TenantA, T0), Breakdown(flags: TimeAttributionFlags.BlockingSetTruncated, espAppsSeconds: 300)),
|
||||
(Session(TenantA, T0), Breakdown(
|
||||
flags: TimeAttributionFlags.BlockingSetUnknown | TimeAttributionFlags.PartialObservation,
|
||||
espAppsSeconds: 99999)), // duration-critical bit set → still excluded
|
||||
};
|
||||
|
||||
var aggregate = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0)
|
||||
.Single(a => a.TenantId == TenantA);
|
||||
|
||||
Assert.Equal(2, aggregate.CleanSessionCount);
|
||||
Assert.Equal(1, aggregate.FlaggedExcludedCount);
|
||||
// Median of {100, 300} = 100 (nearest-rank, lower value — repo convention).
|
||||
Assert.Equal(100, aggregate.SegmentStats.Single(s => s.SegmentKey == TimeAttributionSegments.EspApps).MedianSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_MissingBreakdowns_AreCounted_NeverGuessed()
|
||||
{
|
||||
var missing = new List<SessionSummary> { Session(TenantA, T0) };
|
||||
|
||||
var aggregate = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
Array.Empty<(SessionSummary, SessionTimeBreakdown)>(), missing, T0)
|
||||
.Single(a => a.TenantId == TenantA);
|
||||
|
||||
Assert.Equal(0, aggregate.CleanSessionCount);
|
||||
Assert.Equal(1, aggregate.MissingBreakdownCount);
|
||||
Assert.Empty(aggregate.SegmentStats);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_SegmentStack_AlwaysCarriesAllSegmentsPlusUnattributed_AbsentAsZero()
|
||||
{
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>
|
||||
{
|
||||
(Session(TenantA, T0), Breakdown(unattributed: 42)),
|
||||
};
|
||||
|
||||
var aggregate = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0)
|
||||
.Single(a => a.TenantId == TenantA);
|
||||
|
||||
Assert.Equal(6, aggregate.SegmentStats.Count); // 5 canonical + unattributed — the full stack
|
||||
// The fixture has no user_esp span → honest 0, not absent.
|
||||
Assert.Equal(0, aggregate.SegmentStats.Single(s => s.SegmentKey == TimeAttributionSegments.UserEsp).MedianSeconds);
|
||||
Assert.Equal(42, aggregate.SegmentStats.Single(s => s.SegmentKey == TimeAttributionSegments.Unattributed).MedianSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_RollingDateKey_MergesAllDatesIntoRangeStatistics()
|
||||
{
|
||||
// Two sessions on DIFFERENT days — the rolling row must merge them (a median of
|
||||
// per-day medians would not be the range median; the rolling rows are the honest
|
||||
// range statistics the fleet panel reads).
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>
|
||||
{
|
||||
(Session(TenantA, T0), Breakdown(espAppsSeconds: 100)),
|
||||
(Session(TenantA, T0.AddDays(-3)), Breakdown(espAppsSeconds: 300)),
|
||||
};
|
||||
|
||||
var rolling = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0, MaintenanceService.RollingAggregateDateKey);
|
||||
|
||||
var row = rolling.Single(a => a.TenantId == TenantA);
|
||||
Assert.Equal(MaintenanceService.RollingAggregateDateKey, row.Date);
|
||||
Assert.Equal(2, row.CleanSessionCount);
|
||||
// Rolling rows sort AFTER every "yyyy-MM-dd|…" key so date-range reads and the
|
||||
// age-based retention filter never touch them.
|
||||
Assert.True(string.CompareOrdinal($"{row.Date}|user_driven", "2026-12-31|zzz") > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregates_PerAppRows_GateAtFiveSessions_AndCarryWhatIfBounds()
|
||||
{
|
||||
var pairs = new List<(SessionSummary, SessionTimeBreakdown)>();
|
||||
// App X in 5 sessions (ends last each time → saving = its full tail over device-prep-only baseline).
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
pairs.Add((Session(TenantA, T0), Breakdown(apps: new List<BlockingAppInterval>
|
||||
{
|
||||
Interval(AppX, T0.AddMinutes(5), T0.AddMinutes(15)), // only measured blocking app → saving 600
|
||||
})));
|
||||
}
|
||||
// App Y in only 4 sessions → below the per-app row gate.
|
||||
for (var i = 0; i < 4; i++)
|
||||
{
|
||||
pairs.Add((Session(TenantA, T0), Breakdown(apps: new List<BlockingAppInterval>
|
||||
{
|
||||
Interval(AppY, T0.AddMinutes(5), T0.AddMinutes(6)),
|
||||
})));
|
||||
}
|
||||
|
||||
var aggregate = MaintenanceService.BuildTimeAttributionAggregates(
|
||||
pairs, Array.Empty<SessionSummary>(), T0)
|
||||
.Single(a => a.TenantId == TenantA);
|
||||
|
||||
var row = Assert.Single(aggregate.TopBlockingApps);
|
||||
Assert.Equal(AppX, row.AppId);
|
||||
Assert.Equal(5, row.SessionCount);
|
||||
Assert.Equal(600, row.MedianSeconds);
|
||||
Assert.Equal(600, row.MedianSavingSeconds); // sole blocking app → bound = full interval
|
||||
}
|
||||
|
||||
// ── stale-bucket reconcile key consistency (Codex review) ───────────────
|
||||
|
||||
[Fact]
|
||||
public void RollingAggregateEntity_RowKey_MatchesTargetedDeleteKey()
|
||||
{
|
||||
// The sweep's reconcile deletes by (Date, EnrollmentClass) — the entity builder and the
|
||||
// delete path must agree on the RK shape for BOTH daily and rolling rows, or stale
|
||||
// rolling rows would never be removed.
|
||||
var rolling = new TimeAttributionDailyAggregate
|
||||
{
|
||||
TenantId = TenantA, Date = "rolling30", EnrollmentClass = "whiteglove",
|
||||
};
|
||||
Assert.Equal("rolling30|whiteglove", TableStorageService.BuildTimeAttributionAggregateEntity(rolling).RowKey);
|
||||
|
||||
var daily = new TimeAttributionDailyAggregate
|
||||
{
|
||||
TenantId = TenantA, Date = "2026-07-27", EnrollmentClass = "user_driven",
|
||||
};
|
||||
Assert.Equal("2026-07-27|user_driven", TableStorageService.BuildTimeAttributionAggregateEntity(daily).RowKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimeAttribution_InclusiveWindowStart_YieldsExactly30CalendarDays()
|
||||
{
|
||||
// Codex review: both range ends are inclusive — subtracting the full WindowDays
|
||||
// returned 31 day keys under a "windowDays: 30" label.
|
||||
var today = new System.DateTime(2026, 7, 27);
|
||||
Assert.Equal(today.AddDays(-29),
|
||||
Functions.Metrics.TimeAttributionResponse.InclusiveWindowStart(today, Functions.Metrics.TimeAttributionResponse.WindowDays));
|
||||
}
|
||||
}
|
||||
+211
-5
@@ -1,6 +1,8 @@
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -9,7 +11,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{
|
||||
/// <summary>
|
||||
/// Table Storage implementation of <see cref="IHardwareRejectionNotificationTracker"/>.
|
||||
/// PartitionKey = tenantId (lowercased), RowKey = "{manufacturer-lower}|{model-lower}" (trimmed).
|
||||
/// PartitionKey = tenantId (lowercased). RowKey encodes the dedup subject:
|
||||
/// - hardware rejection: "{manufacturer-lower}|{model-lower}" (trimmed)
|
||||
/// - TPM PSS unsupported: "tpmpss|{serial-lower}" (trimmed; the literal "tpmpss" prefix
|
||||
/// cannot collide with a hardware key because no real manufacturer is named "tpmpss")
|
||||
/// - F3 rule regression: "ruleregression|{ruleId-lower}" (trimmed; payload-carrying rows
|
||||
/// that are refreshed while the episode is active and deleted on re-arm)
|
||||
/// Race-safe via AddEntityAsync: Azure Table Storage returns 409 Conflict if the entity already exists.
|
||||
/// </summary>
|
||||
public class TableHardwareRejectionNotificationTracker : IHardwareRejectionNotificationTracker
|
||||
@@ -68,14 +75,140 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> TryRegisterFirstTpmPssNotificationAsync(string tenantId, string serialNumber)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(serialNumber))
|
||||
return false;
|
||||
|
||||
var partitionKey = tenantId.ToLowerInvariant();
|
||||
var rowKey = BuildTpmPssRowKey(serialNumber);
|
||||
|
||||
var entity = new TableEntity(partitionKey, rowKey)
|
||||
{
|
||||
["TenantId"] = tenantId,
|
||||
["SerialNumber"] = serialNumber,
|
||||
["FirstNotifiedAt"] = DateTime.UtcNow
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
await _table.AddEntityAsync(entity);
|
||||
_logger.LogInformation(
|
||||
"TpmPssUnsupported tracker registered: tenant={TenantId} sn={SerialNumber}",
|
||||
tenantId, serialNumber);
|
||||
return true;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
// Already notified for this (tenant, serial) — no second bell.
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"TpmPssUnsupported tracker failed: tenant={TenantId} sn={SerialNumber}",
|
||||
tenantId, serialNumber);
|
||||
// On unexpected failure, return false so we do not double-fire if the row was actually written.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> TryRegisterRuleRegressionAsync(string tenantId, RuleRegressionAlert alert)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(alert?.RuleId))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
await _table.AddEntityAsync(BuildRuleRegressionEntity(tenantId, alert!));
|
||||
_logger.LogInformation(
|
||||
"RuleRegression tracker registered: tenant={TenantId} rule={RuleId}", tenantId, alert!.RuleId);
|
||||
return true;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
// Episode already active — one bell per episode.
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"RuleRegression tracker failed: tenant={TenantId} rule={RuleId}", tenantId, alert!.RuleId);
|
||||
// Fail closed so we do not double-fire if the row was actually written.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RefreshRuleRegressionAsync(string tenantId, RuleRegressionAlert alert)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(alert?.RuleId))
|
||||
return;
|
||||
try
|
||||
{
|
||||
// Replace-mode upsert with the FULL payload incl. the caller-carried
|
||||
// FirstNotifiedAt — the retention sweep re-arms on the ORIGINAL age, so a
|
||||
// refresh must never rejuvenate the row.
|
||||
await _table.UpsertEntityAsync(BuildRuleRegressionEntity(tenantId, alert!), TableUpdateMode.Replace);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"RuleRegression tracker refresh failed: tenant={TenantId} rule={RuleId} (stale numbers remain)",
|
||||
tenantId, alert!.RuleId);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteRuleRegressionAsync(string tenantId, string ruleId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(ruleId))
|
||||
return;
|
||||
try
|
||||
{
|
||||
await _table.DeleteEntityAsync(tenantId.ToLowerInvariant(), BuildRuleRegressionRowKey(ruleId));
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Already gone (retention sweep or concurrent pass) — idempotent.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"RuleRegression tracker delete failed: tenant={TenantId} rule={RuleId} (episode stays until retention)",
|
||||
tenantId, ruleId);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<RuleRegressionAlert>> GetRuleRegressionsAsync(string tenantId)
|
||||
{
|
||||
var results = new List<RuleRegressionAlert>();
|
||||
if (string.IsNullOrWhiteSpace(tenantId)) return results;
|
||||
try
|
||||
{
|
||||
var partitionKey = tenantId.ToLowerInvariant();
|
||||
// '}' (0x7D) sorts directly after '|' (0x7C) — prefix range over "ruleregression|…".
|
||||
var filter = $"PartitionKey eq '{partitionKey}' and RowKey ge '{RuleRegressionRowKeyPrefix}' and RowKey lt 'ruleregression}}'";
|
||||
await foreach (var entity in _table.QueryAsync<TableEntity>(filter: filter))
|
||||
{
|
||||
results.Add(MapToRuleRegressionAlert(entity));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "RuleRegression tracker list failed for tenant {TenantId}", tenantId);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
public async Task<int> DeleteOlderThanAsync(DateTime cutoffUtc)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Rows are insert-once (AddEntityAsync, never updated), so FirstNotifiedAt == creation time.
|
||||
// Pruning here resets the lifetime dedup for a given model: if the same model is rejected
|
||||
// again after the cutoff, the bell fires once more — acceptable, since the portal only
|
||||
// surfaces recent rejections anyway.
|
||||
// FirstNotifiedAt == the row's first registration (rule-regression refreshes carry
|
||||
// it forward unchanged). Pruning here resets the dedup for EVERY key space in this
|
||||
// table: a hardware model rejected again after the cutoff rings once more, so does
|
||||
// a TPM-PSS device that reports again ("tpmpss|{serial}" rows), and a rule
|
||||
// regression still active after the window fires a fresh bell — a month-old
|
||||
// still-burning regression is worth a reminder (spec §F3 retention re-arm).
|
||||
var filter = $"FirstNotifiedAt lt datetime'{cutoffUtc:yyyy-MM-ddTHH:mm:ss}Z'";
|
||||
var query = _table.QueryAsync<TableEntity>(filter: filter, select: new[] { "PartitionKey", "RowKey" });
|
||||
|
||||
@@ -111,5 +244,78 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
var mdl = (model ?? string.Empty).Trim().ToLowerInvariant();
|
||||
return $"{mfr}|{mdl}";
|
||||
}
|
||||
|
||||
internal static string BuildTpmPssRowKey(string serialNumber)
|
||||
{
|
||||
var sn = (serialNumber ?? string.Empty).Trim().ToLowerInvariant();
|
||||
return $"tpmpss|{sn}";
|
||||
}
|
||||
|
||||
internal const string RuleRegressionRowKeyPrefix = "ruleregression|";
|
||||
|
||||
internal static string BuildRuleRegressionRowKey(string ruleId)
|
||||
{
|
||||
var id = (ruleId ?? string.Empty).Trim().ToLowerInvariant();
|
||||
return $"{RuleRegressionRowKeyPrefix}{id}";
|
||||
}
|
||||
|
||||
// internal static: entity builder + mapper are pinned by round-trip unit tests
|
||||
// (table-storage-serialization rule — every property must survive Store→Map).
|
||||
internal static TableEntity BuildRuleRegressionEntity(string tenantId, RuleRegressionAlert alert)
|
||||
{
|
||||
var entity = new TableEntity(tenantId.ToLowerInvariant(), BuildRuleRegressionRowKey(alert.RuleId))
|
||||
{
|
||||
["TenantId"] = tenantId,
|
||||
["RuleId"] = alert.RuleId,
|
||||
["RuleTitle"] = alert.RuleTitle ?? string.Empty,
|
||||
["WindowFireCount"] = alert.WindowFireCount,
|
||||
["WindowSessionCount"] = alert.WindowSessionCount,
|
||||
["BaselineFireCount"] = alert.BaselineFireCount,
|
||||
["BaselineSessionCount"] = alert.BaselineSessionCount,
|
||||
["WindowRatePct"] = alert.WindowRatePct,
|
||||
["BaselineRatePct"] = alert.BaselineRatePct,
|
||||
["WindowStartDate"] = alert.WindowStartDate ?? string.Empty,
|
||||
["WindowEndDate"] = alert.WindowEndDate ?? string.Empty,
|
||||
["FirstNotifiedAt"] = alert.FirstNotifiedAt,
|
||||
["LastEvaluatedAt"] = alert.LastEvaluatedAt,
|
||||
};
|
||||
// Tri-states: absent column = "no claim" — a zero-baseline alert has no finite lift,
|
||||
// and a missing dimension means "no clear concentration", never a guessed one.
|
||||
if (alert.Lift.HasValue)
|
||||
entity["Lift"] = alert.Lift.Value;
|
||||
if (alert.Dimension != null)
|
||||
entity["DimensionJson"] = JsonSerializer.Serialize(alert.Dimension);
|
||||
return entity;
|
||||
}
|
||||
|
||||
internal static RuleRegressionAlert MapToRuleRegressionAlert(TableEntity entity)
|
||||
{
|
||||
RuleRegressionDimension? dimension = null;
|
||||
var dimensionJson = entity.GetString("DimensionJson");
|
||||
if (!string.IsNullOrEmpty(dimensionJson))
|
||||
{
|
||||
try { dimension = JsonSerializer.Deserialize<RuleRegressionDimension>(dimensionJson!); }
|
||||
catch (JsonException) { dimension = null; } // corrupt column — degrade to "no claim"
|
||||
}
|
||||
|
||||
return new RuleRegressionAlert
|
||||
{
|
||||
TenantId = entity.GetString("TenantId") ?? entity.PartitionKey,
|
||||
RuleId = entity.GetString("RuleId") ?? entity.RowKey.Substring(RuleRegressionRowKeyPrefix.Length),
|
||||
RuleTitle = entity.GetString("RuleTitle") ?? string.Empty,
|
||||
WindowFireCount = entity.GetInt32("WindowFireCount") ?? 0,
|
||||
WindowSessionCount = entity.GetInt32("WindowSessionCount") ?? 0,
|
||||
BaselineFireCount = entity.GetInt32("BaselineFireCount") ?? 0,
|
||||
BaselineSessionCount = entity.GetInt32("BaselineSessionCount") ?? 0,
|
||||
WindowRatePct = entity.GetDouble("WindowRatePct") ?? 0,
|
||||
BaselineRatePct = entity.GetDouble("BaselineRatePct") ?? 0,
|
||||
Lift = entity.GetDouble("Lift"),
|
||||
WindowStartDate = entity.GetString("WindowStartDate") ?? string.Empty,
|
||||
WindowEndDate = entity.GetString("WindowEndDate") ?? string.Empty,
|
||||
Dimension = dimension,
|
||||
FirstNotifiedAt = entity.GetDateTimeOffset("FirstNotifiedAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
LastEvaluatedAt = entity.GetDateTimeOffset("LastEvaluatedAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
@@ -99,6 +99,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
|
||||
=> _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId);
|
||||
|
||||
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync()
|
||||
=> _storage.GetAllSessionTombstoneKeysAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Tenant offboarding cascade-worker session enumerator. Yields one row per session
|
||||
/// belonging to <paramref name="tenantId"/>. Storage exceptions are intentionally
|
||||
|
||||
+48
@@ -111,5 +111,53 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
public Task<int> DeleteRuleStatsOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteRuleStatsOlderThanAsync(cutoffDate);
|
||||
|
||||
public Task<SessionTimeBreakdown?> ComputeAndStoreSessionTimeBreakdownAsync(string tenantId, string sessionId)
|
||||
=> _storage.ComputeAndStoreSessionTimeBreakdownAsync(tenantId, sessionId);
|
||||
|
||||
public Task<SessionTimeBreakdown?> GetSessionTimeBreakdownAsync(string tenantId, string sessionId)
|
||||
=> _storage.GetSessionTimeBreakdownAsync(tenantId, sessionId);
|
||||
|
||||
public Task ResolveEspBlockingForSessionAsync(string tenantId, string sessionId)
|
||||
=> _storage.ResolveEspBlockingForSessionAsync(tenantId, sessionId);
|
||||
|
||||
public Task<bool> SaveTimeAttributionAggregateAsync(TimeAttributionDailyAggregate aggregate)
|
||||
=> _storage.SaveTimeAttributionAggregateAsync(aggregate);
|
||||
|
||||
public Task<List<TimeAttributionDailyAggregate>> GetTimeAttributionAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate)
|
||||
=> _storage.GetTimeAttributionAggregatesAsync(tenantId, startDate, endDate);
|
||||
|
||||
public Task<List<TimeAttributionDailyAggregate>> GetRollingTimeAttributionAggregatesAsync(string tenantId)
|
||||
=> _storage.GetRollingTimeAttributionAggregatesAsync(tenantId);
|
||||
|
||||
public Task DeleteTimeAttributionAggregateAsync(string tenantId, string dateKey, string enrollmentClass)
|
||||
=> _storage.DeleteTimeAttributionAggregateAsync(tenantId, dateKey, enrollmentClass);
|
||||
|
||||
public Task<int> DeleteTimeAttributionAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteTimeAttributionAggregatesOlderThanAsync(cutoffDate);
|
||||
|
||||
public Task<DeviceHistory?> GetDeviceHistoryAsync(string tenantId, string serialKey)
|
||||
=> _storage.GetDeviceHistoryAsync(tenantId, serialKey);
|
||||
|
||||
public Task<List<DeviceHistory>> GetDeviceHistoriesByTenantAsync(string tenantId)
|
||||
=> _storage.GetDeviceHistoriesByTenantAsync(tenantId);
|
||||
|
||||
public Task<bool> UpsertDeviceHistoryAsync(DeviceHistory history)
|
||||
=> _storage.UpsertDeviceHistoryAsync(history);
|
||||
|
||||
public Task DeleteDeviceHistoryAsync(string tenantId, string serialKey)
|
||||
=> _storage.DeleteDeviceHistoryAsync(tenantId, serialKey);
|
||||
|
||||
public Task<bool> SaveDeviceJourneyAggregateAsync(DeviceJourneyDailyAggregate aggregate)
|
||||
=> _storage.SaveDeviceJourneyAggregateAsync(aggregate);
|
||||
|
||||
public Task<List<DeviceJourneyDailyAggregate>> GetDeviceJourneyAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate)
|
||||
=> _storage.GetDeviceJourneyAggregatesAsync(tenantId, startDate, endDate);
|
||||
|
||||
public Task DeleteDeviceJourneyAggregateAsync(string tenantId, string dateKey)
|
||||
=> _storage.DeleteDeviceJourneyAggregateAsync(tenantId, dateKey);
|
||||
|
||||
public Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteDeviceJourneyAggregatesOlderThanAsync(cutoffDate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<List<RuleResult>> GetRuleResultsAsync(string tenantId, string sessionId)
|
||||
=> _storage.GetRuleResultsAsync(tenantId, sessionId);
|
||||
|
||||
public Task<List<string>> GetRuleHitSessionIdsAsync(string tenantId, string ruleId, DateTime sinceUtc, int maxResults = 2000)
|
||||
=> _storage.GetRuleHitSessionIdsAsync(tenantId, ruleId, sinceUtc, maxResults);
|
||||
|
||||
public Task<bool> StoreGatherRuleAsync(GatherRule rule, string tenantId = "global")
|
||||
=> _storage.StoreGatherRuleAsync(rule, tenantId);
|
||||
|
||||
|
||||
+29
@@ -84,7 +84,32 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
// otherwise a swept-Incomplete session keeps whatever stale live count it had.
|
||||
// Idempotent + fail-soft, so the redundancy is cheap.
|
||||
if (transitioned && (status == SessionStatus.Succeeded || status == SessionStatus.Failed || status == SessionStatus.Incomplete))
|
||||
{
|
||||
await _storage.ReconcileSessionCountersAsync(tenantId, sessionId);
|
||||
// F1 PR1: same single-writer seam — join the session's app rows against its
|
||||
// latest esp_config_detected lists exactly once, when the event stream is
|
||||
// complete. Positive evidence only (listed ⇒ EspBlocking=true, absent ⇒ stays
|
||||
// unknown). Idempotent + fail-soft like the counter reconcile above.
|
||||
await _storage.ResolveEspBlockingForSessionAsync(tenantId, sessionId);
|
||||
|
||||
// F1 PR2: compute + persist the time-attribution breakdown once, now that the
|
||||
// terminal write stamped CompletedAt/DurationSeconds and the event stream is
|
||||
// complete. Succeeded/Failed only — Incomplete deliberately stores no duration,
|
||||
// so there is no wall clock to partition (the compute would no-op anyway; the
|
||||
// gate just saves the point-reads). Fail-soft; the 30d maintenance sweep
|
||||
// self-heals any miss.
|
||||
if (status == SessionStatus.Succeeded || status == SessionStatus.Failed)
|
||||
await _storage.ComputeAndStoreSessionTimeBreakdownAsync(tenantId, sessionId);
|
||||
|
||||
// F2 PR4: upsert this terminal session into its device's history chain at the
|
||||
// same seam, so the session-detail "Attempt N" banner is fresh rather than a
|
||||
// sweep-cycle old. ALL three terminal statuses — Incomplete is a non-successful
|
||||
// journey attempt, unlike the duration-gated breakdown above. Re-terminal
|
||||
// reclassifications (admin mark, late completion) flow through here too and
|
||||
// update the chain entry's status. Fail-soft + idempotent; the maintenance
|
||||
// sweep self-heals any miss.
|
||||
await _storage.UpdateDeviceHistoryForSessionAsync(tenantId, sessionId);
|
||||
}
|
||||
|
||||
return transitioned;
|
||||
}
|
||||
@@ -123,6 +148,10 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task UpdateSessionImeAgentVersionAsync(string tenantId, string sessionId, string version)
|
||||
=> _storage.UpdateSessionImeAgentVersionAsync(tenantId, sessionId, version);
|
||||
|
||||
public Task UpdateSessionNetworkLatencyAsync(string tenantId, string sessionId,
|
||||
double avgLatencyMs, int requestCount)
|
||||
=> _storage.UpdateSessionNetworkLatencyAsync(tenantId, sessionId, avgLatencyMs, requestCount);
|
||||
|
||||
public Task<List<SessionSummary>> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold)
|
||||
=> _storage.GetSessionsWithEventCountAboveAsync(tenantId, threshold);
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Specialized;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
@@ -140,42 +141,39 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
|
||||
var summaries = allSummaries.Where(s => s.StartedAt >= cutoff).ToList();
|
||||
|
||||
var apps = summaries.GroupBy(s => s.AppName).Select(g =>
|
||||
{
|
||||
var total = g.Count();
|
||||
var succeeded = g.Count(s => s.Status == "Succeeded");
|
||||
var failed = g.Count(s => s.Status == "Failed");
|
||||
var completed = g.Where(s => s.Status == "Succeeded").ToList();
|
||||
var failureRate = total > 0 ? Math.Round((double)failed / total * 100, 1) : 0;
|
||||
// F1 PR1 (audit Q3): appId-collision rows merge two distinct apps under one name —
|
||||
// they are excluded from every per-app group and disclosed via collisionExcluded.
|
||||
var collisionExcluded = summaries.Count(s => s.AppIdCollision);
|
||||
|
||||
var apps = summaries.Where(s => !s.AppIdCollision).GroupBy(s => s.AppName).Select(g =>
|
||||
{
|
||||
// PR0 classification (see MetricsMath.IsSkipTerminalState / HasMeasuredDuration):
|
||||
// skips leave the rate + durations; duration stats read measured rows only.
|
||||
var total = g.Count();
|
||||
var succeededAll = g.Where(s => s.Status == "Succeeded").ToList();
|
||||
var skipped = succeededAll.Count(MetricsMath.IsSkipTerminalState);
|
||||
var installed = succeededAll.Where(s => !MetricsMath.IsSkipTerminalState(s)).ToList();
|
||||
var measured = installed.Where(MetricsMath.HasMeasuredDuration).ToList();
|
||||
var failed = g.Count(s => s.Status == "Failed");
|
||||
var failureRate = MetricsMath.TerminalFailureRatePct(failed, installed.Count);
|
||||
|
||||
// Trend: compare failure rate in first half vs second half of window.
|
||||
// Only emit non-stable trend if BOTH halves have >= 5 installs (too noisy otherwise).
|
||||
var firstHalf = g.Where(s => s.StartedAt < midpoint).ToList();
|
||||
var secondHalf = g.Where(s => s.StartedAt >= midpoint).ToList();
|
||||
|
||||
double? trendDelta = null;
|
||||
string trend = "stable";
|
||||
if (firstHalf.Count >= 5 && secondHalf.Count >= 5)
|
||||
{
|
||||
var fhRate = (double)firstHalf.Count(s => s.Status == "Failed") / firstHalf.Count * 100;
|
||||
var shRate = (double)secondHalf.Count(s => s.Status == "Failed") / secondHalf.Count * 100;
|
||||
var delta = Math.Round(shRate - fhRate, 1);
|
||||
trendDelta = delta;
|
||||
if (delta < -1) trend = "improving";
|
||||
else if (delta > 1) trend = "worsening";
|
||||
}
|
||||
var (trend, trendDelta) = ComputeFailureTrend(firstHalf, secondHalf);
|
||||
|
||||
return new
|
||||
{
|
||||
appName = g.Key,
|
||||
appType = g.Select(s => s.AppType).FirstOrDefault(t => !string.IsNullOrEmpty(t)) ?? string.Empty,
|
||||
totalInstalls = total,
|
||||
succeeded,
|
||||
succeeded = installed.Count,
|
||||
skipped,
|
||||
unmeasured = installed.Count - measured.Count,
|
||||
failed,
|
||||
failureRate,
|
||||
avgDurationSeconds = completed.Count > 0 ? Math.Round(completed.Average(s => s.DurationSeconds), 0) : 0,
|
||||
maxDurationSeconds = completed.Count > 0 ? completed.Max(s => s.DurationSeconds) : 0,
|
||||
avgDownloadBytes = completed.Count > 0 ? (long)completed.Average(s => s.DownloadBytes) : 0,
|
||||
avgDurationSeconds = measured.Count > 0 ? Math.Round(measured.Average(s => s.DurationSeconds), 0) : 0,
|
||||
maxDurationSeconds = measured.Count > 0 ? measured.Max(s => s.DurationSeconds) : 0,
|
||||
avgDownloadBytes = measured.Count > 0 ? (long)measured.Average(s => s.DownloadBytes) : 0,
|
||||
trend,
|
||||
trendDelta,
|
||||
lastSeenAt = g.Max(s => s.CompletedAt ?? s.StartedAt)
|
||||
@@ -194,6 +192,7 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
success = true,
|
||||
totalApps = apps.Count,
|
||||
totalInstalls = summaries.Count,
|
||||
collisionExcluded,
|
||||
windowDays = days,
|
||||
apps
|
||||
};
|
||||
@@ -210,6 +209,7 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
success = true,
|
||||
totalApps = apps.Count,
|
||||
totalInstalls = summaries.Count,
|
||||
collisionExcluded,
|
||||
windowDays = days,
|
||||
count = page.Count,
|
||||
offset,
|
||||
@@ -235,11 +235,16 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
var cutoff = now.AddDays(-days);
|
||||
var midpoint = now.AddDays(-days / 2.0);
|
||||
|
||||
var summaries = allSummaries
|
||||
var inWindow = allSummaries
|
||||
.Where(s => string.Equals(s.AppName, appName, StringComparison.OrdinalIgnoreCase)
|
||||
&& s.StartedAt >= cutoff)
|
||||
.ToList();
|
||||
|
||||
// F1 PR1 (audit Q3): collision rows under this name mix in a second app's outcomes —
|
||||
// excluded from every stat below, disclosed via collisionExcluded.
|
||||
var collisionExcluded = inWindow.Count(s => s.AppIdCollision);
|
||||
var summaries = inWindow.Where(s => !s.AppIdCollision).ToList();
|
||||
|
||||
if (summaries.Count == 0)
|
||||
{
|
||||
return new
|
||||
@@ -248,11 +253,14 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
appName,
|
||||
appType = string.Empty,
|
||||
windowDays = days,
|
||||
collisionExcluded,
|
||||
bucket = "day",
|
||||
summary = new
|
||||
{
|
||||
totalInstalls = 0,
|
||||
succeeded = 0,
|
||||
skipped = 0,
|
||||
unmeasured = 0,
|
||||
failed = 0,
|
||||
failureRate = 0,
|
||||
avgDurationSeconds = 0,
|
||||
@@ -272,27 +280,22 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
}
|
||||
|
||||
var total = summaries.Count;
|
||||
var succeeded = summaries.Count(s => s.Status == "Succeeded");
|
||||
// PR0 classification — skips leave rate + durations; durations read measured rows only.
|
||||
var succeededAll = summaries.Where(s => s.Status == "Succeeded").ToList();
|
||||
var skipped = succeededAll.Count(MetricsMath.IsSkipTerminalState);
|
||||
var installed = succeededAll.Where(s => !MetricsMath.IsSkipTerminalState(s)).ToList();
|
||||
var measured = installed.Where(MetricsMath.HasMeasuredDuration).ToList();
|
||||
var succeeded = installed.Count;
|
||||
var failed = summaries.Count(s => s.Status == "Failed");
|
||||
var completed = summaries.Where(s => s.Status == "Succeeded").ToList();
|
||||
var failureRate = Math.Round((double)failed / total * 100, 1);
|
||||
var avgDurationSeconds = completed.Count > 0 ? Math.Round(completed.Average(s => s.DurationSeconds), 0) : 0;
|
||||
var p95DurationSeconds = Percentile(completed.Select(s => s.DurationSeconds).ToList(), 0.95);
|
||||
var avgDownloadBytes = completed.Count > 0 ? (long)completed.Average(s => s.DownloadBytes) : 0;
|
||||
var failureRate = MetricsMath.TerminalFailureRatePct(failed, succeeded);
|
||||
var avgDurationSeconds = measured.Count > 0 ? Math.Round(measured.Average(s => s.DurationSeconds), 0) : 0;
|
||||
var p95DurationSeconds = Percentile(measured.Select(s => s.DurationSeconds).ToList(), 0.95);
|
||||
var avgDownloadBytes = measured.Count > 0 ? (long)measured.Average(s => s.DownloadBytes) : 0;
|
||||
|
||||
// Trend (same rule as list endpoint).
|
||||
var firstHalf = summaries.Where(s => s.StartedAt < midpoint).ToList();
|
||||
var secondHalf = summaries.Where(s => s.StartedAt >= midpoint).ToList();
|
||||
double? trendDelta = null;
|
||||
string trend = "stable";
|
||||
if (firstHalf.Count >= 5 && secondHalf.Count >= 5)
|
||||
{
|
||||
var fhRate = (double)firstHalf.Count(s => s.Status == "Failed") / firstHalf.Count * 100;
|
||||
var shRate = (double)secondHalf.Count(s => s.Status == "Failed") / secondHalf.Count * 100;
|
||||
trendDelta = Math.Round(shRate - fhRate, 1);
|
||||
if (trendDelta < -1) trend = "improving";
|
||||
else if (trendDelta > 1) trend = "worsening";
|
||||
}
|
||||
var (trend, trendDelta) = ComputeFailureTrend(firstHalf, secondHalf);
|
||||
|
||||
var flakinessScore = total > 0
|
||||
? Math.Round((double)summaries.Count(s => s.AttemptNumber > 1) / total, 3)
|
||||
@@ -308,12 +311,14 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
{
|
||||
var vTotal = g.Count();
|
||||
var vFailed = g.Count(s => s.Status == "Failed");
|
||||
// Same PR0 convention as the top-level rate: skips are not attempts.
|
||||
var vSucceeded = g.Count(s => s.Status == "Succeeded" && !MetricsMath.IsSkipTerminalState(s));
|
||||
return new
|
||||
{
|
||||
appVersion = g.Key,
|
||||
installs = vTotal,
|
||||
failed = vFailed,
|
||||
failureRate = vTotal > 0 ? Math.Round((double)vFailed / vTotal * 100, 1) : 0
|
||||
failureRate = MetricsMath.TerminalFailureRatePct(vFailed, vSucceeded)
|
||||
};
|
||||
})
|
||||
.OrderByDescending(v => v.installs)
|
||||
@@ -359,12 +364,17 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
Model = sessionLookup[$"{s.TenantId}|{s.SessionId}"].Model ?? "Unknown"
|
||||
})
|
||||
.GroupBy(x => new { x.Manufacturer, x.Model })
|
||||
.Where(g => g.Count() >= 5)
|
||||
// Sample floor on FINISHED installs — the displayed metric is the terminal-only
|
||||
// failure rate, so a model with many in-flight installs but few outcomes is still
|
||||
// too noisy to rank.
|
||||
.Where(g => g.Count(x => x.Summary.Status == "Failed" || x.Summary.Status == "Succeeded") >= 5)
|
||||
.Select(g =>
|
||||
{
|
||||
var modelTotal = g.Count();
|
||||
var modelFailed = g.Count(x => x.Summary.Status == "Failed");
|
||||
var modelFailureRate = Math.Round((double)modelFailed / modelTotal * 100, 1);
|
||||
// PR0 convention: skips are not attempts (same population as the headline rate).
|
||||
var modelSucceeded = g.Count(x => x.Summary.Status == "Succeeded" && !MetricsMath.IsSkipTerminalState(x.Summary));
|
||||
var modelFailureRate = MetricsMath.TerminalFailureRatePct(modelFailed, modelSucceeded);
|
||||
var lift = failureRate > 0
|
||||
? Math.Round(modelFailureRate / failureRate, 2)
|
||||
: 0;
|
||||
@@ -389,11 +399,14 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
appName,
|
||||
appType,
|
||||
windowDays = days,
|
||||
collisionExcluded,
|
||||
bucket,
|
||||
summary = new
|
||||
{
|
||||
totalInstalls = total,
|
||||
succeeded,
|
||||
skipped,
|
||||
unmeasured = installed.Count - measured.Count,
|
||||
failed,
|
||||
failureRate,
|
||||
avgDurationSeconds,
|
||||
@@ -496,6 +509,33 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Failure-rate trend between the two halves of the window, shared by the list and
|
||||
/// analytics endpoints. Rates follow the terminal-only convention
|
||||
/// (<see cref="MetricsMath.TerminalFailureRatePct"/>), and a non-stable trend requires
|
||||
/// at least 5 FINISHED installs in BOTH halves — too noisy otherwise, and a half made
|
||||
/// up of in-flight installs has no outcome to trend on.
|
||||
/// </summary>
|
||||
private static (string Trend, double? TrendDelta) ComputeFailureTrend(
|
||||
List<AppInstallSummary> firstHalf, List<AppInstallSummary> secondHalf)
|
||||
{
|
||||
// PR0 convention: skips are not attempts — keep the trend on the same population
|
||||
// as the headline failure rate, or a wave of not-applicable update policies would
|
||||
// fake an "improving" trend.
|
||||
var fhFailed = firstHalf.Count(s => s.Status == "Failed");
|
||||
var fhSucceeded = firstHalf.Count(s => s.Status == "Succeeded" && !MetricsMath.IsSkipTerminalState(s));
|
||||
var shFailed = secondHalf.Count(s => s.Status == "Failed");
|
||||
var shSucceeded = secondHalf.Count(s => s.Status == "Succeeded" && !MetricsMath.IsSkipTerminalState(s));
|
||||
if (fhFailed + fhSucceeded < 5 || shFailed + shSucceeded < 5)
|
||||
return ("stable", null);
|
||||
|
||||
var delta = Math.Round(
|
||||
MetricsMath.TerminalFailureRatePct(shFailed, shSucceeded)
|
||||
- MetricsMath.TerminalFailureRatePct(fhFailed, fhSucceeded), 1);
|
||||
var trend = delta < -1 ? "improving" : delta > 1 ? "worsening" : "stable";
|
||||
return (trend, delta);
|
||||
}
|
||||
|
||||
private static List<object> BuildTimeSeries(List<AppInstallSummary> summaries, DateTime cutoff, DateTime now, string bucket)
|
||||
{
|
||||
var start = bucket == "week" ? StartOfWeek(cutoff) : cutoff.Date;
|
||||
@@ -523,16 +563,17 @@ namespace AutopilotMonitor.Functions.Functions.Apps
|
||||
var items = kv.Value;
|
||||
var bTotal = items.Count;
|
||||
var bFailed = items.Count(s => s.Status == "Failed");
|
||||
var bSucceeded = items.Count(s => s.Status == "Succeeded");
|
||||
var bCompleted = items.Where(s => s.Status == "Succeeded").ToList();
|
||||
// PR0 convention: skips are not attempts; durations read measured rows only.
|
||||
var bInstalled = items.Where(s => s.Status == "Succeeded" && !MetricsMath.IsSkipTerminalState(s)).ToList();
|
||||
var bMeasured = bInstalled.Where(MetricsMath.HasMeasuredDuration).ToList();
|
||||
return (object)new
|
||||
{
|
||||
bucketStart = DateTime.SpecifyKind(kv.Key, DateTimeKind.Utc),
|
||||
installs = bTotal,
|
||||
succeeded = bSucceeded,
|
||||
succeeded = bInstalled.Count,
|
||||
failed = bFailed,
|
||||
failureRate = bTotal > 0 ? Math.Round((double)bFailed / bTotal * 100, 1) : 0,
|
||||
avgDurationSeconds = bCompleted.Count > 0 ? Math.Round(bCompleted.Average(s => s.DurationSeconds), 0) : 0
|
||||
failureRate = MetricsMath.TerminalFailureRatePct(bFailed, bInstalled.Count),
|
||||
avgDurationSeconds = bMeasured.Count > 0 ? Math.Round(bMeasured.Average(s => s.DurationSeconds), 0) : 0
|
||||
};
|
||||
})
|
||||
.ToList();
|
||||
|
||||
+8
-5
@@ -29,7 +29,8 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + TenantAdminOrGA authorization enforced by PolicyEnforcementMiddleware
|
||||
// Authentication + MemberRead authorization enforced by PolicyEnforcementMiddleware —
|
||||
// Operators/Viewers are admitted for a READ-ONLY view (secrets redacted below).
|
||||
var requestCtx = req.GetRequestContext();
|
||||
var userIdentifier = requestCtx.UserPrincipalName;
|
||||
|
||||
@@ -39,8 +40,9 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
|
||||
// Only a caller with WRITE authority over THIS target tenant sees the full config including
|
||||
// DiagnosticsBlobSasUrl / webhook URLs / custom headers (they manage those in the Settings
|
||||
// UI). Everyone else admitted to this read tier — a cross-tenant Global Reader, or a
|
||||
// delegated reader viewing one of their managed tenants — gets the secrets redacted (returns
|
||||
// UI). Everyone else admitted to this read tier — an own-tenant Operator/Viewer, a
|
||||
// cross-tenant Global Reader, or a delegated reader viewing one of their managed
|
||||
// tenants — gets the secrets redacted (returns
|
||||
// a copy; the cached instance is never mutated). Redact-by-default: only Global Admin (any
|
||||
// tenant) or the tenant's OWN admin clears secrets. The own-tenant check (target == JWT
|
||||
// tenant) is what keeps a delegated reader — whose target is a DIFFERENT tenant — redacted
|
||||
@@ -66,8 +68,9 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
/// <summary>
|
||||
/// Redact-by-default secret clearance for the config GET. Only a caller with WRITE authority over
|
||||
/// THIS target tenant sees the unredacted secrets (SAS URLs, webhook URLs, custom headers): a Global
|
||||
/// Admin (any tenant), or the tenant's OWN admin. Everyone else admitted to this read tier — a
|
||||
/// cross-tenant Global Reader, or a delegated reader viewing one of their managed tenants — is
|
||||
/// Admin (any tenant), or the tenant's OWN admin. Everyone else admitted to this read tier — an
|
||||
/// own-tenant Operator/Viewer (read-only Settings view), a cross-tenant Global Reader, or a
|
||||
/// delegated reader viewing one of their managed tenants — is
|
||||
/// redacted. The own-tenant check (target == JWT tenant) is what keeps a delegated reader redacted
|
||||
/// even when they are an admin of their own home tenant, since their target is a DIFFERENT tenant.
|
||||
/// Extracted as a pure static so the security decision is unit-testable without an HttpRequestData mock.
|
||||
|
||||
+8
@@ -76,6 +76,14 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
return new
|
||||
{
|
||||
bootstrapTokenEnabled = config.BootstrapTokenEnabled,
|
||||
// Session-detail "Collect Logs" button: whether an on-demand diagnostics upload can
|
||||
// succeed right now (mode not Off + a usable destination). Members below Admin cannot
|
||||
// read the full config, so this boolean is their only signal. Deliberately exposes no
|
||||
// destination detail — just "would an upload work".
|
||||
diagnosticsUploadConfigured =
|
||||
!string.Equals(config.DiagnosticsUploadMode ?? "Off", "Off", StringComparison.OrdinalIgnoreCase)
|
||||
&& GetAgentConfigFunction.ResolveDiagnosticsUploadEnabled(
|
||||
config.DiagnosticsBlobSasUrl, config.DiagnosticsUploadDestination),
|
||||
// Drives the "Autopilot Device Validation disabled" dashboard banner
|
||||
// (useTenantSecurityConfig).
|
||||
validateAutopilotDevice = config.ValidateAutopilotDevice,
|
||||
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Infrastructure
|
||||
{
|
||||
/// <summary>
|
||||
/// Receives Azure Monitor alert notifications (Common Alert Schema) from an Action Group
|
||||
/// webhook receiver and re-emits them as ops events, so platform alerts flow through the
|
||||
/// same OpsEventService → OpsAlertDispatchService fan-out (Telegram/Teams/Slack providers +
|
||||
/// the portal ops-events list) as every internally raised operational event.
|
||||
///
|
||||
/// The delivery path is deliberately independent of the portal: Azure Monitor → this
|
||||
/// function → providers. A portal (SWA) outage cannot suppress these notifications — the
|
||||
/// action group's e-mail receiver additionally stays as the backend-independent fallback,
|
||||
/// because a self-hosted channel cannot alert on its own outage.
|
||||
///
|
||||
/// Security: anonymous route (Action Group webhook receivers can only call a bare URI — no
|
||||
/// custom headers), gated by a shared secret in the query string compared in constant time
|
||||
/// against the OpsAlertWebhookSecret app setting. Fail-closed: while the setting is unset,
|
||||
/// every request is rejected. Dispatch to providers only happens if an enabled ops alert
|
||||
/// rule for EventType "AzureMonitorAlert" matches the mapped severity — the event itself is
|
||||
/// always stored.
|
||||
/// </summary>
|
||||
public class AzureMonitorAlertWebhookFunction
|
||||
{
|
||||
internal const string SecretSettingName = "OpsAlertWebhookSecret";
|
||||
|
||||
// Common Alert Schema payloads for log-search alerts carry the search query, link URLs
|
||||
// and dimension sets — a few KB in practice. 64 KB is a generous shape gate, not a throttle.
|
||||
internal const int MaxContentLength = 64 * 1024;
|
||||
|
||||
private readonly OpsEventService _opsEventService;
|
||||
private readonly ILogger<AzureMonitorAlertWebhookFunction> _logger;
|
||||
|
||||
public AzureMonitorAlertWebhookFunction(
|
||||
OpsEventService opsEventService,
|
||||
ILogger<AzureMonitorAlertWebhookFunction> logger)
|
||||
{
|
||||
_opsEventService = opsEventService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[Function("AzureMonitorAlertWebhook")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "ops/alert-webhook")] HttpRequestData req)
|
||||
{
|
||||
var configuredSecret = Environment.GetEnvironmentVariable(SecretSettingName);
|
||||
var providedSecret = System.Web.HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty)["secret"];
|
||||
|
||||
if (!SecretMatches(providedSecret, configuredSecret))
|
||||
{
|
||||
// Deliberately no detail in the response; the warning makes a misconfigured
|
||||
// (unset) secret operator-visible in the logs.
|
||||
_logger.LogWarning(
|
||||
"Azure Monitor alert webhook rejected: {Reason}",
|
||||
string.IsNullOrEmpty(configuredSecret) ? "secret not configured (fail-closed)" : "secret mismatch");
|
||||
return req.CreateResponse(HttpStatusCode.Unauthorized);
|
||||
}
|
||||
|
||||
// Bounded read instead of a Body.Length check: with ASP.NET Core integration the
|
||||
// request body is a forward-only stream whose Length property throws
|
||||
// NotSupportedException. Reading at most cap+1 chars detects oversize without ever
|
||||
// needing to know the total size. (Char-based cap ≈ byte cap for a shape gate.)
|
||||
string body;
|
||||
using (var reader = new StreamReader(req.Body, Encoding.UTF8))
|
||||
{
|
||||
var buffer = new char[MaxContentLength + 1];
|
||||
var read = await reader.ReadBlockAsync(buffer, 0, buffer.Length);
|
||||
if (read > MaxContentLength)
|
||||
{
|
||||
_logger.LogWarning("Azure Monitor alert webhook rejected: payload exceeds {Cap} char cap", MaxContentLength);
|
||||
return req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
}
|
||||
body = new string(buffer, 0, read);
|
||||
}
|
||||
|
||||
var alert = TryParse(body);
|
||||
if (alert == null)
|
||||
{
|
||||
_logger.LogWarning("Azure Monitor alert webhook rejected: body is not a Common Alert Schema payload");
|
||||
return req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
}
|
||||
|
||||
await _opsEventService.RecordAzureMonitorAlertAsync(
|
||||
alert.AlertRule,
|
||||
MapSeverity(alert.Severity),
|
||||
alert.MonitorCondition,
|
||||
alert.Description,
|
||||
alert.Severity,
|
||||
alert.MonitoringService,
|
||||
alert.TargetResource,
|
||||
alert.MetricValue);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Azure Monitor alert '{AlertRule}' {Condition} ({Severity}) recorded as ops event",
|
||||
alert.AlertRule, alert.MonitorCondition, alert.Severity);
|
||||
|
||||
return req.CreateResponse(HttpStatusCode.OK);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Constant-time secret comparison. Fail-closed: an unset/empty configured secret never
|
||||
/// matches anything, so the endpoint stays dark until the app setting is provisioned.
|
||||
/// </summary>
|
||||
internal static bool SecretMatches(string? provided, string? configured)
|
||||
{
|
||||
if (string.IsNullOrEmpty(configured) || string.IsNullOrEmpty(provided))
|
||||
return false;
|
||||
|
||||
return CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.UTF8.GetBytes(provided),
|
||||
Encoding.UTF8.GetBytes(configured));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps Azure Monitor severities (Sev0 = most severe … Sev4 = verbose) onto the ops event
|
||||
/// severity scale. Unknown values fall back to Info so a schema drift can never escalate.
|
||||
/// </summary>
|
||||
internal static string MapSeverity(string? azureSeverity) => azureSeverity switch
|
||||
{
|
||||
"Sev0" => Shared.DataAccess.OpsEventSeverity.Critical,
|
||||
"Sev1" => Shared.DataAccess.OpsEventSeverity.Error,
|
||||
"Sev2" => Shared.DataAccess.OpsEventSeverity.Warning,
|
||||
_ => Shared.DataAccess.OpsEventSeverity.Info,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Parsed subset of the Common Alert Schema. Only <see cref="AlertRule"/> is mandatory;
|
||||
/// everything else is best-effort so a partial payload still produces a usable ops event.
|
||||
/// </summary>
|
||||
internal sealed record ParsedAlert(
|
||||
string AlertRule,
|
||||
string? Severity,
|
||||
string MonitorCondition,
|
||||
string? Description,
|
||||
string? MonitoringService,
|
||||
string? TargetResource,
|
||||
double? MetricValue);
|
||||
|
||||
/// <summary>
|
||||
/// Defensively extracts data.essentials (+ the first condition's metricValue) from a
|
||||
/// Common Alert Schema payload. Returns null when the body is not JSON or carries no
|
||||
/// alert rule name — the two states in which no meaningful ops event can be formed.
|
||||
/// </summary>
|
||||
internal static ParsedAlert? TryParse(string body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
if (!doc.RootElement.TryGetProperty("data", out var data) ||
|
||||
!data.TryGetProperty("essentials", out var essentials))
|
||||
return null;
|
||||
|
||||
var alertRule = GetString(essentials, "alertRule");
|
||||
if (string.IsNullOrWhiteSpace(alertRule))
|
||||
return null;
|
||||
|
||||
var targetResource = GetString(essentials, "targetResourceName");
|
||||
if (string.IsNullOrEmpty(targetResource) &&
|
||||
essentials.TryGetProperty("alertTargetIDs", out var targets) &&
|
||||
targets.ValueKind == JsonValueKind.Array && targets.GetArrayLength() > 0 &&
|
||||
targets[0].ValueKind == JsonValueKind.String)
|
||||
{
|
||||
// Fall back to the resource name segment of the first target resource ID.
|
||||
var id = targets[0].GetString() ?? string.Empty;
|
||||
targetResource = id[(id.LastIndexOf('/') + 1)..];
|
||||
}
|
||||
|
||||
// Log-search alerts carry the observed value under alertContext.condition.allOf[0].
|
||||
double? metricValue = null;
|
||||
if (data.TryGetProperty("alertContext", out var ctx) &&
|
||||
ctx.ValueKind == JsonValueKind.Object &&
|
||||
ctx.TryGetProperty("condition", out var condition) &&
|
||||
condition.ValueKind == JsonValueKind.Object &&
|
||||
condition.TryGetProperty("allOf", out var allOf) &&
|
||||
allOf.ValueKind == JsonValueKind.Array && allOf.GetArrayLength() > 0 &&
|
||||
allOf[0].ValueKind == JsonValueKind.Object &&
|
||||
allOf[0].TryGetProperty("metricValue", out var mv) &&
|
||||
mv.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
metricValue = mv.GetDouble();
|
||||
}
|
||||
|
||||
return new ParsedAlert(
|
||||
alertRule!,
|
||||
GetString(essentials, "severity"),
|
||||
GetString(essentials, "monitorCondition") ?? "Fired",
|
||||
GetString(essentials, "description"),
|
||||
GetString(essentials, "monitoringService"),
|
||||
targetResource,
|
||||
metricValue);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? GetString(JsonElement element, string property)
|
||||
=> element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String
|
||||
? value.GetString()
|
||||
: null;
|
||||
}
|
||||
}
|
||||
@@ -269,6 +269,36 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
}
|
||||
|
||||
// In-app bell notification on first-ever TPM-PSS incompatibility per (tenant, serial).
|
||||
// Same lifetime-dedup pattern as the hardware-rejection bell above. No webhook —
|
||||
// volume is tiny and the remediation (TPM firmware update / device replacement)
|
||||
// is not time-critical. Reports without a serial number cannot be deduped and
|
||||
// fire no bell; they still appear in the tenant insights panel.
|
||||
if (report.ErrorType == DistressErrorType.TpmPssUnsupported
|
||||
&& !string.IsNullOrEmpty(serialNumber))
|
||||
{
|
||||
try
|
||||
{
|
||||
var isFirst = await _hardwareBellTracker
|
||||
.TryRegisterFirstTpmPssNotificationAsync(tenantId, serialNumber);
|
||||
if (isFirst)
|
||||
{
|
||||
var deviceLabel = string.Join(" ",
|
||||
new[] { manufacturer, model }.Where(s => !string.IsNullOrEmpty(s)));
|
||||
await _tenantNotificationService.CreateNotificationAsync(
|
||||
tenantId,
|
||||
type: "tpm_pss_unsupported",
|
||||
title: "Device with incompatible TPM detected",
|
||||
message: $"Device {serialNumber}{(deviceLabel.Length > 0 ? $" ({deviceLabel})" : "")} reported a TPM that cannot perform RSA-PSS signing — the agent on this device cannot authenticate. A TPM firmware update or device replacement is required.",
|
||||
href: "/settings/tenant/hardware-whitelist");
|
||||
}
|
||||
}
|
||||
catch (Exception bellEx)
|
||||
{
|
||||
_logger.LogWarning(bellEx, "TpmPssUnsupported bell notification failed for tenant {TenantId}", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
// Structured log (Warning, not Critical — data is unverified)
|
||||
_logger.LogWarning(
|
||||
"AgentDistress [{ErrorType}] tenant={TenantId} mfr={Manufacturer} model={Model} sn={SerialNumber} http={HttpStatusCode} ver={AgentVersion} certState={CertSourceState} thumbprint={CertThumbprint}: {Message}",
|
||||
|
||||
+358
@@ -0,0 +1,358 @@
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
{
|
||||
/// <summary>
|
||||
/// F2 device-history / First-Time-Right read surfaces (insights spec §F2 "Surfaces", PR5).
|
||||
/// All endpoints serve rows the terminal seam + maintenance sweep maintain — the journey
|
||||
/// semantics live server-side (<see cref="DeviceJourneyCalculator"/>); no consumer re-derives
|
||||
/// them. Durations in chain refs are the sessions' authoritative DurationSeconds verbatim.
|
||||
/// </summary>
|
||||
public class GetDeviceHistoryFunction
|
||||
{
|
||||
private readonly ILogger<GetDeviceHistoryFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
|
||||
public GetDeviceHistoryFunction(
|
||||
ILogger<GetDeviceHistoryFunction> logger,
|
||||
IMetricsRepository metricsRepo,
|
||||
ISessionRepository sessionRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One device's enrollment history by serial number. history=null is a NORMAL outcome
|
||||
/// (unknown device, junk/placeholder serial, or every ref pruned) — the banner simply
|
||||
/// stays hidden. With ?sessionId= the response additionally carries that session's
|
||||
/// attempt number within its journey (server-computed; works for live sessions too via
|
||||
/// the virtual-attempt rule). Serves the session-detail banner and the MCP tool; a
|
||||
/// Global Admin passes ?tenantId= for cross-tenant reads (TenantScoping.QueryParam),
|
||||
/// with the sessionId-based owner fallback mirroring GetSessionTimeAttribution.
|
||||
/// </summary>
|
||||
[Function("GetDeviceHistory")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "metrics/device-history")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + MemberRead authorization enforced by PolicyEnforcementMiddleware;
|
||||
// cross-tenant access via TargetTenantId (TenantScoping.QueryParam).
|
||||
var requestCtx = req.GetRequestContext();
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var rawSerial = query["serialNumber"];
|
||||
var sessionId = query["sessionId"];
|
||||
|
||||
if (string.IsNullOrWhiteSpace(rawSerial))
|
||||
{
|
||||
var badRequest = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await badRequest.WriteAsJsonAsync(new { success = false, message = "serialNumber is required" });
|
||||
return badRequest;
|
||||
}
|
||||
|
||||
// Junk/placeholder serials have no device identity — by design they never have a
|
||||
// chain (disclosed in the fleet aggregate instead), so null here is honest.
|
||||
var serialKey = DeviceJourneyCalculator.NormalizeSerial(rawSerial);
|
||||
var effectiveTenantId = requestCtx.TargetTenantId;
|
||||
var history = serialKey == null
|
||||
? null
|
||||
: await _metricsRepo.GetDeviceHistoryAsync(effectiveTenantId, serialKey);
|
||||
|
||||
// Global-scope fallback (mirrors GetSessionTimeAttribution): a GA/operator opening
|
||||
// a session without an explicit tenantId query param reads their home partition —
|
||||
// resolve the owning tenant via the session and retry once.
|
||||
if (history == null && serialKey != null && requestCtx.HasGlobalScope && !string.IsNullOrEmpty(sessionId))
|
||||
{
|
||||
var resolvedTenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId!);
|
||||
if (resolvedTenantId != null &&
|
||||
!string.Equals(resolvedTenantId, effectiveTenantId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
effectiveTenantId = resolvedTenantId;
|
||||
history = await _metricsRepo.GetDeviceHistoryAsync(effectiveTenantId, serialKey);
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt number for the requesting session — fail-soft: a missing session or an
|
||||
// empty basis yields null, never a guessed position.
|
||||
int? attemptNumber = null;
|
||||
if (history != null && !string.IsNullOrEmpty(sessionId))
|
||||
{
|
||||
var session = await _sessionRepo.GetSessionAsync(effectiveTenantId, sessionId!);
|
||||
if (session != null)
|
||||
attemptNumber = DeviceJourneyCalculator.ComputeAttemptNumber(history.Chain, sessionId!, session.StartedAt);
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true, history, attemptNumber });
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching device history");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tenant fleet view: daily First-Time-Right rows of the requested window plus their sums,
|
||||
/// the merged attempt histogram, and the "repeat devices" violator list. FTR counts are
|
||||
/// additive, so unlike the median-based time-attribution panel this endpoint honors the
|
||||
/// page's days selector honestly (window rate = sum over daily rows).
|
||||
/// </summary>
|
||||
public class GetDeviceJourneyMetricsFunction
|
||||
{
|
||||
private readonly ILogger<GetDeviceJourneyMetricsFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
|
||||
public GetDeviceJourneyMetricsFunction(
|
||||
ILogger<GetDeviceJourneyMetricsFunction> logger,
|
||||
IMetricsRepository metricsRepo,
|
||||
ISessionRepository sessionRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
}
|
||||
|
||||
[Function("GetDeviceJourneyMetrics")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "metrics/device-journeys")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + MemberRead authorization enforced by PolicyEnforcementMiddleware
|
||||
var tenantId = TenantHelper.GetTenantId(req);
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var days = DeviceJourneyMetricsResponse.ClampDays(query["days"]);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(await DeviceJourneyMetricsResponse.BuildAsync(
|
||||
_metricsRepo, _sessionRepo, tenantId, days, includeRepeatDevices: true));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching device journey metrics");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Global-admin variant: tenantId filters to one tenant (full response incl. repeat
|
||||
/// devices); absent = the cross-tenant "global" aggregate rows, WITHOUT a repeat-devices
|
||||
/// list — that would require scanning every tenant's DeviceHistories partition, and the
|
||||
/// aggregated view has no per-device drill anyway. repeatDevices=null discloses the gap.
|
||||
/// </summary>
|
||||
public class GetGlobalDeviceJourneyMetricsFunction
|
||||
{
|
||||
private readonly ILogger<GetGlobalDeviceJourneyMetricsFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
|
||||
public GetGlobalDeviceJourneyMetricsFunction(
|
||||
ILogger<GetGlobalDeviceJourneyMetricsFunction> logger,
|
||||
IMetricsRepository metricsRepo,
|
||||
ISessionRepository sessionRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
}
|
||||
|
||||
[Function("GetGlobalDeviceJourneyMetrics")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/metrics/device-journeys")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + GlobalReadOrAdmin authorization enforced by PolicyEnforcementMiddleware
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var tenantIdFilter = query["tenantId"];
|
||||
var days = DeviceJourneyMetricsResponse.ClampDays(query["days"]);
|
||||
var partition = string.IsNullOrWhiteSpace(tenantIdFilter) ? "global" : tenantIdFilter!;
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(await DeviceJourneyMetricsResponse.BuildAsync(
|
||||
_metricsRepo, _sessionRepo, partition, days, includeRepeatDevices: partition != "global"));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching global device journey metrics");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Shared response shape for the tenant and global variants (mirrors TimeAttributionResponse).</summary>
|
||||
internal static class DeviceJourneyMetricsResponse
|
||||
{
|
||||
internal const int DefaultWindowDays = 30;
|
||||
internal const int MaxWindowDays = 180; // aggregate retention — older rows no longer exist
|
||||
internal const int MaxRepeatDevices = 10;
|
||||
|
||||
internal static int ClampDays(string? raw)
|
||||
{
|
||||
if (!int.TryParse(raw, out var days)) return DefaultWindowDays;
|
||||
return Math.Clamp(days, 1, MaxWindowDays);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// "Last N days" = exactly N calendar day keys including today — both range ends are
|
||||
/// inclusive, so subtracting the full N would return N+1 days (Codex review: days=1
|
||||
/// summed yesterday AND today).
|
||||
/// </summary>
|
||||
internal static DateTime InclusiveWindowStart(DateTime today, int days) => today.AddDays(-(days - 1));
|
||||
|
||||
internal static async Task<object> BuildAsync(
|
||||
IMetricsRepository metricsRepo, ISessionRepository sessionRepo,
|
||||
string partition, int days, bool includeRepeatDevices)
|
||||
{
|
||||
var today = DateTime.UtcNow.Date;
|
||||
var daily = await metricsRepo.GetDeviceJourneyAggregatesAsync(partition, InclusiveWindowStart(today, days), today);
|
||||
var totals = SumAggregates(daily);
|
||||
|
||||
List<object>? repeatDevices = null;
|
||||
if (includeRepeatDevices)
|
||||
{
|
||||
var histories = await metricsRepo.GetDeviceHistoriesByTenantAsync(partition);
|
||||
var candidates = SelectRepeatDevices(histories, today.AddDays(-days));
|
||||
repeatDevices = new List<object>(candidates.Count);
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
// Failure reason of the newest failed attempt — a bounded point-read per top
|
||||
// row (≤10), fail-soft: an unreadable/deleted session just leaves it empty.
|
||||
var lastFailureReason = string.Empty;
|
||||
if (candidate.NewestFailed != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
var failedSession = await sessionRepo.GetSessionAsync(partition, candidate.NewestFailed.SessionId);
|
||||
lastFailureReason = failedSession?.FailureReason ?? string.Empty;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// fail-soft — the row renders without a reason
|
||||
}
|
||||
}
|
||||
repeatDevices.Add(new
|
||||
{
|
||||
serialNumber = candidate.History.SerialNumber,
|
||||
manufacturer = candidate.History.Manufacturer,
|
||||
model = candidate.History.Model,
|
||||
attempts = candidate.History.CurrentJourneyAttempts,
|
||||
journeyCount = candidate.History.JourneyCount,
|
||||
lastStatus = candidate.Newest.Status,
|
||||
lastSessionId = candidate.Newest.SessionId,
|
||||
lastStartedAt = candidate.Newest.StartedAt,
|
||||
lastFailureReason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new
|
||||
{
|
||||
success = true,
|
||||
windowDays = days,
|
||||
totals,
|
||||
daily = daily.OrderBy(d => d.Date, StringComparer.Ordinal).ToList(),
|
||||
repeatDevices,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Window totals (serialized camelCase by the worker's global serializer, like the model rows).</summary>
|
||||
internal sealed class DeviceJourneyWindowTotals
|
||||
{
|
||||
public int CompletedJourneys { get; init; }
|
||||
public int FirstTimeRight { get; init; }
|
||||
|
||||
/// <summary>Null with zero completed journeys — no rate claim, never 0 (truthfulness rule 1).</summary>
|
||||
public double? FtrRatePct { get; init; }
|
||||
|
||||
public int ExcludedSessions { get; init; }
|
||||
public List<DeviceJourneyAttemptBucket> AttemptHistogram { get; init; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Window sums over the daily rows — honest because every field is an additive count
|
||||
/// (rule: the FTR window rate IS the ratio of summed counts). internal static: pinned
|
||||
/// by unit tests.
|
||||
/// </summary>
|
||||
internal static DeviceJourneyWindowTotals SumAggregates(IReadOnlyList<DeviceJourneyDailyAggregate> daily)
|
||||
{
|
||||
var completed = daily.Sum(d => d.CompletedJourneyCount);
|
||||
var firstTimeRight = daily.Sum(d => d.FirstTimeRightCount);
|
||||
return new DeviceJourneyWindowTotals
|
||||
{
|
||||
CompletedJourneys = completed,
|
||||
FirstTimeRight = firstTimeRight,
|
||||
FtrRatePct = completed > 0 ? Math.Round(firstTimeRight * 100.0 / completed, 1) : null,
|
||||
ExcludedSessions = daily.Sum(d => d.ExcludedSessionCount),
|
||||
AttemptHistogram = daily
|
||||
.SelectMany(d => d.AttemptHistogram)
|
||||
.GroupBy(b => b.Attempts)
|
||||
.Select(g => new DeviceJourneyAttemptBucket { Attempts = g.Key, JourneyCount = g.Sum(b => b.JourneyCount) })
|
||||
.OrderBy(b => b.Attempts)
|
||||
.ToList(),
|
||||
};
|
||||
}
|
||||
|
||||
internal sealed class RepeatDeviceCandidate
|
||||
{
|
||||
public DeviceHistory History { get; init; } = null!;
|
||||
public DeviceSessionRef Newest { get; init; } = null!;
|
||||
public DeviceSessionRef? NewestFailed { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Violator selection (spec: "repeat devices" table): devices whose LAST journey took
|
||||
/// ≥2 attempts and whose newest terminal session falls inside the window — current pain,
|
||||
/// not ancient history (a device that retried long ago but enrolls cleanly today does
|
||||
/// not belong on a violator list). Ordered by attempts desc, then recency; capped at
|
||||
/// <see cref="MaxRepeatDevices"/>. internal static: pinned by unit tests.
|
||||
/// </summary>
|
||||
internal static List<RepeatDeviceCandidate> SelectRepeatDevices(
|
||||
IReadOnlyList<DeviceHistory> histories, DateTime windowStartUtc)
|
||||
{
|
||||
var candidates = new List<RepeatDeviceCandidate>();
|
||||
foreach (var history in histories)
|
||||
{
|
||||
if (history.CurrentJourneyAttempts < 2 || history.Chain.Count == 0)
|
||||
continue;
|
||||
var newest = history.Chain[history.Chain.Count - 1];
|
||||
if (newest.StartedAt < windowStartUtc)
|
||||
continue;
|
||||
candidates.Add(new RepeatDeviceCandidate
|
||||
{
|
||||
History = history,
|
||||
Newest = newest,
|
||||
NewestFailed = history.Chain.LastOrDefault(r =>
|
||||
string.Equals(r.Status, nameof(SessionStatus.Failed), StringComparison.OrdinalIgnoreCase)),
|
||||
});
|
||||
}
|
||||
return candidates
|
||||
.OrderByDescending(c => c.History.CurrentJourneyAttempts)
|
||||
.ThenByDescending(c => c.Newest.StartedAt)
|
||||
.Take(MaxRepeatDevices)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
+38
-1
@@ -106,6 +106,13 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
var first = group.First();
|
||||
var succeeded = group.Where(s => s.Status == SessionStatus.Succeeded).ToList();
|
||||
var failed = group.Where(s => s.Status == SessionStatus.Failed).ToList();
|
||||
// Success rate is an outcome quota over finished enrollments only
|
||||
// (Succeeded + Failed), matching the SLA convention. In-flight sessions
|
||||
// (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal,
|
||||
// non-failure) must not dilute it — a site mid-rollout would otherwise
|
||||
// show a poor rate despite zero actual failures. 0 finished → 0; the
|
||||
// frontend renders "—" off Succeeded+Failed instead of a 0% badge.
|
||||
var finished = succeeded.Count + failed.Count;
|
||||
|
||||
// Duration stats from succeeded sessions with valid duration
|
||||
var durations = succeeded
|
||||
@@ -146,6 +153,18 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
.Where(s => appsBySession.ContainsKey(s.SessionId))
|
||||
.Count(s => appsBySession[s.SessionId].Any(a => a.DoDownloadMode >= 0));
|
||||
|
||||
// Agent→backend API latency: weight each session's stored average by its request
|
||||
// count so the location figure equals Σ(latency·requests)/Σ(requests) — the exact
|
||||
// whole-population average, not an average of session averages. Sessions from
|
||||
// agents predating the feature carry no data and drop out.
|
||||
var latencySessions = group
|
||||
.Where(s => s.AvgApiLatencyMs is > 0 && s.ApiRequestCount is > 0)
|
||||
.ToList();
|
||||
var latencyRequestSum = latencySessions.Sum(s => (double)s.ApiRequestCount!.Value);
|
||||
var avgApiLatency = latencyRequestSum > 0
|
||||
? latencySessions.Sum(s => s.AvgApiLatencyMs!.Value * s.ApiRequestCount!.Value) / latencyRequestSum
|
||||
: 0;
|
||||
|
||||
locations.Add(new LocationMetrics
|
||||
{
|
||||
LocationKey = group.Key,
|
||||
@@ -156,7 +175,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
SessionCount = group.Count(),
|
||||
Succeeded = succeeded.Count,
|
||||
Failed = failed.Count,
|
||||
SuccessRate = group.Count() > 0 ? Math.Round((double)succeeded.Count / group.Count() * 100, 1) : 0,
|
||||
SuccessRate = finished > 0 ? Math.Round((double)succeeded.Count / finished * 100, 1) : 0,
|
||||
AvgDurationMinutes = Math.Round(avgDuration, 1),
|
||||
MedianDurationMinutes = Math.Round(medianDuration, 1),
|
||||
P95DurationMinutes = Math.Round(p95Duration, 1),
|
||||
@@ -164,6 +183,8 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
MinutesPerApp = Math.Round(minutesPerApp, 2),
|
||||
AvgThroughputBytesPerSec = Math.Round(avgThroughput, 0),
|
||||
TotalDownloadBytes = totalBytes,
|
||||
AvgApiLatencyMs = Math.Round(avgApiLatency, 0),
|
||||
ApiLatencySessionCount = latencySessions.Count,
|
||||
// Delivery Optimization
|
||||
DoSessionCount = doSessionCount,
|
||||
AvgDoPercentPeerCaching = Math.Round(doAgg.PercentPeerCaching, 1),
|
||||
@@ -195,6 +216,17 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
.Select(l => l.AvgThroughputBytesPerSec).ToList();
|
||||
var globalAvgThroughput = allThroughputs.Count > 0 ? allThroughputs.Average() : 0;
|
||||
|
||||
// Global API latency: request-weighted over ALL geo sessions (not location averages),
|
||||
// so big sites weigh in proportionally — same Σ(latency·requests)/Σ(requests) math as
|
||||
// the per-location figure.
|
||||
var globalLatencySessions = geoSessions
|
||||
.Where(s => s.AvgApiLatencyMs is > 0 && s.ApiRequestCount is > 0)
|
||||
.ToList();
|
||||
var globalLatencyRequestSum = globalLatencySessions.Sum(s => (double)s.ApiRequestCount!.Value);
|
||||
var globalAvgApiLatency = globalLatencyRequestSum > 0
|
||||
? globalLatencySessions.Sum(s => s.AvgApiLatencyMs!.Value * s.ApiRequestCount!.Value) / globalLatencyRequestSum
|
||||
: 0;
|
||||
|
||||
// Global DO metrics (weighted by bytes, not simple average)
|
||||
var globalDoPeers = locations.Sum(l => l.TotalDoBytesFromPeers);
|
||||
var globalDoHttp = locations.Sum(l => l.TotalDoBytesFromHttp);
|
||||
@@ -212,6 +244,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
loc.ThroughputVsGlobalPct = globalAvgThroughput > 0
|
||||
? Math.Round((loc.AvgThroughputBytesPerSec - globalAvgThroughput) / globalAvgThroughput * 100, 1) : 0;
|
||||
|
||||
// API latency vs global (positive = slower/farther from the backend region)
|
||||
loc.ApiLatencyVsGlobalPct = globalAvgApiLatency > 0 && loc.AvgApiLatencyMs > 0
|
||||
? Math.Round((loc.AvgApiLatencyMs - globalAvgApiLatency) / globalAvgApiLatency * 100, 1) : 0;
|
||||
|
||||
// AppLoadScore: normalize minutesPerApp to global median = 100
|
||||
loc.AppLoadScore = globalMedianMinutesPerApp > 0
|
||||
? Math.Round(loc.MinutesPerApp / globalMedianMinutesPerApp * 100, 0) : 0;
|
||||
@@ -236,6 +272,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
AvgMinutesPerApp = Math.Round(globalAvgMinutesPerApp, 2),
|
||||
AvgThroughputBytesPerSec = Math.Round(globalAvgThroughput, 0),
|
||||
StdDevDurationMinutes = Math.Round(globalStdDev, 1),
|
||||
AvgApiLatencyMs = Math.Round(globalAvgApiLatency, 0),
|
||||
AvgDoPercentPeerCaching = Math.Round(globalDoPct, 1),
|
||||
TotalDoBytesFromPeers = globalDoPeers,
|
||||
TotalDoBytesFromHttp = globalDoHttp
|
||||
|
||||
+32
-18
@@ -1,6 +1,7 @@
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -34,24 +35,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
var versions = await _sessionRepo.GetImeVersionHistoryAsync();
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
|
||||
if (requestCtx.HasGlobalScope)
|
||||
{
|
||||
await response.WriteAsJsonAsync(versions);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Strip sensitive cross-tenant data for non-Global Admins
|
||||
var redacted = versions.Select(v => new
|
||||
{
|
||||
v.Version,
|
||||
v.FirstSeenAt,
|
||||
v.LastSeenAt,
|
||||
v.SessionCount
|
||||
});
|
||||
await response.WriteAsJsonAsync(redacted);
|
||||
}
|
||||
|
||||
await response.WriteAsJsonAsync(BuildResponsePayload(versions, requestCtx.HasGlobalScope));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -62,5 +46,35 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Projects the archive for the caller's scope.
|
||||
///
|
||||
/// Extracted from <see cref="Run"/> so the cross-tenant redaction boundary is
|
||||
/// directly testable: this suite has no HTTP-level harness, so an inline projection
|
||||
/// is unreachable from a test — and an unguarded redaction is exactly what a later
|
||||
/// refactor opens silently.
|
||||
///
|
||||
/// FirstSeenTenantId and FirstSeenSessionId name ANOTHER tenant's device and are
|
||||
/// dropped for non-global callers. Version, dates and SessionCount deliberately stay:
|
||||
/// the archive is a platform-wide view of Microsoft's IME rollout, and the route is
|
||||
/// MemberRead precisely so any tenant member can read that view.
|
||||
///
|
||||
/// Returned as <see cref="object"/> on purpose — System.Text.Json serializes an
|
||||
/// `object` declared type using the RUNTIME type, so both branches keep the exact
|
||||
/// wire format they had while the projection was inline.
|
||||
/// </summary>
|
||||
public static object BuildResponsePayload(
|
||||
IEnumerable<ImeVersionHistoryEntry> versions, bool hasGlobalScope)
|
||||
{
|
||||
if (hasGlobalScope)
|
||||
{
|
||||
return versions.ToList();
|
||||
}
|
||||
|
||||
return versions
|
||||
.Select(v => new { v.Version, v.FirstSeenAt, v.LastSeenAt, v.SessionCount })
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
{
|
||||
/// <summary>
|
||||
/// F1 time-attribution read surfaces (insights spec §F1 "Surfaces", PR3). All three
|
||||
/// endpoints serve PRE-COMPUTED rows — the per-session breakdown written at the terminal
|
||||
/// transition and the aggregate rows the maintenance sweep maintains. No request-time
|
||||
/// derivation: what the calculator computed is what every consumer (web, MCP) sees.
|
||||
/// </summary>
|
||||
public class GetSessionTimeAttributionFunction
|
||||
{
|
||||
private readonly ILogger<GetSessionTimeAttributionFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
|
||||
public GetSessionTimeAttributionFunction(
|
||||
ILogger<GetSessionTimeAttributionFunction> logger,
|
||||
IMetricsRepository metricsRepo,
|
||||
ISessionRepository sessionRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-session breakdown for the session-detail attribution lane. A missing row is a
|
||||
/// NORMAL outcome (pre-feature session, non-terminal, Incomplete — no wall clock) and
|
||||
/// returns success with breakdown=null so the UI simply omits the lane.
|
||||
/// </summary>
|
||||
[Function("GetSessionTimeAttribution")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "sessions/{sessionId}/time-attribution")] HttpRequestData req,
|
||||
string sessionId)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + MemberRead authorization enforced by PolicyEnforcementMiddleware;
|
||||
// cross-tenant access via TargetTenantId (TenantScoping.QueryParam).
|
||||
var requestCtx = req.GetRequestContext();
|
||||
|
||||
var breakdown = await _metricsRepo.GetSessionTimeBreakdownAsync(requestCtx.TargetTenantId, sessionId);
|
||||
|
||||
// Global-scope fallback (mirrors GetSessionEventsFunction): a GA/operator opening
|
||||
// a session without an explicit tenantId query param reads their home partition —
|
||||
// resolve the owning tenant and retry once.
|
||||
if (breakdown == null && requestCtx.HasGlobalScope)
|
||||
{
|
||||
var resolvedTenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId);
|
||||
if (resolvedTenantId != null &&
|
||||
!string.Equals(resolvedTenantId, requestCtx.TargetTenantId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
breakdown = await _metricsRepo.GetSessionTimeBreakdownAsync(resolvedTenantId, sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true, breakdown });
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching time attribution for session {SessionId}", sessionId);
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tenant fleet view: the rolling 30-day range statistics per enrollment class (the panel's
|
||||
/// stacked medians + top blocking apps) plus the daily rows for the per-day trend. The range
|
||||
/// window is FIXED at the sweep's 30 days — daily medians cannot be merged into an arbitrary
|
||||
/// range honestly, so no days parameter is offered.
|
||||
/// </summary>
|
||||
public class GetTimeAttributionMetricsFunction
|
||||
{
|
||||
private readonly ILogger<GetTimeAttributionMetricsFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
|
||||
public GetTimeAttributionMetricsFunction(
|
||||
ILogger<GetTimeAttributionMetricsFunction> logger,
|
||||
IMetricsRepository metricsRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
}
|
||||
|
||||
[Function("GetTimeAttributionMetrics")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "metrics/time-attribution")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + MemberRead authorization enforced by PolicyEnforcementMiddleware
|
||||
var tenantId = TenantHelper.GetTenantId(req);
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(await TimeAttributionResponse.BuildAsync(_metricsRepo, tenantId));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching time attribution metrics");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Global-admin variant: tenantId filters to one tenant; absent = cross-tenant "global" rows.</summary>
|
||||
public class GetGlobalTimeAttributionMetricsFunction
|
||||
{
|
||||
private readonly ILogger<GetGlobalTimeAttributionMetricsFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
|
||||
public GetGlobalTimeAttributionMetricsFunction(
|
||||
ILogger<GetGlobalTimeAttributionMetricsFunction> logger,
|
||||
IMetricsRepository metricsRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
}
|
||||
|
||||
[Function("GetGlobalTimeAttributionMetrics")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/metrics/time-attribution")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + GlobalReadOrAdmin authorization enforced by PolicyEnforcementMiddleware
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var tenantIdFilter = query["tenantId"];
|
||||
var partition = string.IsNullOrWhiteSpace(tenantIdFilter) ? "global" : tenantIdFilter!;
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(await TimeAttributionResponse.BuildAsync(_metricsRepo, partition));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching global time attribution metrics");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Shared response shape for the tenant and global variants (mirrors the MetricsMath single-source pattern).</summary>
|
||||
internal static class TimeAttributionResponse
|
||||
{
|
||||
internal const int WindowDays = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Rolling rows are rewritten by every 2h sweep as long as the partition has window
|
||||
/// sessions; a partition that went fully quiet stops being rewritten, so its rows would
|
||||
/// otherwise serve arbitrarily old numbers as "last 30 days" (Codex review). 48h
|
||||
/// tolerates a full day of sweep outage before the panel honestly goes empty.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan RollingMaxAge = TimeSpan.FromHours(48);
|
||||
|
||||
/// <summary>
|
||||
/// "Last N days" = exactly N calendar day keys including today — both range ends are
|
||||
/// inclusive, so subtracting the full N would return N+1 days (Codex review).
|
||||
/// </summary>
|
||||
internal static DateTime InclusiveWindowStart(DateTime today, int days) => today.AddDays(-(days - 1));
|
||||
|
||||
internal static async Task<object> BuildAsync(IMetricsRepository metricsRepo, string partition)
|
||||
{
|
||||
var freshCutoff = DateTime.UtcNow - RollingMaxAge;
|
||||
var rolling = (await metricsRepo.GetRollingTimeAttributionAggregatesAsync(partition))
|
||||
.Where(r => r.ComputedAt >= freshCutoff)
|
||||
.ToList();
|
||||
var today = DateTime.UtcNow.Date;
|
||||
var daily = await metricsRepo.GetTimeAttributionAggregatesAsync(partition, InclusiveWindowStart(today, WindowDays), today);
|
||||
|
||||
return new
|
||||
{
|
||||
success = true,
|
||||
windowDays = WindowDays,
|
||||
// Range statistics per enrollment class (never mixed) — the UI gates rendering
|
||||
// at cleanSessionCount >= 20 and shows "insufficient data (n=…)" below it.
|
||||
classes = rolling.OrderBy(r => r.EnrollmentClass, StringComparer.Ordinal).ToList(),
|
||||
daily = daily.OrderBy(d => d.Date, StringComparer.Ordinal).ToList(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,13 +16,16 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
{
|
||||
private readonly ILogger<RuleStatsFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly IHardwareRejectionNotificationTracker _notificationTracker;
|
||||
|
||||
public RuleStatsFunction(
|
||||
ILogger<RuleStatsFunction> logger,
|
||||
IMetricsRepository metricsRepo)
|
||||
IMetricsRepository metricsRepo,
|
||||
IHardwareRejectionNotificationTracker notificationTracker)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
_notificationTracker = notificationTracker;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -81,6 +84,9 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
var result = new
|
||||
{
|
||||
rules = aggregated,
|
||||
// F3 PR6: active regression episodes (tracker rows) — the rules-page badge
|
||||
// and MCP get_rule_stats read them from here. Empty list = nothing regressed.
|
||||
regressions = await _notificationTracker.GetRuleRegressionsAsync(tenantId),
|
||||
summary = new
|
||||
{
|
||||
totalEvaluations,
|
||||
@@ -165,6 +171,11 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
var result = new
|
||||
{
|
||||
rules = aggregated,
|
||||
// Regression episodes are tenant-scoped: present when drilling into one
|
||||
// tenant, empty for the cross-tenant "global" aggregate (no episode there).
|
||||
regressions = queryTenantId != "global"
|
||||
? await _notificationTracker.GetRuleRegressionsAsync(queryTenantId)
|
||||
: new List<AutopilotMonitor.Shared.Models.RuleRegressionAlert>(),
|
||||
summary = new
|
||||
{
|
||||
totalEvaluations,
|
||||
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
{
|
||||
/// <summary>
|
||||
/// Tenant-scoped endpoint: returns devices whose TPM-backed client certificate cannot
|
||||
/// perform RSA-PSS signing (Schannel silently drops the cert from TLS client-auth on
|
||||
/// Windows 11 25H2+, so the agent can never authenticate), aggregated by serial number.
|
||||
/// Data comes from distress reports with ErrorType == "TpmPssUnsupported".
|
||||
///
|
||||
/// Informational only — the remediation is a TPM firmware update or device replacement,
|
||||
/// not a portal action. The 14-day horizon is enforced by the distress-report retention
|
||||
/// cleanup, not by this function.
|
||||
/// Authentication + MemberRead authorization enforced by PolicyEnforcementMiddleware.
|
||||
/// </summary>
|
||||
public class GetTpmPssUnsupportedFunction
|
||||
{
|
||||
private readonly ILogger<GetTpmPssUnsupportedFunction> _logger;
|
||||
private readonly IDistressReportRepository _repository;
|
||||
|
||||
public GetTpmPssUnsupportedFunction(
|
||||
ILogger<GetTpmPssUnsupportedFunction> logger,
|
||||
IDistressReportRepository repository)
|
||||
{
|
||||
_logger = logger;
|
||||
_repository = repository;
|
||||
}
|
||||
|
||||
[Function("GetTpmPssUnsupported")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "audit/tpm-pss-unsupported")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
string tenantId = TenantHelper.GetTenantId(req);
|
||||
|
||||
var reports = await _repository.GetDistressReportsAsync(tenantId, maxResults: 200);
|
||||
var (aggregated, totalRawReports) = BuildAggregatedResult(reports);
|
||||
|
||||
return await req.OkAsync(new
|
||||
{
|
||||
success = true,
|
||||
aggregated,
|
||||
totalRawReports,
|
||||
dataQualityNotice = "This data is from pre-authentication distress reports and is UNVERIFIED. Devices reported here have a TPM that cannot perform RSA-PSS signing, so their agent cannot authenticate to the backend. Serial number, manufacturer, and model values are self-reported by devices."
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return await req.InternalServerErrorAsync(_logger, ex, "Get TPM PSS unsupported devices");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Filters distress reports to TpmPssUnsupported and aggregates by serial number.
|
||||
/// Reports with no serial number are grouped into a single "unknown serial" bucket.
|
||||
/// Extracted as public static for testability.
|
||||
/// </summary>
|
||||
public static (List<object> aggregated, int totalRawReports) BuildAggregatedResult(
|
||||
List<DistressReportEntry> reports)
|
||||
{
|
||||
var tpmReports = reports
|
||||
.Where(r => string.Equals(r.ErrorType, "TpmPssUnsupported", StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
var aggregated = tpmReports
|
||||
.GroupBy(r => (r.SerialNumber ?? "").Trim().ToLowerInvariant())
|
||||
.Select(g =>
|
||||
{
|
||||
var mostRecent = g.OrderByDescending(r => r.IngestedAt).First();
|
||||
|
||||
return new
|
||||
{
|
||||
serialNumber = mostRecent.SerialNumber ?? "",
|
||||
manufacturer = mostRecent.Manufacturer ?? "",
|
||||
model = mostRecent.Model ?? "",
|
||||
attemptCount = g.Count(),
|
||||
firstSeen = g.Min(r => r.IngestedAt),
|
||||
lastSeen = g.Max(r => r.IngestedAt)
|
||||
};
|
||||
})
|
||||
.OrderByDescending(a => a.lastSeen)
|
||||
.ToList<object>();
|
||||
|
||||
return (aggregated, tpmReports.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -10,8 +10,8 @@ using Microsoft.Extensions.Logging;
|
||||
namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
{
|
||||
/// <summary>
|
||||
/// Tenant-admin entry point for submitting diagnostic files (logs, JSON state files,
|
||||
/// screenshots) without a session context. Lands in the same SessionReports table +
|
||||
/// Tenant-admin/-operator entry point for submitting diagnostic files (logs, JSON state
|
||||
/// files, screenshots) without a session context. Lands in the same SessionReports table +
|
||||
/// session-reports blob container as session reports, distinguished by
|
||||
/// <see cref="SessionReportMetadata.ReportType"/> = "diagFiles".
|
||||
/// </summary>
|
||||
@@ -45,7 +45,7 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
|
||||
try
|
||||
{
|
||||
// Authentication + TenantAdminOrGA authorization enforced by PolicyEnforcementMiddleware
|
||||
// Authentication + TenantAdminOrOperator authorization enforced by PolicyEnforcementMiddleware
|
||||
var requestCtx = req.GetRequestContext();
|
||||
var tenantId = requestCtx.TenantId;
|
||||
var userIdentifier = requestCtx.UserPrincipalName;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user