mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 20:53:06 +00:00
Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8e8bb66371 | |||
| 7e1389cf32 | |||
| 3a89319472 | |||
| b37e6f975c | |||
| cacbd24667 | |||
| a157d98a40 | |||
| 7fa50d09c1 | |||
| 94afc02e62 | |||
| 5e4c5e80be | |||
| b535973531 | |||
| 7e0c578a59 | |||
| 1257e0d195 | |||
| 98f8114d90 | |||
| 7b3230d507 | |||
| 87b7a20223 | |||
| 4c19be1de4 |
+1
-1
Submodule .claude/commands updated: 2866e07882...a115d243a3
@@ -78,7 +78,8 @@ Durations shown for attempts are the sessions' authoritative `DurationSeconds` v
|
||||
completed journeys, first-time-right count, attempt histogram, junk-serial exclusion
|
||||
count, `JourneyVersion`. A journey buckets on the StartedAt date of its completing
|
||||
success session (the platform's StartedAt-date convention; late terminals converge via
|
||||
the sweep). All counts are **additive**, so a window rate is the ratio of summed daily
|
||||
the sweep, which slices the tick's shared projected window scan — see
|
||||
`time-attribution.md`). All counts are **additive**, so a window rate is the ratio of summed daily
|
||||
rows — there is no rolling-window row (unlike the median-based time-attribution
|
||||
aggregates, which cannot merge). 180-day retention. Rows exist below the ≥20 UI gate
|
||||
on purpose (the UI needs the n for "insufficient data (n=…)").
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Silent-Session Classification (Sweep, Max-Lifetime, Retro-Reconcile)
|
||||
description: How a session whose agent went silent gets its honest status — the two-stage maintenance sweep (2h Stalled marker, 5h timeout classification), the EnrollmentTimeoutClassifier rule order incl. the WhiteGlove Part-2 and self-deploying gates, which ingest paths share the classifier, and the admin retro-reconcile modes.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs
|
||||
tags:
|
||||
- backend
|
||||
- maintenance
|
||||
- session-status
|
||||
- classification
|
||||
- whiteglove
|
||||
- self-deploying
|
||||
timestamp: 2026-08-23T18:00:00+02:00
|
||||
---
|
||||
|
||||
# Silent-Session Classification
|
||||
|
||||
A session only ever becomes terminal through evidence. When the agent stops reporting
|
||||
(power-off, reboot without relaunch, Wi-Fi never re-associating, max-lifetime watchdog),
|
||||
the backend must decide the status from what it last saw — it never hard-fails a session
|
||||
for being silent. One pure function holds that decision,
|
||||
`EnrollmentTimeoutClassifier.ClassifyTimedOutSession`, and every silent-session path calls it
|
||||
so a session gets the same verdict regardless of which path noticed the silence first.
|
||||
|
||||
# Schema
|
||||
|
||||
## Inputs (`EspProvisioningRollup` + session flags)
|
||||
|
||||
`ExtractRollup(events)` distills the event stream into facts: Device/Account Setup
|
||||
subcategory counts (incl. the 30s fallback-confirmed messages), `HasExplicitFailure`,
|
||||
`HasEnrollmentComplete` vs. `HasTerminalComplete` (the Part-1 `whiteglove_complete`),
|
||||
`HasAgentEmergencyBreak`, `DesktopArrived`, `HelloResolved`, `HelloPolicyDisabled`,
|
||||
`SkipUserEsp`, RealmJoin detection/resolution. Session-row flags are passed alongside:
|
||||
`IsPreProvisioned` + `ResumedAt` (WhiteGlove Part 2) and `IsSelfDeployingProfile`
|
||||
(registry-confirmed `CloudAssignedOobeConfig 0x20|0x40`, set at registration).
|
||||
|
||||
## Rule order (first match wins)
|
||||
|
||||
| # | Condition | Verdict |
|
||||
| --- | --- | --- |
|
||||
| 1 | `HasExplicitFailure` | Failed |
|
||||
| 1b | `IsWhiteGloveAwaitingUser`: pre-provisioned + resumed + Device Setup done + **no** user evidence since resume + no explicit failure/enrollment_complete | AwaitingUser within grace; Succeeded (honest WG reason) past grace |
|
||||
| 1c | `IsSelfDeployingProvisioned`: self-deploying profile + Device Setup done + no explicit failure | **Succeeded** (reconcile) — no grace, no AwaitingUser: the profile has no user phase |
|
||||
| 2 | Account Setup all-succeeded, or `enrollment_complete`, or `whiteglove_complete` on a **non-resumed** session | Succeeded |
|
||||
| 3 | `HasAgentEmergencyBreak` | Incomplete immediately (agent self-destructed, nothing can arrive) |
|
||||
| 4 | `DesktopArrived` + (Hello terminal, or Hello disabled + SkipUserEsp) | Succeeded (user provably finished) |
|
||||
| 5 | Device Setup done | AwaitingUser within grace, Incomplete past grace |
|
||||
| 6 | otherwise | Incomplete |
|
||||
|
||||
Grace = `ResolveGraceHours(SessionGraceHours, AbsoluteMaxSessionHours)` (~51h default),
|
||||
measured from `ResumedAt ?? StartedAt`. Every Succeeded-reconcile reason carries the
|
||||
silence-transparency clause (`AppendReconcileTiming`: last agent contact, silence, verdict time).
|
||||
|
||||
## Why rule 1c exists (kiosk tenant audit 2026-08-23)
|
||||
|
||||
On a self-deploying profile Device ESP all-succeeded IS the enrollment's end; the agent's
|
||||
own SelfDeploying terminal only adds a 5-min confirmation window
|
||||
(`DecisionEngine.SelfDeploying.s_deviceOnlyEspDetectionWindow`). Tenant `aebdce78` (school,
|
||||
Wi-Fi ThinkBooks, a language-change app forcing a reboot right after Device ESP) lost that
|
||||
window on 836 of ~970 sessions in one week: the agent died 0–4 min after
|
||||
`DeviceSetup fallback_confirmed`, the reboot relaunch never registered (30s registration
|
||||
budget on a Wi-Fi link still associating), and rule 5 labeled finished kiosks
|
||||
"awaiting user / Account Setup 0/5" → Incomplete 51h later. Nobody signs in on these devices
|
||||
(prepared during the holidays, boxed, in service) — AwaitingUser was factually wrong and
|
||||
Incomplete turned a working rollout into a red wall. Other self-deploying tenants completed
|
||||
normally in the same window, so this is a classification gap, not a profile-wide agent bug.
|
||||
Agent-side companion fix: `SessionRegistrationHelper` now waits up to 15s for a network link
|
||||
and retries 6× (~62s backoff) instead of 5× (~30s).
|
||||
|
||||
## Callers (all pass both gate inputs)
|
||||
|
||||
| Path | File | Notes |
|
||||
| --- | --- | --- |
|
||||
| Sweep stage 1 (2h agent-silent marker) | `MaintenanceService.MarkStalledSessionsAsTimedOutAsync` | Default: InProgress → Stalled. Gates run BEFORE the Stalled write: `TryMarkWhiteGloveAwaitingUserAsync` (AwaitingUser) and `TryReconcileSelfDeployingAsync` (Succeeded). Only flagged sessions pay the event read. |
|
||||
| Sweep stage 2 (SessionTimeoutHours, 5h default) | same | InProgress/Stalled/AwaitingUser past the window and past the 2h silence cutoff → full classification; AwaitingUser within grace is skipped without an event read. |
|
||||
| Agent max-lifetime (`enrollment_failed`/max_lifetime, `agent_shutting_down`/max_lifetime) | `EventIngestProcessor.ApplyMaxLifetimeVerdictAsync` | Watchdog = "agent stopped waiting", not a verdict; classified from evidence. Pending sessions exempt. |
|
||||
| Late-telemetry reconcile | `EventIngestProcessor.TryLateTelemetryReconcileAsync` | Heal-only: applies a Succeeded verdict to Incomplete/AwaitingUser rows when straggler events arrive. |
|
||||
| Agent stall probe (`session_stalled`) | `EventIngestProcessor.TryMapStallToWhiteGloveAwaitingUserAsync` | Only the WG gate (1b); a stall probe on a self-deploying device with Device Setup done means the agent is alive and something else blocks — stays Stalled. |
|
||||
| Admin retro-reconcile | `LegacyReclassificationService` | Modes `legacy_timeouts`, `pending_orphans`, `self_deploying_silent` (Incomplete/AwaitingUser/Stalled rows with the profile flag, heal-only to Succeeded, admin-marked rows skipped). `POST /api/maintenance/reclassify-legacy?mode=…&dryRun=false&tenantId=…&maxSessions=…`, dry-run by default. |
|
||||
|
||||
## Known gap (decided, not built)
|
||||
|
||||
After max-lifetime the agent writes `enrollment-complete.marker` and self-destructs, so an
|
||||
AwaitingUser verdict from that path can never heal — it is a 51h countdown to Incomplete.
|
||||
The emergency-break path (rule 3) already terminalizes immediately; max-lifetime still goes
|
||||
through the grace window by design ("user simply absent"). Revisit if customers read
|
||||
AwaitingUser as "still possible".
|
||||
|
||||
# Examples
|
||||
|
||||
* Session `195593e2` (tenant aebdce78): `DeviceSetup 4/4` + `fallback_confirmed`, `AccountSetup 0/5`
|
||||
(IME false positive), Hello disabled, SkipUser=True, silent after 13:01 → rule 1c →
|
||||
Succeeded "self-deploying profile — Device Setup fully provisioned … agent went silent
|
||||
before confirming completion. Agent last reported 2026-08-21 13:01 UTC …".
|
||||
* Same event shape on a user-driven profile → rule 5 → AwaitingUser (within grace) /
|
||||
Incomplete (past grace). Guarded by
|
||||
`Classify_same_stream_without_self_deploying_flag_keeps_user_phase_verdicts`.
|
||||
* fairstone WG Part 2 reseal-reboot, technician powers off at logon screen → rule 1b →
|
||||
AwaitingUser; user unboxes days later → ingest completion heals to Succeeded.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs` — rollup + rule order, `IsWhiteGloveAwaitingUser`, `IsSelfDeployingProvisioned`, reason helpers.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs` — `MarkStalledSessionsAsTimedOutAsync`, `TryMarkWhiteGloveAwaitingUserAsync`, `TryReconcileSelfDeployingAsync`.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs` — max-lifetime verdict, late-telemetry reconcile, stall mapping.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/LegacyReclassificationService.cs`, `Functions/Admin/ReclassifyLegacySessionsFunction.cs` — retro modes.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs` — `GetAgentSilentSessionsAsync`, `GetStalledSessionsAsync`, `GetSelfDeployingSilentSessionsAsync`.
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs` — link wait + 6-attempt registration budget.
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.SelfDeploying.cs` — the agent-side 5-min SelfDeploying terminal window.
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs` — rule coverage incl. the 2026-08-23 self-deploying block.
|
||||
@@ -0,0 +1,67 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Table Schema Sentinel (startup table initialization)
|
||||
description: Why a cold start no longer issues one CreateTableIfNotExists per registered table - a hash over TableNames.All stored in AdminConfiguration gates the full pass, the SessionsIndex backfill only runs after a full pass and behind a conditional-insert claim, and daily maintenance repairs out-of-band deletions.
|
||||
resource: /src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs
|
||||
tags:
|
||||
- backend
|
||||
- table-storage
|
||||
- startup
|
||||
- maintenance
|
||||
timestamp: 2026-08-23
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
## Problem
|
||||
|
||||
`TableInitializerService` (hosted service) called `InitializeTablesAsync` on every process start, which issued `CreateTableIfNotExistsAsync` for every entry of `Constants.TableNames.All` (66 at the time of writing, DoP 8). Each call is a Create POST whose 409 is swallowed - on Flex Consumption with frequent cold starts that was ~9 serial round-trip waves before the first request. It was followed by a SessionsIndex emptiness query whose *error* path returned "empty", so a transient storage error triggered a full Sessions scan with serial upserts - on every scaled-out instance at once.
|
||||
|
||||
## Sentinel
|
||||
|
||||
* Row `AdminConfiguration` / PK `SchemaSentinel` / RK `tables`, property `TableSchemaHash` (+ `TableCount`, `UpdatedUtc`).
|
||||
* Value = `ComputeTableSchemaHash(TableNames.All)`: SHA-256 over the ordinal-sorted, newline-joined table names. Derived, never hand-maintained: adding a table to `TableNames.All` changes the hash by construction (`TableLifecycleBucketTests` already forces every constant into `All`).
|
||||
* `InitializeTablesAsync` (returns `bool fullPassRan`):
|
||||
1. Point-read the sentinel. Match -> mark initialized, **zero** table calls, return `false`.
|
||||
2. Missing / stale / unreadable (fresh storage has no AdminConfiguration table yet) -> `EnsureAllTablesAsync` (the old full pass), then upsert the sentinel **only if no table failed**, return `true`. A failed pass leaves `_tablesInitialized = false`, so the next call retries.
|
||||
* Scale-out: two instances that both see a stale sentinel both run the idempotent pass - harmless.
|
||||
|
||||
## SessionsIndex backfill
|
||||
|
||||
* Only evaluated when the full pass ran (`fullPassRan == true`): an empty index can only exist on fresh storage, which is exactly the case without a sentinel.
|
||||
* `IsSessionIndexEmptyAsync` now returns `false` on error (skip), not `true`.
|
||||
* `TryClaimSessionIndexBackfillAsync`: conditional `AddEntityAsync` on `SchemaSentinel` / `sessionIndexBackfill` - only the winner scans; 409 or any error -> skip. The manual maintenance backfill (`RunManualAsync`) remains the safety net.
|
||||
|
||||
## Known gap and repair
|
||||
|
||||
A table deleted out-of-band (cleanup skill, portal, tests) is not noticed at startup anymore. `IStorageInitializer.EnsureAllAsync` runs as the first step of the daily timer maintenance (`MaintenanceService.RunAllAsync`), recreating it within 24h. Storage helpers are fail-soft in the meantime.
|
||||
|
||||
## Measuring it
|
||||
|
||||
`StartupTelemetryService` (hosted service) emits two App Insights metrics on `IHostApplicationLifetime.ApplicationStarted` - i.e. after every hosted service finished `StartAsync`:
|
||||
|
||||
* `BackendStartupMs` - process start (`Process.StartTime`) to host ready; includes runtime, DI and all hosted services.
|
||||
* `BackendTableInitMs` - the `InitializeTablesAsync` slice alone (`TableStorageService.LastInitialization`).
|
||||
|
||||
Dimensions: `version`, `commit`, `tableInitFullPass`, `instance`. Metrics bypass adaptive sampling and the worker log level, so they arrive even though Information traces do not. The MCP server logs the same idea as one `{"type":"boot","readyMs":…}` stderr line per start (ContainerAppConsoleLogs_CL, see `.claude/commands/mcp-telemetry.md`).
|
||||
|
||||
```kusto
|
||||
customMetrics
|
||||
| where name in ("BackendStartupMs", "BackendTableInitMs")
|
||||
| summarize p50 = percentile(value, 50), p95 = percentile(value, 95), n = count()
|
||||
by name, version = tostring(customDimensions.version), fullPass = tostring(customDimensions.tableInitFullPass)
|
||||
```
|
||||
|
||||
# Examples
|
||||
|
||||
* Deploy that adds a table: first cold start after the deploy runs the full pass once (log: "Table schema sentinel missing or stale"), every later start logs "Table schema sentinel matches (...)".
|
||||
* Fresh storage account: sentinel read throws TableNotFound -> full pass -> index empty -> one instance claims and backfills.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.cs` - sentinel, `InitializeTablesAsync`, `EnsureAllTablesAsync`, `TryClaimSessionIndexBackfillAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableInitializerService.cs` - startup gating
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs` - daily `EnsureAllTablesAsync`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/StartupTelemetryService.cs` - `BackendStartupMs` / `BackendTableInitMs`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/index.ts` - `{"type":"boot"}` line
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/TableSchemaSentinelTests.cs`, `StartupTelemetryServiceTests.cs`
|
||||
@@ -105,6 +105,11 @@ the partition).
|
||||
* Self-healing 30-day maintenance sweep (2h tick): backfills missing breakdowns, folds
|
||||
late-terminating sessions into their StartedAt-date aggregates, and enforces version
|
||||
purity — a stale-version row whose events aged out counts as missing, never mixed.
|
||||
Reads the tick's **shared window scan** (`MaintenanceService.LoadSweepWindowSessionsAsync`,
|
||||
one projected 35-day cross-tenant drain via `MaintenanceSweepSessionProjection` that
|
||||
also feeds the device-journey and verdict-calibration sweeps) — the StartedAt-only
|
||||
filter is a full-table read in Table Storage, so it is done once per tick, not once per
|
||||
sweep; `MaintenanceSweepProjectionEquivalenceTests` pins the column set.
|
||||
* `TimeAttributionAggregates` (PK = TenantId or `global`, RK = `{yyyy-MM-dd}|{class}` and
|
||||
`rolling30|{class}`): per enrollment class (`user_driven`, `whiteglove`, `self_deploying`,
|
||||
`device_preparation` — never mixed) median/p75/p90 per segment over the fixed six-segment
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Verdict Calibration (classifier thermometer)
|
||||
description: How every session status write declares its VerdictPath, how overriding a verdict preserves it as PriorStatus/PriorVerdictPath, how the daily calibration aggregate counts paths with the re-enrollment proxy and the correction stream, and how the drift radar (share / silence / evidence-gap) turns that into operator-only ops events.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs
|
||||
tags:
|
||||
- backend
|
||||
- sessions
|
||||
- classifier
|
||||
- calibration
|
||||
- regression-radar
|
||||
timestamp: 2026-08-23T18:00:00+02:00
|
||||
---
|
||||
|
||||
# Verdict Calibration (classifier thermometer)
|
||||
|
||||
A session's terminal status is written by ~20 code paths: the agent's own terminal events,
|
||||
the silence classifier (`EnrollmentTimeoutClassifier`, rules 1–6) reached via the
|
||||
maintenance sweep / the agent's max-lifetime shutdown / late-telemetry heal / retro
|
||||
reclassification, the analyze-rule engine, admin marks and session superseding. Verdict
|
||||
calibration makes every one of those paths **countable**, tracks how often a path's verdict
|
||||
was later **overridden**, attaches the delayed **re-enrollment proxy**, and raises an
|
||||
operator-only alarm when a path's share **drifts**. It is deterministic, fact-based tuning
|
||||
input for the rules — explicitly not a learned model, and never a customer-facing feature.
|
||||
|
||||
# Schema
|
||||
|
||||
## VerdictPath on the session row
|
||||
|
||||
`SessionSummary.VerdictPath` (`VerdictPaths` vocabulary, `origin:detail`, append-only) is
|
||||
stamped by **every** status write — `UpdateSessionStatusAsync` takes it as a required
|
||||
parameter, `StoreSessionAsync` stamps `register:new` / `register:whiteglove_resume` and
|
||||
preserves an existing path through the re-registration Replace, `SetSessionPreProvisionedAsync`
|
||||
requires it whenever it writes a status. Unlike `FailureSource` (persisted only on `Failed`),
|
||||
the path survives for Incomplete, AwaitingUser, Stalled and Pending.
|
||||
|
||||
| Origin | Paths |
|
||||
| --- | --- |
|
||||
| `agent` | `complete`, `complete_soft` (Continue-Anyway), `failed`, `esp_failure_fallback`, `gather_complete`, `whiteglove_pending`, `whiteglove_resumed`, `stall_probe`, `stall_heal` |
|
||||
| `ingest` | `wg_awaiting` (stall probe mapped to WhiteGlove AwaitingUser) |
|
||||
| `sweep` | `stalled`, `wg_awaiting`, `sd_reconcile`, `r{rule}` (stage-2 classifier) |
|
||||
| `maxlife` / `late` / `retro` | `r{rule}` (classifier via max-lifetime shutdown / late-telemetry heal / retro reclassification); `retro:sd_reconcile`, `retro:superseded` |
|
||||
| `register` | `new`, `whiteglove_resume`, `superseded` |
|
||||
| `rule` | `rule:{ruleId}` (analyze rule with MarkSessionAsFailed) |
|
||||
| `manual` | `failed`, `succeeded` |
|
||||
| `legacy` | read-side derivation only (`legacy:{rule}` when the rule is recoverable from the reason literal, `legacy:unknown` otherwise) — never written |
|
||||
|
||||
Classifier rule ids (`ClassifierRules`): `r1`, `r1b_awaiting`, `r1b_succeeded`, `r1c`, `r2`,
|
||||
`r3`, `r4`, `r5_awaiting`, `r5_incomplete`, `r6`. `ClassifyTimedOutSession` returns
|
||||
`(Status, Reason, Rule)` so each `return` names its rule.
|
||||
|
||||
## PriorStatus / PriorVerdictPath — the correction stream
|
||||
|
||||
When a write replaces a prior **verdict** (existing status ∈ Succeeded/Failed/Incomplete/
|
||||
AwaitingUser with a stamped path, incoming status different), the storage seam preserves the
|
||||
pair (`TableStorageService.ComputePriorVerdict`, both the ETag path and the force path). A
|
||||
stall marker healing to InProgress, a same-status refresh, or an unstamped legacy row never
|
||||
sets it. Admin marks, late agent completions and retro reclassifications all flow through it.
|
||||
|
||||
## Read-side derivation for pre-instrumentation rows
|
||||
|
||||
`VerdictPathDerivation.Derive(session)` returns the stamped path verbatim, otherwise derives
|
||||
from `AdminMarkedAction`, `FailureSource`, `ReconcileReason` and `FailureReason` using the
|
||||
writers' fixed literals (classifier rule prefixes, the `max-lifetime watchdog` / `Retro-
|
||||
reclassified` suffixes, `Rule: `, `Superseded by session`, `Late completion report received`).
|
||||
Unambiguous rows get the real path (`agent:complete`, `manual:failed`, `rule:…`,
|
||||
`maxlife:r6`), ambiguous-but-rule-known rows `legacy:{rule}`, the rest `legacy:unknown`. No
|
||||
mass backfill: the aggregate counts derived rows with `DerivedCount` so the matrix shows
|
||||
history while flagging its weaker evidence. Tests pin the literals against the classifier
|
||||
output so a wording change fails loudly.
|
||||
|
||||
## Daily aggregate — `VerdictCalibrationAggregates`
|
||||
|
||||
PK = `TenantId` | `"global"`, RK = `yyyy-MM-dd` (session **StartedAt** date, like every daily
|
||||
aggregate). `VerdictCalibrationDailyAggregate { Version, SessionCount, TerminalSessionCount,
|
||||
Buckets[], ComputedAt }` with one bucket per (VerdictPath, Status):
|
||||
|
||||
| Field | Meaning |
|
||||
| --- | --- |
|
||||
| `Count` / `DerivedCount` | sessions currently on the path; of those, derived read-side |
|
||||
| `Eligible7d` | terminal sessions whose end (`CompletedAt ?? LastEventAt ?? StartedAt`) lies ≥ 7 d before compute time |
|
||||
| `ReEnrolled7d` | of the eligible: the device's `DeviceHistories` chain shows the next terminal session starting < 7 d after this one's end |
|
||||
| `OverriddenByAdmin` / `OverriddenByLateCompletion` / `OverriddenOther` | sessions whose **PriorVerdictPath** is this bucket — attributed to the path that was overridden, so a bucket can carry overrides with `Count = 0` |
|
||||
|
||||
`MaintenanceService.SweepVerdictCalibrationAsync` (2-hourly, after the device-journey sweep
|
||||
so the chains are fresh; manual maintenance parity; kill switch
|
||||
`VerdictCalibrationSweepDisabled`) slices the tick's shared projected window scan
|
||||
(`LoadSweepWindowSessionsAsync` — one 35-day cross-tenant drain feeds the time-attribution,
|
||||
device-journey and calibration sweeps plus the radar's tenant discovery; the StartedAt-only
|
||||
filter is a full-table read, so never per sweep), recomputes the rolling 30-day window whole (Replace),
|
||||
deletes stale date rows it did not regenerate, and the unbounded-tables cleanup applies the
|
||||
180-day retention. Sessions inside a deletion cascade are skipped; the table is regenerable
|
||||
and is wiped with the tenant on offboarding.
|
||||
|
||||
## Read surfaces
|
||||
|
||||
* `GET global/metrics/verdict-calibration?days=&tenantId=` (GlobalReadOrAdmin,
|
||||
TenantScoping.QueryParam; `VerdictCalibrationResponse.Build`): per path — count, share of
|
||||
all window sessions, derived count, eligible/re-enrolled with `reEnrollRatePct` **null below
|
||||
20 eligible**, the three override counters, a today-anchored 7d-vs-28d share trend
|
||||
(`window7`, `baseline28`, `lift` null without baseline) — plus totals, the trend basis and
|
||||
the active `alerts[]`. Empty but well-formed before the first sweep.
|
||||
* MCP `get_verdict_calibration` (platform scope: GA + Global Reader). Same endpoint, LLM-shaped in
|
||||
`verdict-calibration-shape.ts`: per-row `window7.sessions` / `baseline28.sessions` are dropped (the
|
||||
denominators live once in `trend`), `reEnrollRatePct` / `lift` are always present (`null` = withheld),
|
||||
and `minSharePct` / `top` trim the one-session tail — rows carrying overrides are never trimmed and
|
||||
everything dropped is reported in `omitted`. The wire shape the admin page consumes is untouched.
|
||||
* Portal: Global Admin → Metrics → Verdict Calibration (`/admin/metrics/verdict-calibration`):
|
||||
matrix grouped by origin, status pill, `derived n` gray pill, trend glyph (arrow only from
|
||||
5 window hits and lift ≥ 2 / ≤ 0.5), re-enrollment rate or `— (n=…)`, overrides
|
||||
`admin / late / other`, `↑ Drift` pill on rows with an active episode, and the active
|
||||
episodes list.
|
||||
|
||||
## Drift radar — `VerdictCalibrationRadar`
|
||||
|
||||
Same statistics as the [rule regression radar](rule-regression-radar.md) (7-day window vs the
|
||||
prior 28 days, ≥ 5 window hits, ≥ 20 window sessions, lift ≥ 2, `MetricsMath.
|
||||
RateIncreaseSeparated`), anchored on yesterday, over every partition with rows (each tenant
|
||||
and `global`):
|
||||
|
||||
| Kind | Signal | Re-arm |
|
||||
| --- | --- | --- |
|
||||
| `share_regression` | one **backend-decided** path's share of all sessions doubled (`sweep`/`maxlife`/`late`/`retro`/`rule`/`manual`/`ingest`; `agent:*` and `register:*` mirror customer workflow mix — first prod pass fired on `agent:whiteglove_pending` rollout weeks — and `legacy:*` is derived, all excluded) | share < 1.5× baseline, the path stops occurring, or the path is no longer eligible |
|
||||
| `silence_share_regression` | the `sweep:*` + `maxlife:*` share doubled — the backend had to decide more often because the agent went silent (a liveness signal, not a classifier one) | same |
|
||||
| `evidence_gap` | absolute: `r6` (pure fallthrough) decides ≥ 20 % of the window's classifier verdicts, ≥ 20 verdicts | < 15 % or < 20 verdicts |
|
||||
|
||||
Episodes are tracker rows `verdictcalibration|{kind}|{path}|{status}` (register = `AddEntity`,
|
||||
409 = already burning, fail-closed; refresh carries `FirstNotifiedAt`; 30-day tracker
|
||||
retention re-arms long burners). A new episode fires **one** `VerdictCalibrationDrift` ops
|
||||
event (category Maintenance, Warning; dual-registered in the web `OPS_EVENT_TYPES`) — there
|
||||
is deliberately **no tenant bell**. Per-path findings on a tenant partition get the on-fire
|
||||
dimension correlation (`RuleRegressionRadar.ComputeDimensionConcentration` over the window's
|
||||
sessions whose derived path matches), reported with the shared wording contract "correlated,
|
||||
not necessarily causal"; group kinds and the global partition carry no dimension claim.
|
||||
Kill switch `VerdictCalibrationRadarDisabled`.
|
||||
|
||||
# Examples
|
||||
|
||||
* **Reading the thermometer (prod, 2026-08-23 hand analysis that motivated this):**
|
||||
re-enrollment within 7 days after Succeeded 8.9 % (lab background), after Failed 24.3 %,
|
||||
after Incomplete 10.6 % — Incomplete behaves like Succeeded, so rules 5/6 are probably too
|
||||
cautious; the matrix now shows this per rule instead of per status.
|
||||
* **Correction stream:** a session swept to `sweep:r5_incomplete`, later completed by the
|
||||
agent → row `agent:complete` with `PriorVerdictPath = sweep:r5_incomplete`; the aggregate
|
||||
increments `OverriddenByLateCompletion` on the `sweep:r5_incomplete / Incomplete` bucket.
|
||||
Four admin marks since launch were invisible before — now each one counts against the path
|
||||
it corrected.
|
||||
* **Evidence gap:** a tenant whose agent build stopped emitting the Device Setup rollup line
|
||||
drives its sweep verdicts into `r6`; the absolute gate fires at 20 % and names the
|
||||
tenant — the fix is agent-side, the classifier is only the messenger.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/VerdictPaths.cs` — vocabulary + `ClassifierRules`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs` — `ComputePriorVerdict`, `ApplyVerdictPath`, re-registration preserve
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/VerdictPathDerivation.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.VerdictCalibration.cs` — sweep + `BuildVerdictCalibrationAggregates`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetVerdictCalibrationFunction.cs`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/VerdictCalibrationRadar.cs`, `Services/MaintenanceService.VerdictCalibrationRadar.cs`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts` — `get_verdict_calibration`
|
||||
* `src/McpServer/autopilot-monitor-mcp/src/verdict-calibration-shape.ts` — LLM-facing response shaping (`minSharePct` / `top`, explicit nulls)
|
||||
* `src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionVerdictCalibration.tsx`
|
||||
* Tests: `VerdictPathTests`, `VerdictCalibrationTests`, `VerdictCalibrationRadarTests`
|
||||
* Related: [Silent-Session Classification](silent-session-classification.md), [Device Journeys](device-journeys.md), [Rule Regression Radar](rule-regression-radar.md)
|
||||
@@ -45,13 +45,16 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
|
||||
# Backend
|
||||
|
||||
* [Table Schema Sentinel](backend/table-schema-sentinel.md) - Startup no longer issues one CreateTableIfNotExists per table: a SHA-256 over TableNames.All in AdminConfiguration gates the full pass (derived, never hand-maintained), the SessionsIndex backfill runs only after a full pass behind a conditional-insert claim, and daily maintenance repairs out-of-band deletions.
|
||||
* [Business Timestamps](backend/business-timestamps.md) - Why the Azure Tables system Timestamp is never authoritative (migrations reset it), and the OccurredUtc + RowKey-decode compensation for AuditLogs/OpsEvents/Events after the 2026-07-18 migration.
|
||||
* [Send-Time Frame Separation](backend/send-time-frame-separation.md) - One device-clock send timestamp per upload batch (X-Send-Time-Utc → SentAt on Events) splits ReceivedAt − OccurredUtc into pure spool delay and pure device-vs-server clock offset; the clock_skew condition measures SentAt batches directly (spool-immune), the CMTrace tripwire deliberately stays on its differential math.
|
||||
* [OOBE Bootstrap go-Code Flow](backend/oobe-bootstrap-go-flow.md) - The bootstrap short-code lifecycle (portal-created session → `irm go.autopilotmonitor.com/CODE | iex` at Shift+F10 → GetBootstrapScriptFunction), the always-HTTP-200 text/plain contract, the two-line injection defense (GUID/URL/expiry validator + single-quoted PS literals, golden-pinned against the legacy TS template), and the per-IP rate-limit story behind Front Door.
|
||||
* [Silent-Session Classification](backend/silent-session-classification.md) - How a session whose agent went silent gets its honest status: the two-stage maintenance sweep (2h Stalled marker with the WhiteGlove Part-2 AwaitingUser gate and the self-deploying Succeeded gate, 5h timeout classification), the EnrollmentTimeoutClassifier rule order (explicit failure → WG Part 2 → self-deploying → completion evidence → emergency break → desktop+Hello → Device Setup/grace), the ingest paths sharing it (max-lifetime, late-telemetry heal, stall probe) and the admin retro-reconcile modes incl. self_deploying_silent.
|
||||
* [Device Journeys & First-Time-Right](backend/device-journeys.md) - How a device's terminal enrollment attempts are grouped into journeys (trim+casefold device key with junk-serial exclusion, first-success end, 30-day gap, WhiteGlove = one attempt), the DeviceHistories chain with its tombstone-driven deleted-session pruning, the additive daily FTR aggregates, and the banner/fleet/MCP surfaces.
|
||||
* [Enrollment Time Attribution](backend/time-attribution.md) - How a terminal session's authoritative wall clock (DurationSeconds, WhiteGlove pause excluded) is partitioned into named segments with an explicit unattributed remainder, ESP-blocking apps as positive evidence with "up to" what-if bounds, reboot spans from event gaps, and the breakdown/aggregate tables the sweep maintains.
|
||||
* [Attempt-Scoped App Install Durations](backend/app-install-duration-attempt.md) - AppInstallSummary.DurationSeconds = status-defining (last) install attempt via the LastAttemptStartedAt max-fold anchor instead of the first-observation→last-terminal span that billed multi-pass IME gaps to trivial apps; InstallPassCount, the weaker-terminal guard, the radar's cutover gate, and time attribution's union-of-attempt-segments occupancy.
|
||||
* [Rule Regression Radar](backend/rule-regression-radar.md) - How analyze-rule hit-rate regressions are detected (7-day window ≥2× the 28-day baseline with disjoint Wilson intervals), which suppressions keep the radar quiet (grace period, edit-in-window, kill switch), how alert episodes live in the ruleregression tracker keyspace with re-arm semantics, and the on-fire dimension correlation with its "correlated, not necessarily causal" wording contract.
|
||||
* [Verdict Calibration](backend/verdict-calibration.md) - How every session status write declares its VerdictPath (origin:detail vocabulary incl. classifier rule ids), how overriding a verdict preserves PriorStatus/PriorVerdictPath as the correction stream, the daily per-path calibration aggregate with the 7-day re-enrollment proxy and read-side legacy derivation, the GA endpoint / MCP tool / admin matrix, and the drift radar (share, silence-share, r6 evidence gap) with its operator-only ops events.
|
||||
* [App Version Duration Regression Radar](backend/app-version-regression-radar.md) - How app versions whose median install duration regressed vs. the previous version are detected (first-seen-ordered versions, median lift ≥2× plus a 5-minute absolute floor over measured installs only), how alert episodes live in the appversionregression tracker keyspace with drain/median re-arm semantics, and the bell/ops-event/analytics surfaces.
|
||||
* [Plan Downgrade Policy & the Retention Grace Period](backend/plan-downgrade-retention-grace.md) - What happens when a tenant loses Pro: every entitlement gates read-time and reversibly ("gate, don't destroy"), except retention — a 30-day grace keeps the Pro cap before the Community cap starts hard-deleting; the ProDowngradedUtc/trial-expiry anchor model, the TenantPlanDowngraded/RetentionGrace* ops events, and the deliberate non-grace of everything else.
|
||||
* [Tenant Activation & the Once-Only Welcome Mail](backend/tenant-activation-welcome-mail.md) - How a signup becomes an activated tenant (manual approve or the auto-approve queue), and the write-then-read + conditional-marker arbitration that sends the welcome mail exactly once even though activation and the user-typed notification address race each other.
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
|
||||
## 2026-08-23
|
||||
|
||||
* **Update**: `backend/table-schema-sentinel.md` — "Measuring it": `StartupTelemetryService` emits `BackendStartupMs` (process start → ApplicationStarted) and `BackendTableInitMs` as App Insights custom metrics with version/commit/fullPass dimensions; the MCP server writes one `{"type":"boot","readyMs"}` stderr line per start (query in `.claude/commands/mcp-telemetry.md` §A2).
|
||||
* **New**: `backend/table-schema-sentinel.md` — startup table initialization gated by a schema sentinel (SHA-256 over `TableNames.All` in `AdminConfiguration`/`SchemaSentinel`/`tables`): matching sentinel ⇒ zero `CreateTableIfNotExists` calls on a cold start, missing/stale ⇒ one full pass + sentinel rewrite. SessionsIndex startup backfill only after a full pass, `IsSessionIndexEmptyAsync` no longer treats errors as "empty", conditional-insert claim so one scaled-out instance backfills. Daily maintenance runs `IStorageInitializer.EnsureAllAsync` to repair out-of-band table deletions.
|
||||
* **Update**: `backend/verdict-calibration.md` — MCP `get_verdict_calibration` response shaping: per-row trend denominators dropped, explicit nulls, `minSharePct` / `top` with an `omitted` report.
|
||||
* **Update**: `backend/verdict-calibration.md` — share-regression radar restricted to backend-decided paths; agent-declared/registration paths (customer workflow mix) and derived legacy paths no longer alert, stale episodes re-arm themselves.
|
||||
* **Update**: `backend/time-attribution.md`, `backend/device-journeys.md`, `backend/verdict-calibration.md` — the rolling maintenance sweeps (time attribution, device journeys, verdict calibration, calibration-radar discovery) now share ONE projected 35-day cross-tenant Sessions scan per tick (`LoadSweepWindowSessionsAsync` + `MaintenanceSweepSessionProjection`, pinned by `MaintenanceSweepProjectionEquivalenceTests`) instead of four full-table drains; each sweep slices its own window by StartedAt.
|
||||
* **New**: `backend/verdict-calibration.md` — the classifier thermometer: every status write now stamps `VerdictPath` (required parameter on `UpdateSessionStatusAsync`; `EnrollmentTimeoutClassifier` returns its rule id), overriding a verdict preserves `PriorStatus`/`PriorVerdictPath`, `VerdictPathDerivation` attributes pre-instrumentation rows read-side (`legacy:*` when ambiguous). New daily aggregate `VerdictCalibrationAggregates` (per-path count/derived/eligible7d/reEnrolled7d/overridden-by-admin|late|other, sweep after the device-journey pass, 180d retention), `GET global/metrics/verdict-calibration`, MCP `get_verdict_calibration`, admin metrics section, and the `VerdictCalibrationRadar` (share / silence-share / r6 evidence-gap episodes in the tracker keyspace `verdictcalibration|`, `VerdictCalibrationDrift` ops event, no tenant bell).
|
||||
* **New**: `backend/silent-session-classification.md` — one place for the silent-session verdict: `EnrollmentTimeoutClassifier` rule order incl. the new rule 1c `IsSelfDeployingProvisioned` (self-deploying profile + Device Setup all-succeeded + no explicit failure ⇒ Succeeded reconcile, no grace/AwaitingUser — kiosk tenant aebdce78 audit: 836 finished devices parked AwaitingUser/Incomplete), the sweep stage-1 gates (`TryMarkWhiteGloveAwaitingUserAsync`, new `TryReconcileSelfDeployingAsync`), the ingest callers (max-lifetime, late-telemetry heal, stall probe) and the new admin retro mode `self_deploying_silent` (`GetSelfDeployingSilentSessionsAsync`). Agent companion: `SessionRegistrationHelper` waits up to 15s for a network link and retries 6× (~62s) instead of 5× (~30s) so the reboot relaunch on Wi-Fi no longer exits 7 before the link is back.
|
||||
* **New**: `agent/diagnostics-package.md` — the diagnostics ZIP's built-in collection is now ONE catalog in Shared (`DiagnosticsBuiltInSections.All`): the agent iterates it in order and the backend serves the same list to the portal (`GET diagnostics/paths`, MemberRead), so what administrators see is exactly what the agent collects. New conditional sections: RealmJoin client/package/Chocolatey/per-user logs behind the tenant's RealmJoin Watcher toggle (mirrored onto `AgentConfiguration.EnableRealmJoinWatcher` by the merger) and the Device Preparation bootstrapper event log behind the deterministic WDP marker (probe once per build, fail toward Classic); every skip is a manifest line. Built-ins bypass `DiagnosticsPathGuards` by design (reviewed code, `%ProgramData%\RealmJoin` is not allowlisted); caps/reparse/evtx export still apply. `MergeDiagnosticsLogPaths` dedups global ∪ tenant paths. Portal: collapsed read-only catalog + one-line rows in both cards, tenant context dropped the GA-only global fetch.
|
||||
* **Update**: `rules/gather-rule-guardrails.md` — `DiagnosticsPathGuards` scope clarified: configured diagnostics paths only, never the built-in catalog.
|
||||
* **Update**: `backend/tenant-activation-welcome-mail.md` — new "Template overrides" section: Global Admins can replace the welcome/farewell HTML in the portal (Settings → Email Templates); `EmailTemplateService` resolves override-or-built-in per send (PreviewConfig table, partition `EmailTemplates`, 5-min cache, `{{domainName}}` placeholder, 30k limit, fail-soft send path), GA endpoints `global/email-templates/{kind}` (+`/test` to the caller's tenant contact address). Also: the mail header is image-free (CSS-drawn brand mark, gradient stripes and clip-path edge as progressive enhancement).
|
||||
|
||||
+14
-2
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
|
||||
@@ -11,8 +12,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
internal sealed class FakeSignalIngressSink : ISignalIngressSink
|
||||
{
|
||||
private readonly List<PostedSignal> _posted = new List<PostedSignal>();
|
||||
private readonly object _gate = new object();
|
||||
|
||||
public IReadOnlyList<PostedSignal> Posted => _posted;
|
||||
/// <summary>
|
||||
/// Snapshot, not the live list: producers post from timer / worker threads (e.g. the
|
||||
/// parked-dwell tripwire) while tests poll this in a SpinWait loop. Enumerating the live
|
||||
/// list across an Add throws "Collection was modified" — a rare CI-only failure
|
||||
/// (run 32644425162) with no assertion text.
|
||||
/// </summary>
|
||||
public IReadOnlyList<PostedSignal> Posted
|
||||
{
|
||||
get { lock (_gate) return _posted.ToArray(); }
|
||||
}
|
||||
|
||||
/// <summary>When non-null, <see cref="Post"/> throws instead of capturing.</summary>
|
||||
public Exception? ThrowOnPost { get; set; }
|
||||
@@ -27,7 +38,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
object? typedPayload = null)
|
||||
{
|
||||
if (ThrowOnPost != null) throw ThrowOnPost;
|
||||
_posted.Add(new PostedSignal(kind, occurredAtUtc, sourceOrigin, evidence, payload, kindSchemaVersion, typedPayload));
|
||||
lock (_gate)
|
||||
_posted.Add(new PostedSignal(kind, occurredAtUtc, sourceOrigin, evidence, payload, kindSchemaVersion, typedPayload));
|
||||
}
|
||||
|
||||
internal sealed class PostedSignal
|
||||
|
||||
+45
-3
@@ -97,7 +97,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fails_after_five_exceptions_without_auth_error()
|
||||
public async Task Fails_after_six_exceptions_without_auth_error()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
@@ -111,8 +111,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
backoffDelay: _ => Task.CompletedTask);
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.Failed, result.Outcome);
|
||||
Assert.Equal(5, apiClient.CallCount);
|
||||
Assert.Contains("transient attempt 5", result.ErrorMessage);
|
||||
Assert.Equal(6, apiClient.CallCount);
|
||||
Assert.Contains("transient attempt 6", result.ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Waits_for_network_link_before_first_attempt()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
var config = NewConfig();
|
||||
|
||||
var order = new System.Collections.Generic.List<string>();
|
||||
var apiClient = new FakeApiClient(_ =>
|
||||
{
|
||||
order.Add("register");
|
||||
return new RegisterSessionResponse { Success = true, SessionId = config.SessionId };
|
||||
});
|
||||
|
||||
var result = await SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient, config, agentVersion: "0.0.0", logger: logger,
|
||||
backoffDelay: _ => Task.CompletedTask,
|
||||
networkLinkWait: _ => { order.Add("link-wait"); return Task.CompletedTask; });
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.Succeeded, result.Outcome);
|
||||
Assert.Equal(new[] { "link-wait", "register" }, order);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Network_link_wait_failure_does_not_block_registration()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
var config = NewConfig();
|
||||
|
||||
var apiClient = new FakeApiClient(_ =>
|
||||
new RegisterSessionResponse { Success = true, SessionId = config.SessionId });
|
||||
|
||||
var result = await SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient, config, agentVersion: "0.0.0", logger: logger,
|
||||
backoffDelay: _ => Task.CompletedTask,
|
||||
networkLinkWait: _ => throw new InvalidOperationException("probe broken"));
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.Succeeded, result.Outcome);
|
||||
Assert.Equal(1, apiClient.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
@@ -19,7 +20,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// <b>Retry contract (V1 <c>MonitoringService.RegisterSessionAsync</c>):</b>
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>5 attempts total; between attempts <c>2^attempt</c> seconds (2s, 4s, 8s, 16s).</item>
|
||||
/// <item>Boot-time NIC grace before the first attempt: when no network link is up yet
|
||||
/// (BootTrigger relaunch after a mid-enrollment reboot, Wi-Fi still associating) wait
|
||||
/// up to <see cref="NetworkLinkWaitMax"/> for the link-level signal. Best-effort and
|
||||
/// free on the normal path — a live link returns immediately.</item>
|
||||
/// <item>6 attempts total; between attempts <c>2^attempt</c> seconds (2s, 4s, 8s, 16s, 32s —
|
||||
/// ~62 s of backoff). The original V1 budget of 5 attempts / ~30 s was too short for the
|
||||
/// reboot-relaunch on Wi-Fi kiosks: the relaunched agent gave up and exited 7 before
|
||||
/// the network came back, leaving the session silent forever (tenant aebdce78,
|
||||
/// 2026-08-23 audit). A relaunch that misses here is lost — nothing retries later.</item>
|
||||
/// <item>On <c>response.Success == true</c> → stop immediately.</item>
|
||||
/// <item>On <see cref="BackendAuthException"/> (401/403) → report to the
|
||||
/// <see cref="AuthFailureTracker"/> (which fires the first-failure distress + may trip
|
||||
@@ -37,16 +46,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// </summary>
|
||||
public static class SessionRegistrationHelper
|
||||
{
|
||||
private const int MaxAttempts = 5;
|
||||
private const int MaxAttempts = 6;
|
||||
|
||||
/// <summary>Upper bound for the pre-registration link wait (see class remarks).</summary>
|
||||
internal static readonly TimeSpan NetworkLinkWaitMax = TimeSpan.FromSeconds(15);
|
||||
|
||||
/// <summary>
|
||||
/// Executes the 5-retry register-session handshake against the backend. Returns the
|
||||
/// Executes the retrying register-session handshake against the backend. Returns the
|
||||
/// response + classified outcome. Never throws.
|
||||
/// </summary>
|
||||
/// <param name="backoffDelay">
|
||||
/// Optional delay-provider used by tests to avoid real-time waits. Production callers
|
||||
/// pass <c>null</c> → V1 parity <c>Task.Delay(2^attempt * 1000)</c> between attempts.
|
||||
/// </param>
|
||||
/// <param name="networkLinkWait">
|
||||
/// Optional pre-registration link wait used by tests. Production callers pass
|
||||
/// <c>null</c> → <see cref="WaitForNetworkLinkAsync"/> (bounded by <see cref="NetworkLinkWaitMax"/>).
|
||||
/// </param>
|
||||
public static async Task<SessionRegistrationResult> RegisterWithRetryAsync(
|
||||
BackendApiClient apiClient,
|
||||
AgentConfiguration agentConfig,
|
||||
@@ -56,14 +72,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
EmergencyReporter emergencyReporter = null,
|
||||
Func<int, Task> backoffDelay = null,
|
||||
Func<Exception, Task> onTerminalTransportFailure = null,
|
||||
(string Manufacturer, string Model, string SerialNumber)? deviceHardware = null)
|
||||
(string Manufacturer, string Model, string SerialNumber)? deviceHardware = null,
|
||||
Func<AgentLogger, Task> networkLinkWait = null)
|
||||
{
|
||||
if (apiClient == null) throw new ArgumentNullException(nameof(apiClient));
|
||||
if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig));
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
backoffDelay ??= DefaultBackoffDelay;
|
||||
networkLinkWait ??= WaitForNetworkLinkAsync;
|
||||
var registration = BuildRegistration(agentConfig, agentVersion, deviceHardware);
|
||||
|
||||
try { await networkLinkWait(logger).ConfigureAwait(false); }
|
||||
catch (Exception ex) { logger.Debug($"Pre-registration network wait failed ({ex.Message}) — registering anyway."); }
|
||||
string lastError = null;
|
||||
Exception lastException = null;
|
||||
|
||||
@@ -113,7 +134,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
|
||||
if (attempt < MaxAttempts)
|
||||
{
|
||||
var delaySeconds = (int)Math.Pow(2, attempt); // 2, 4, 8, 16
|
||||
var delaySeconds = (int)Math.Pow(2, attempt); // 2, 4, 8, 16, 32
|
||||
logger.Info($"Retrying session registration in {delaySeconds}s");
|
||||
await backoffDelay(attempt).ConfigureAwait(false);
|
||||
}
|
||||
@@ -143,6 +164,31 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private static Task DefaultBackoffDelay(int attempt)
|
||||
=> Task.Delay(((int)Math.Pow(2, attempt)) * 1000);
|
||||
|
||||
/// <summary>
|
||||
/// Boot-time NIC grace (same pattern as the emergency-break reporter): polls the cheap
|
||||
/// link-level signal once per second for at most <see cref="NetworkLinkWaitMax"/>. It
|
||||
/// cannot prove backend reachability — the retry loop handles that — it only keeps the
|
||||
/// first attempts from being burned into a link that is provably still down. Probe
|
||||
/// errors end the wait, never the registration.
|
||||
/// </summary>
|
||||
internal static async Task WaitForNetworkLinkAsync(AgentLogger logger)
|
||||
{
|
||||
if (NetworkInterface.GetIsNetworkAvailable()) return;
|
||||
|
||||
logger?.Info($"Session registration: no network link yet — waiting up to {NetworkLinkWaitMax.TotalSeconds:F0}s.");
|
||||
var deadline = DateTime.UtcNow + NetworkLinkWaitMax;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(1000).ConfigureAwait(false);
|
||||
if (NetworkInterface.GetIsNetworkAvailable())
|
||||
{
|
||||
logger?.Info("Session registration: network link is up.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
logger?.Warning($"Session registration: still no network link after {NetworkLinkWaitMax.TotalSeconds:F0}s — attempting anyway.");
|
||||
}
|
||||
|
||||
private static SessionRegistration BuildRegistration(
|
||||
AgentConfiguration agentConfig,
|
||||
string agentVersion,
|
||||
@@ -194,7 +240,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// <summary>401/403 from the backend — cert/token rejected. No retry was attempted.</summary>
|
||||
AuthFailed = 1,
|
||||
|
||||
/// <summary>All 5 attempts failed with non-auth errors (network, 5xx, malformed response).</summary>
|
||||
/// <summary>All attempts failed with non-auth errors (network, 5xx, malformed response).</summary>
|
||||
Failed = 2,
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
/// <summary>
|
||||
/// Phase 6 of <see cref="Program"/>'s <c>RunAgent</c>: blocking
|
||||
/// <c>POST /api/agent/register-session</c> with 5-retry exponential backoff before the
|
||||
/// <c>POST /api/agent/register-session</c> with link-wait + 6-retry exponential backoff before the
|
||||
/// orchestrator starts. V1 parity (MonitoringService.RegisterSessionAsync) — without
|
||||
/// this call the backend's Sessions table never gets a row for this session, so
|
||||
/// <c>IncrementSessionEventCountAsync</c> / <c>UpdateSessionStatusAsync</c> silently
|
||||
|
||||
@@ -163,6 +163,12 @@ public class EnrollmentTimeoutClassifierTests
|
||||
// -------- ClassifyTimedOutSession --------
|
||||
|
||||
private static (SessionStatus, string) Classify(IReadOnlyList<EnrollmentEvent> events, double hoursSinceStart = 6, int grace = 72)
|
||||
{
|
||||
var (status, reason, _) = ClassifyWithRule(events, hoursSinceStart, grace);
|
||||
return (status, reason);
|
||||
}
|
||||
|
||||
private static (SessionStatus, string, string) ClassifyWithRule(IReadOnlyList<EnrollmentEvent> events, double hoursSinceStart = 6, int grace = 72)
|
||||
{
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var now = Start.AddHours(hoursSinceStart);
|
||||
@@ -220,7 +226,7 @@ public class EnrollmentTimeoutClassifierTests
|
||||
Esp(DeviceSetup44), Esp(AccountSetup15),
|
||||
Evt("desktop_arrived"), Evt("hello_provisioning_completed"), Evt("realmjoin_detected"),
|
||||
});
|
||||
var (status, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (status, reason, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, Start, now, graceHours: 72, lastEventAtUtc: lastEvent);
|
||||
|
||||
Assert.Equal(SessionStatus.Succeeded, status);
|
||||
@@ -481,8 +487,9 @@ public class EnrollmentTimeoutClassifierTests
|
||||
{
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var now = Start.AddHours(hoursSinceStart);
|
||||
return EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (status, reason, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, Start, now, grace, isPreProvisioned: true, resumedAt: Resumed);
|
||||
return (status, reason);
|
||||
}
|
||||
|
||||
/// <summary>The fairstone event shape: Part 1 sealed, Part 2 resumed, nobody signed in.</summary>
|
||||
@@ -611,8 +618,89 @@ public class EnrollmentTimeoutClassifierTests
|
||||
{
|
||||
Esp(DeviceSetup44), Evt("whiteglove_complete"), Esp(AccountSetup05),
|
||||
});
|
||||
var (status, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (status, _, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, Start, Start.AddHours(6), 51, isPreProvisioned: true, resumedAt: null);
|
||||
Assert.Equal(SessionStatus.Succeeded, status);
|
||||
}
|
||||
|
||||
// -------- Self-deploying profile gate (kiosk tenant aebdce78, audit 2026-08-23) --------
|
||||
|
||||
private static (SessionStatus, string) ClassifySelfDeploying(
|
||||
IReadOnlyList<EnrollmentEvent> events, double hoursSinceStart = 3, int grace = 51)
|
||||
{
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var (status, reason, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, Start, Start.AddHours(hoursSinceStart), grace,
|
||||
lastEventAtUtc: Start.AddMinutes(4), isSelfDeployingProfile: true);
|
||||
return (status, reason);
|
||||
}
|
||||
|
||||
// Session 195593e2 replay: DeviceSetup 4/4 (fallback-confirmed), AccountSetup registry
|
||||
// at 0/5 (the IME false positive — user ESP never runs on this profile), Hello disabled,
|
||||
// SkipUser=True, then silence after the post-ESP reboot. No user will ever sign in.
|
||||
private static readonly EnrollmentEvent[] KioskSilentStream =
|
||||
{
|
||||
Esp(DeviceSetup44), Esp(AccountSetup05), Esp(DeviceSetupFallback),
|
||||
HelloPolicy(enabled: false), EspConfig(skipUser: true),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void IsSelfDeployingProvisioned_requires_flag_device_setup_and_no_failure()
|
||||
{
|
||||
var provisioned = EnrollmentTimeoutClassifier.ExtractRollup(KioskSilentStream);
|
||||
Assert.True(EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned(provisioned, isSelfDeployingProfile: true));
|
||||
Assert.False(EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned(provisioned, isSelfDeployingProfile: false));
|
||||
|
||||
var beforeDeviceSetup = EnrollmentTimeoutClassifier.ExtractRollup(new[] { Esp(AccountSetup05) });
|
||||
Assert.False(EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned(beforeDeviceSetup, isSelfDeployingProfile: true));
|
||||
|
||||
var failed = EnrollmentTimeoutClassifier.ExtractRollup(new[] { Esp(DeviceSetup44), Evt("enrollment_failed") });
|
||||
Assert.False(EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned(failed, isSelfDeployingProfile: true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_self_deploying_silent_after_device_setup_reconciles_to_Succeeded_within_grace()
|
||||
{
|
||||
var (status, reason) = ClassifySelfDeploying(KioskSilentStream, hoursSinceStart: 3);
|
||||
Assert.Equal(SessionStatus.Succeeded, status);
|
||||
Assert.Contains("self-deploying profile", reason);
|
||||
Assert.Contains("no user / Account Setup phase", reason);
|
||||
Assert.Contains("Agent last reported 2026-07-06 15:04 UTC", reason);
|
||||
Assert.DoesNotContain("Account Setup completed", reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_self_deploying_silent_after_device_setup_reconciles_to_Succeeded_past_grace()
|
||||
{
|
||||
// Grace is a user-phase concept; it does not apply when there is no user phase.
|
||||
var (status, _) = ClassifySelfDeploying(KioskSilentStream, hoursSinceStart: 60, grace: 51);
|
||||
Assert.Equal(SessionStatus.Succeeded, status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_same_stream_without_self_deploying_flag_keeps_user_phase_verdicts()
|
||||
{
|
||||
// Guard against the gate leaking into user-driven sessions: the identical event shape
|
||||
// on a user-driven profile is still "awaiting user" within grace / Incomplete past it.
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(KioskSilentStream);
|
||||
var (within, _, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(rollup, Start, Start.AddHours(3), 51);
|
||||
var (past, _, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(rollup, Start, Start.AddHours(60), 51);
|
||||
Assert.Equal(SessionStatus.AwaitingUser, within);
|
||||
Assert.Equal(SessionStatus.Incomplete, past);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_self_deploying_explicit_failure_stays_Failed()
|
||||
{
|
||||
var (status, _) = ClassifySelfDeploying(new[] { Esp(DeviceSetup44), Evt("enrollment_failed") });
|
||||
Assert.Equal(SessionStatus.Failed, status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_self_deploying_silent_before_device_setup_is_Incomplete()
|
||||
{
|
||||
// The profile flag alone proves nothing — Device ESP must have finished.
|
||||
var (status, _) = ClassifySelfDeploying(new[] { Esp(AccountSetup05), Evt("agent_started") });
|
||||
Assert.Equal(SessionStatus.Incomplete, status);
|
||||
}
|
||||
}
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins that the ONE projected window scan the maintenance tick shares across its rolling
|
||||
/// sweeps (<c>GetMaintenanceWindowSessionsAsync</c> / <c>MaintenanceSweepSessionProjection</c>)
|
||||
/// carries every field the time-attribution, device-journey and verdict-calibration sweeps
|
||||
/// read. In Azure Table Storage a <c>$select</c> returns ONLY the listed properties, so the
|
||||
/// projected row here is built from the production array — dropping a column there fails the
|
||||
/// sweep that consumes it. Mirrors <see cref="UsageMetricsProjectionEquivalenceTests"/>.
|
||||
/// </summary>
|
||||
public class MaintenanceSweepProjectionEquivalenceTests
|
||||
{
|
||||
private const string TenantId = "00000000-0000-0000-0000-000000000abc";
|
||||
private static readonly DateTime Started = new(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static readonly TableStorageService Sut =
|
||||
new(new Mock<TableServiceClient>().Object, NullLogger<TableStorageService>.Instance);
|
||||
|
||||
/// <summary>Full wide Sessions row with every column the sweeps read set to a distinct non-default value, plus noise.</summary>
|
||||
private static TableEntity FullRow() => new(TenantId, "11111111-1111-1111-1111-111111111111")
|
||||
{
|
||||
["Status"] = "Succeeded",
|
||||
["StartedAt"] = new DateTimeOffset(Started),
|
||||
["CompletedAt"] = new DateTimeOffset(Started.AddHours(1)),
|
||||
["LastEventAt"] = new DateTimeOffset(Started.AddHours(2)),
|
||||
["DurationSeconds"] = 3600,
|
||||
["EventCount"] = 321,
|
||||
["DeletionState"] = "None",
|
||||
["SerialNumber"] = "SN-1234",
|
||||
["Manufacturer"] = "Contoso",
|
||||
["Model"] = "Model-X",
|
||||
["EnrollmentType"] = "v2",
|
||||
["IsPreProvisioned"] = true,
|
||||
["IsSelfDeployingProfile"] = true,
|
||||
["IsUserDriven"] = false,
|
||||
["ResumedAt"] = new DateTimeOffset(Started.AddMinutes(30)),
|
||||
["AdminMarkedAction"] = "Succeeded",
|
||||
["VerdictPath"] = "manual:succeeded",
|
||||
["PriorStatus"] = "Incomplete",
|
||||
["PriorVerdictPath"] = "sweep:r6",
|
||||
["FailureReason"] = "No Device Setup completion or explicit failure signal observed before timeout",
|
||||
["ReconcileReason"] = "Reconciled at timeout: user completed setup (desktop + Windows Hello)",
|
||||
["FailureSource"] = "manual",
|
||||
["EspSoftFailure"] = true,
|
||||
// Noise the sweeps never read — present on the wide row, dropped by the projection.
|
||||
["DeviceName"] = "PC-FULL",
|
||||
["OsName"] = "Windows 11",
|
||||
["OsBuild"] = "26200.1234",
|
||||
["GeoCountry"] = "DE",
|
||||
["AgentVersion"] = "2.9.0",
|
||||
["DiagnosticsBlobName"] = "diag.zip",
|
||||
["FailureSnapshotJson"] = "{\"big\":\"" + new string('x', 2000) + "\"}",
|
||||
};
|
||||
|
||||
private static TableEntity Project(TableEntity full)
|
||||
{
|
||||
var keep = new HashSet<string>(TableStorageService.MaintenanceSweepSessionProjection, StringComparer.Ordinal);
|
||||
var projected = new TableEntity(full.PartitionKey, full.RowKey);
|
||||
foreach (var kv in full)
|
||||
{
|
||||
if (keep.Contains(kv.Key)) projected[kv.Key] = kv.Value;
|
||||
}
|
||||
return projected;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Projection_drops_the_wide_columns()
|
||||
{
|
||||
var projected = Project(FullRow());
|
||||
Assert.False(projected.ContainsKey("FailureSnapshotJson"));
|
||||
Assert.False(projected.ContainsKey("GeoCountry"));
|
||||
Assert.False(projected.ContainsKey("OsBuild"));
|
||||
Assert.True(projected.Count < FullRow().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_field_the_sweeps_read_survives_the_projection()
|
||||
{
|
||||
var full = Sut.MapToSessionSummary(FullRow());
|
||||
var proj = Sut.MapToSessionSummary(Project(FullRow()));
|
||||
|
||||
// Shared / structural
|
||||
Assert.Equal(full.TenantId, proj.TenantId);
|
||||
Assert.Equal(full.SessionId, proj.SessionId);
|
||||
Assert.Equal(full.Status, proj.Status);
|
||||
Assert.Equal(full.StartedAt, proj.StartedAt);
|
||||
Assert.Equal(full.CompletedAt, proj.CompletedAt);
|
||||
Assert.Equal(full.DeletionState, proj.DeletionState);
|
||||
|
||||
// Time attribution: Status/DurationSeconds/DeletionState/EventCount + enrollment class
|
||||
Assert.Equal(full.DurationSeconds, proj.DurationSeconds);
|
||||
Assert.Equal(full.EventCount, proj.EventCount);
|
||||
Assert.Equal(TimeAttributionCalculator.GetEnrollmentClass(full), TimeAttributionCalculator.GetEnrollmentClass(proj));
|
||||
|
||||
// Device journeys: BuildSessionRef + BuildDeviceHistoryRow display fields
|
||||
Assert.Equal(full.SerialNumber, proj.SerialNumber);
|
||||
Assert.Equal(full.Manufacturer, proj.Manufacturer);
|
||||
Assert.Equal(full.Model, proj.Model);
|
||||
Assert.Equal(full.EnrollmentType, proj.EnrollmentType);
|
||||
Assert.Equal(full.IsPreProvisioned, proj.IsPreProvisioned);
|
||||
Assert.Equal(full.AdminMarkedAction, proj.AdminMarkedAction);
|
||||
var refFull = DeviceJourneyCalculator.BuildSessionRef(full)!;
|
||||
var refProj = DeviceJourneyCalculator.BuildSessionRef(proj)!;
|
||||
Assert.Equal(System.Text.Json.JsonSerializer.Serialize(refFull), System.Text.Json.JsonSerializer.Serialize(refProj));
|
||||
|
||||
// Verdict calibration: attribution + legacy derivation inputs + proxy timestamps
|
||||
Assert.Equal(full.LastEventAt, proj.LastEventAt);
|
||||
Assert.Equal(full.VerdictPath, proj.VerdictPath);
|
||||
Assert.Equal(full.PriorStatus, proj.PriorStatus);
|
||||
Assert.Equal(full.PriorVerdictPath, proj.PriorVerdictPath);
|
||||
Assert.Equal(full.FailureReason, proj.FailureReason);
|
||||
Assert.Equal(full.ReconcileReason, proj.ReconcileReason);
|
||||
Assert.Equal(full.FailureSource, proj.FailureSource);
|
||||
Assert.Equal(full.EspSoftFailure, proj.EspSoftFailure);
|
||||
Assert.Equal(VerdictPathDerivation.Derive(full), VerdictPathDerivation.Derive(proj));
|
||||
// Unstamped variant exercises the derivation on the projected inputs.
|
||||
var unstampedFull = FullRow(); unstampedFull.Remove("VerdictPath"); unstampedFull.Remove("AdminMarkedAction"); unstampedFull.Remove("FailureSource");
|
||||
Assert.Equal(VerdictPathDerivation.Derive(Sut.MapToSessionSummary(unstampedFull)),
|
||||
VerdictPathDerivation.Derive(Sut.MapToSessionSummary(Project(unstampedFull))));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregation_cores_produce_identical_rows_from_the_projected_scan()
|
||||
{
|
||||
var now = new DateTime(2026, 8, 23, 12, 0, 0, DateTimeKind.Utc);
|
||||
var fullList = new List<SessionSummary> { Sut.MapToSessionSummary(FullRow()) };
|
||||
var projList = new List<SessionSummary> { Sut.MapToSessionSummary(Project(FullRow())) };
|
||||
var histories = new Dictionary<string, List<DeviceHistory>>();
|
||||
|
||||
var a = MaintenanceService.BuildVerdictCalibrationAggregates(fullList, histories, now);
|
||||
var b = MaintenanceService.BuildVerdictCalibrationAggregates(projList, histories, now);
|
||||
Assert.Equal(System.Text.Json.JsonSerializer.Serialize(a), System.Text.Json.JsonSerializer.Serialize(b));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SliceSweepWindow_keeps_sessions_from_the_window_start()
|
||||
{
|
||||
var window = new List<SessionSummary>
|
||||
{
|
||||
new() { SessionId = "old", StartedAt = Started.AddDays(-40) },
|
||||
new() { SessionId = "edge", StartedAt = Started },
|
||||
new() { SessionId = "new", StartedAt = Started.AddDays(1) },
|
||||
};
|
||||
var sliced = MaintenanceService.SliceSweepWindow(window, Started);
|
||||
Assert.Equal(new[] { "edge", "new" }, sliced.Select(s => s.SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Shared_window_covers_every_sweeps_horizon()
|
||||
{
|
||||
// 7d window + 28d baseline anchored on yesterday = 35 days back ≥ the 30-day rolling sweeps.
|
||||
Assert.Equal(35, MaintenanceService.SweepWindowDays);
|
||||
Assert.True(MaintenanceService.SweepWindowDays >= 30);
|
||||
}
|
||||
}
|
||||
@@ -112,9 +112,11 @@ public class OffboardingSessionEnumeratorTests
|
||||
public Task<List<SessionSummary>> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate, int maxResults = int.MaxValue, bool excludeInFlightDeletions = false) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetUsageWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetMaintenanceWindowSessionsAsync(DateTime startDate, DateTime endDate) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetGeoWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSelfDeployingSilentSessionsAsync(string tenantId, int maxResults) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
|
||||
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
using System.Collections.Concurrent;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Both sliding-window limiters keep the request history and its lock in one cache entry.
|
||||
/// Previously the entry was re-Set without its eviction callback, which (a) leaked per-key lock
|
||||
/// objects and (b) let two threads mutate the same List under different locks. These tests hammer
|
||||
/// one key from many threads: the count of allowed requests must be exactly the limit and nothing
|
||||
/// may throw (a List mutated concurrently throws InvalidOperationException or corrupts Count).
|
||||
/// </summary>
|
||||
public class RateLimitServiceConcurrencyTests
|
||||
{
|
||||
private const int Threads = 16;
|
||||
private const int RequestsPerThread = 500;
|
||||
|
||||
[Fact]
|
||||
public void CheckRateLimit_ManyThreadsSameKey_AllowsExactlyTheLimit()
|
||||
{
|
||||
var sut = new RateLimitService(new MemoryCache(new MemoryCacheOptions()), NullLogger<RateLimitService>.Instance);
|
||||
const int limit = 1000;
|
||||
var allowed = 0;
|
||||
var errors = new ConcurrentBag<Exception>();
|
||||
|
||||
RunInParallel(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (sut.CheckRateLimit("thumb-shared", limit).IsAllowed)
|
||||
Interlocked.Increment(ref allowed);
|
||||
}
|
||||
catch (Exception ex) { errors.Add(ex); }
|
||||
});
|
||||
|
||||
Assert.Empty(errors);
|
||||
Assert.Equal(limit, allowed);
|
||||
Assert.False(sut.CheckRateLimit("thumb-shared", limit).IsAllowed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DistressCheck_ManyThreadsSameIp_AllowsExactlyTheIpLimit()
|
||||
{
|
||||
var sut = new DistressRateLimitService(new MemoryCache(new MemoryCacheOptions()), NullLogger<DistressRateLimitService>.Instance);
|
||||
var allowed = 0;
|
||||
var errors = new ConcurrentBag<Exception>();
|
||||
|
||||
RunInParallel(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (sut.Check("10.0.0.9", "a1b2c3d4-e5f6-7890-abcd-ef1234567890").IsAllowed)
|
||||
Interlocked.Increment(ref allowed);
|
||||
}
|
||||
catch (Exception ex) { errors.Add(ex); }
|
||||
});
|
||||
|
||||
Assert.Empty(errors);
|
||||
Assert.Equal(5, allowed); // MaxPerIp
|
||||
}
|
||||
|
||||
private static void RunInParallel(Action body)
|
||||
{
|
||||
using var gate = new ManualResetEventSlim(false);
|
||||
var tasks = Enumerable.Range(0, Threads).Select(_ => Task.Run(() =>
|
||||
{
|
||||
gate.Wait();
|
||||
for (var i = 0; i < RequestsPerThread; i++) body();
|
||||
})).ToArray();
|
||||
gate.Set();
|
||||
Task.WaitAll(tasks);
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,9 @@ public class SessionIndexFieldManifestTests
|
||||
["ValidatedBy"] = "AutopilotV1",
|
||||
["FailureSnapshotJson"] = "{\"x\":1}",
|
||||
["AdminMarkedAction"] = "failed",
|
||||
["VerdictPath"] = "sweep:r5_incomplete",
|
||||
["PriorStatus"] = "Incomplete",
|
||||
["PriorVerdictPath"] = "maxlife:r6",
|
||||
["DurationSeconds"] = 360,
|
||||
["DiagnosticsBlobName"] = "diag-0001.zip",
|
||||
["DiagnosticsBlobDestination"] = "Hosted",
|
||||
|
||||
+3
-3
@@ -74,7 +74,7 @@ public class SessionStartedAtBackwardShiftGuardTests
|
||||
var harness = new Harness(SessionRow(), earliestStoredEvent: poisoned);
|
||||
|
||||
var ok = await harness.Sut.UpdateSessionStatusAsync(
|
||||
TenantId, SessionId, SessionStatus.Pending,
|
||||
TenantId, SessionId, SessionStatus.Pending, VerdictPaths.AgentWhiteGlovePending,
|
||||
earliestEventTimestamp: poisoned, latestEventTimestamp: lastEvent);
|
||||
|
||||
Assert.True(ok);
|
||||
@@ -93,7 +93,7 @@ public class SessionStartedAtBackwardShiftGuardTests
|
||||
var harness = new Harness(SessionRow(), earliestStoredEvent: earliest);
|
||||
|
||||
var ok = await harness.Sut.UpdateSessionStatusAsync(
|
||||
TenantId, SessionId, SessionStatus.Pending,
|
||||
TenantId, SessionId, SessionStatus.Pending, VerdictPaths.AgentWhiteGlovePending,
|
||||
latestEventTimestamp: lastEvent);
|
||||
|
||||
Assert.True(ok);
|
||||
@@ -132,7 +132,7 @@ public class SessionStartedAtBackwardShiftGuardTests
|
||||
});
|
||||
|
||||
var ok = await harness.Sut.UpdateSessionStatusAsync(
|
||||
TenantId, SessionId, SessionStatus.Pending,
|
||||
TenantId, SessionId, SessionStatus.Pending, VerdictPaths.AgentWhiteGlovePending,
|
||||
latestEventTimestamp: lastEvent);
|
||||
|
||||
Assert.True(ok);
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Data.Tables.Models;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Microsoft.ApplicationInsights.Channel;
|
||||
using Microsoft.ApplicationInsights.DataContracts;
|
||||
using Microsoft.ApplicationInsights.Extensibility;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests
|
||||
{
|
||||
public class StartupTelemetryServiceTests
|
||||
{
|
||||
private sealed class CapturingChannel : ITelemetryChannel
|
||||
{
|
||||
public List<ITelemetry> Items { get; } = new();
|
||||
public bool? DeveloperMode { get; set; }
|
||||
public string EndpointAddress { get; set; } = "";
|
||||
public void Send(ITelemetry item) => Items.Add(item);
|
||||
public void Flush() { }
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
private sealed class Lifetime : IHostApplicationLifetime
|
||||
{
|
||||
private readonly CancellationTokenSource _started = new();
|
||||
public CancellationToken ApplicationStarted => _started.Token;
|
||||
public CancellationToken ApplicationStopping => CancellationToken.None;
|
||||
public CancellationToken ApplicationStopped => CancellationToken.None;
|
||||
public void StopApplication() { }
|
||||
public void FireStarted() => _started.Cancel();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplicationStarted_EmitsStartupAndTableInitMetrics()
|
||||
{
|
||||
var channel = new CapturingChannel();
|
||||
var telemetry = new TelemetryClient(new TelemetryConfiguration { TelemetryChannel = channel, ConnectionString = "InstrumentationKey=00000000-0000-0000-0000-000000000000" });
|
||||
|
||||
// Table storage: unreadable sentinel → full pass → FullPassRan = true.
|
||||
var service = new Mock<TableServiceClient>();
|
||||
var admin = new Mock<TableClient>();
|
||||
service.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(admin.Object);
|
||||
admin.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(404, "TableNotFound"));
|
||||
admin.Setup(c => c.UpsertEntityAsync(It.IsAny<TableEntity>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Mock.Of<Response>());
|
||||
service.Setup(s => s.CreateTableIfNotExistsAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Response.FromValue(new TableItem("x"), Mock.Of<Response>()));
|
||||
var storage = new TableStorageService(service.Object, NullLogger<TableStorageService>.Instance);
|
||||
await storage.InitializeTablesAsync();
|
||||
|
||||
var lifetime = new Lifetime();
|
||||
var sut = new StartupTelemetryService(lifetime, telemetry, storage, new BackendBuildInfo(), NullLogger<StartupTelemetryService>.Instance);
|
||||
await sut.StartAsync(CancellationToken.None);
|
||||
Assert.Empty(channel.Items);
|
||||
|
||||
lifetime.FireStarted();
|
||||
|
||||
var metrics = channel.Items.ConvertAll(i => (MetricTelemetry)i);
|
||||
Assert.Equal(2, metrics.Count);
|
||||
var startup = metrics.Find(m => m.Name == StartupTelemetryService.StartupMetricName)!;
|
||||
var tableInit = metrics.Find(m => m.Name == StartupTelemetryService.TableInitMetricName)!;
|
||||
Assert.True(startup.Sum > 0);
|
||||
Assert.True(tableInit.Sum >= 0);
|
||||
Assert.Equal("True", startup.Properties["tableInitFullPass"]);
|
||||
Assert.False(string.IsNullOrEmpty(startup.Properties["version"]));
|
||||
Assert.True(startup.Sum >= tableInit.Sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Data.Tables.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// Startup table initialization is gated by a schema sentinel (hash over TableNames.All)
|
||||
/// stored in AdminConfiguration: a matching sentinel means zero CreateTableIfNotExists
|
||||
/// calls on a cold start; anything else runs the full idempotent pass and rewrites it.
|
||||
/// </summary>
|
||||
public class TableSchemaSentinelTests
|
||||
{
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<TableServiceClient> Service { get; } = new();
|
||||
public Mock<TableClient> AdminConfig { get; } = new();
|
||||
public TableEntity? UpsertedSentinel { get; private set; }
|
||||
public int CreateCalls;
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Service.Setup(s => s.GetTableClient(Constants.TableNames.AdminConfiguration)).Returns(AdminConfig.Object);
|
||||
Service.Setup(s => s.CreateTableIfNotExistsAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((_, _) =>
|
||||
{
|
||||
Interlocked.Increment(ref CreateCalls);
|
||||
return Task.FromResult(Response.FromValue(new TableItem("x"), Mock.Of<Response>()));
|
||||
});
|
||||
AdminConfig.Setup(c => c.UpsertEntityAsync(It.IsAny<TableEntity>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<ITableEntity, TableUpdateMode, CancellationToken>((e, _, _) => UpsertedSentinel = (TableEntity)e)
|
||||
.ReturnsAsync(Mock.Of<Response>());
|
||||
}
|
||||
|
||||
public void SentinelReturns(string? hash)
|
||||
{
|
||||
var response = new Mock<NullableResponse<TableEntity>>();
|
||||
response.SetupGet(r => r.HasValue).Returns(hash != null);
|
||||
if (hash != null)
|
||||
{
|
||||
response.SetupGet(r => r.Value).Returns(new TableEntity(
|
||||
TableStorageService.SchemaSentinelPartitionKey, TableStorageService.SchemaSentinelRowKey)
|
||||
{
|
||||
[TableStorageService.SchemaSentinelHashProperty] = hash
|
||||
});
|
||||
}
|
||||
AdminConfig.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(
|
||||
TableStorageService.SchemaSentinelPartitionKey, TableStorageService.SchemaSentinelRowKey,
|
||||
It.IsAny<System.Collections.Generic.IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(response.Object);
|
||||
}
|
||||
|
||||
public void SentinelThrows()
|
||||
{
|
||||
AdminConfig.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<System.Collections.Generic.IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(404, "TableNotFound"));
|
||||
}
|
||||
|
||||
public TableStorageService Build() => new(Service.Object, NullLogger<TableStorageService>.Instance);
|
||||
}
|
||||
|
||||
private static string CurrentHash => TableStorageService.ComputeTableSchemaHash(Constants.TableNames.All);
|
||||
|
||||
[Fact]
|
||||
public void Hash_IsOrderIndependent_AndChangesWhenTableAdded()
|
||||
{
|
||||
var a = TableStorageService.ComputeTableSchemaHash(new[] { "B", "A" });
|
||||
var b = TableStorageService.ComputeTableSchemaHash(new[] { "A", "B" });
|
||||
var c = TableStorageService.ComputeTableSchemaHash(new[] { "A", "B", "C" });
|
||||
|
||||
Assert.Equal(a, b);
|
||||
Assert.NotEqual(a, c);
|
||||
Assert.Equal(64, a.Length);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MatchingSentinel_SkipsTableCreation()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SentinelReturns(CurrentHash);
|
||||
|
||||
var fullPass = await h.Build().InitializeTablesAsync();
|
||||
|
||||
Assert.False(fullPass);
|
||||
Assert.Equal(0, h.CreateCalls);
|
||||
Assert.Null(h.UpsertedSentinel);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StaleSentinel_RunsFullPass_AndRewritesSentinel()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SentinelReturns("STALE");
|
||||
|
||||
var fullPass = await h.Build().InitializeTablesAsync();
|
||||
|
||||
Assert.True(fullPass);
|
||||
Assert.Equal(Constants.TableNames.All.Length, h.CreateCalls);
|
||||
Assert.NotNull(h.UpsertedSentinel);
|
||||
Assert.Equal(CurrentHash, h.UpsertedSentinel!.GetString(TableStorageService.SchemaSentinelHashProperty));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MissingSentinel_RunsFullPass()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SentinelReturns(null);
|
||||
|
||||
Assert.True(await h.Build().InitializeTablesAsync());
|
||||
Assert.Equal(Constants.TableNames.All.Length, h.CreateCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnreadableSentinel_FreshStorage_RunsFullPass()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SentinelThrows();
|
||||
|
||||
Assert.True(await h.Build().InitializeTablesAsync());
|
||||
Assert.Equal(Constants.TableNames.All.Length, h.CreateCalls);
|
||||
Assert.NotNull(h.UpsertedSentinel);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FailedTableCreate_DoesNotWriteSentinel_AndRetriesNextCall()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SentinelReturns(null);
|
||||
var failing = Constants.TableNames.All.First();
|
||||
h.Service.Setup(s => s.CreateTableIfNotExistsAsync(failing, It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(500, "boom"));
|
||||
var svc = h.Build();
|
||||
|
||||
Assert.True(await svc.InitializeTablesAsync());
|
||||
Assert.Null(h.UpsertedSentinel);
|
||||
|
||||
// Not marked initialized → a second call runs the pass again instead of short-circuiting.
|
||||
var before = h.CreateCalls;
|
||||
Assert.True(await svc.InitializeTablesAsync());
|
||||
Assert.True(h.CreateCalls > before);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecondCallAfterSuccess_IsNoOp()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SentinelReturns(null);
|
||||
var svc = h.Build();
|
||||
|
||||
await svc.InitializeTablesAsync();
|
||||
var calls = h.CreateCalls;
|
||||
Assert.False(await svc.InitializeTablesAsync());
|
||||
Assert.Equal(calls, h.CreateCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BackfillClaim_WinnerTrue_LoserFalse()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AdminConfig.SetupSequence(c => c.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Mock.Of<Response>())
|
||||
.ThrowsAsync(new RequestFailedException(409, "EntityAlreadyExists"));
|
||||
var svc = h.Build();
|
||||
|
||||
Assert.True(await svc.TryClaimSessionIndexBackfillAsync());
|
||||
Assert.False(await svc.TryClaimSessionIndexBackfillAsync());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Verdict-calibration drift radar (docs/backend/verdict-calibration.md): the three finding
|
||||
/// kinds, the small-n / lift / Wilson gates, re-arm, and the tracker round-trip.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationRadarTests
|
||||
{
|
||||
private static readonly DateTime Target = new(2026, 8, 22, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>35 daily rows ending on Target; per-day buckets chosen by window/baseline membership.</summary>
|
||||
private static List<VerdictCalibrationDailyAggregate> Horizon(
|
||||
Func<bool, (string Path, string Status, int Count)[]> bucketsFor, int sessionsPerDay = 20)
|
||||
{
|
||||
var rows = new List<VerdictCalibrationDailyAggregate>();
|
||||
for (var i = 34; i >= 0; i--)
|
||||
{
|
||||
var inWindow = i < 7;
|
||||
var date = Target.AddDays(-i);
|
||||
rows.Add(new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = "t", Date = date.ToString("yyyy-MM-dd"), SessionCount = sessionsPerDay, TerminalSessionCount = sessionsPerDay,
|
||||
Buckets = bucketsFor(inWindow).Select(b => new VerdictCalibrationBucket { VerdictPath = b.Path, Status = b.Status, Count = b.Count }).ToList(),
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Share_regression_fires_on_a_doubled_separated_share()
|
||||
{
|
||||
// baseline: 1/20 per day (5%), window: 4/20 per day (20%) → lift 4, clearly separated
|
||||
var rows = Horizon(w => new[] { ("sweep:r5_incomplete", "Incomplete", w ? 4 : 1), ("agent:complete", "Succeeded", w ? 16 : 19) });
|
||||
var findings = VerdictCalibrationRadar.Evaluate(rows, Target);
|
||||
|
||||
var f = Assert.Single(findings, x => x.Kind == VerdictCalibrationAlertKinds.ShareRegression);
|
||||
Assert.Equal("sweep:r5_incomplete", f.VerdictPath);
|
||||
Assert.Equal("Incomplete", f.Status);
|
||||
Assert.Equal(28, f.WindowHitCount);
|
||||
Assert.Equal(140, f.WindowSessionCount);
|
||||
Assert.Equal(28, f.BaselineHitCount);
|
||||
Assert.Equal(560, f.BaselineSessionCount);
|
||||
Assert.Equal(20.0, f.WindowRatePct);
|
||||
Assert.Equal(5.0, f.BaselineRatePct);
|
||||
Assert.Equal(4.0, f.Lift);
|
||||
Assert.Equal("2026-08-16", f.WindowStartDate);
|
||||
Assert.Equal("2026-08-22", f.WindowEndDate);
|
||||
// sweep:* is also the silence group → silence share doubled as well
|
||||
Assert.Contains(findings, x => x.Kind == VerdictCalibrationAlertKinds.SilenceShareRegression && x.VerdictPath == VerdictCalibrationRadar.SilenceGroupPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Share_regression_is_suppressed_below_lift_two_or_without_separation()
|
||||
{
|
||||
// 5% → 8%: lift 1.6 — under the gate
|
||||
var mild = Horizon(w => new[] { ("agent:failed", "Failed", w ? 2 : 1), ("agent:complete", "Succeeded", w ? 18 : 19) }, sessionsPerDay: 20);
|
||||
Assert.DoesNotContain(VerdictCalibrationRadar.Evaluate(mild, Target), f => f.Kind == VerdictCalibrationAlertKinds.ShareRegression);
|
||||
|
||||
// tiny n: 1/2 per day in window vs 0/2 baseline — lift infinite but Wilson intervals overlap / hits < 5
|
||||
var tiny = Horizon(w => new[] { ("agent:failed", "Failed", w ? 1 : 0), ("agent:complete", "Succeeded", w ? 1 : 2) }, sessionsPerDay: 2);
|
||||
Assert.Empty(VerdictCalibrationRadar.Evaluate(tiny, Target));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Agent_and_registration_paths_never_share_alert_but_still_feed_the_silence_group()
|
||||
{
|
||||
// Workflow mix, not classifier drift: a WhiteGlove rollout week doubles the pending share.
|
||||
var wg = Horizon(w => new[] { ("agent:whiteglove_pending", "Pending", w ? 6 : 1), ("agent:complete", "Succeeded", w ? 14 : 19) });
|
||||
Assert.DoesNotContain(VerdictCalibrationRadar.Evaluate(wg, Target), f => f.Kind == VerdictCalibrationAlertKinds.ShareRegression);
|
||||
var reg = Horizon(w => new[] { ("register:superseded", "Incomplete", w ? 6 : 1), ("agent:complete", "Succeeded", w ? 14 : 19) });
|
||||
Assert.DoesNotContain(VerdictCalibrationRadar.Evaluate(reg, Target), f => f.Kind == VerdictCalibrationAlertKinds.ShareRegression);
|
||||
// Backend-decided paths stay eligible.
|
||||
Assert.True(VerdictCalibrationRadar.IsShareRegressionEligible("sweep:r6"));
|
||||
Assert.True(VerdictCalibrationRadar.IsShareRegressionEligible("rule:ANALYZE-ESP-001"));
|
||||
Assert.True(VerdictCalibrationRadar.IsShareRegressionEligible("manual:failed"));
|
||||
Assert.False(VerdictCalibrationRadar.IsShareRegressionEligible("agent:complete_soft"));
|
||||
|
||||
// An episode opened by the earlier, broader gate re-arms on the next pass regardless of numbers.
|
||||
var stale = new VerdictCalibrationAlert { Kind = VerdictCalibrationAlertKinds.ShareRegression, VerdictPath = "agent:whiteglove_pending", Status = "Pending" };
|
||||
Assert.True(VerdictCalibrationRadar.ShouldReArm(stale, wg, Target));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Derived_legacy_paths_never_alert()
|
||||
{
|
||||
var rows = Horizon(w => new[] { ("legacy:r6", "Incomplete", w ? 10 : 1), ("agent:complete", "Succeeded", w ? 10 : 19) });
|
||||
Assert.DoesNotContain(VerdictCalibrationRadar.Evaluate(rows, Target), f => f.Kind == VerdictCalibrationAlertKinds.ShareRegression);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evidence_gap_fires_on_r6_share_and_ignores_non_rule_sweep_paths()
|
||||
{
|
||||
// classifier verdicts in window: r6 5/day + r5 15/day = 140; r6 share 25% → fires. sweep:stalled is not a rule verdict.
|
||||
var rows = Horizon(w => new[]
|
||||
{
|
||||
("sweep:r6", "Incomplete", 5), ("sweep:r5_incomplete", "Incomplete", 15), ("sweep:stalled", "Stalled", 30),
|
||||
}, sessionsPerDay: 50);
|
||||
var f = Assert.Single(VerdictCalibrationRadar.Evaluate(rows, Target), x => x.Kind == VerdictCalibrationAlertKinds.EvidenceGap);
|
||||
Assert.Equal(VerdictCalibrationRadar.EvidenceGapPath, f.VerdictPath);
|
||||
Assert.Equal(35, f.WindowHitCount);
|
||||
Assert.Equal(140, f.WindowSessionCount);
|
||||
Assert.Equal(25.0, f.WindowRatePct);
|
||||
Assert.Null(f.Lift);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evidence_gap_needs_twenty_classifier_verdicts()
|
||||
{
|
||||
var rows = Horizon(w => new[] { ("sweep:r6", "Incomplete", w ? 2 : 0), ("agent:complete", "Succeeded", 18) });
|
||||
Assert.DoesNotContain(VerdictCalibrationRadar.Evaluate(rows, Target), f => f.Kind == VerdictCalibrationAlertKinds.EvidenceGap);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReArm_when_share_falls_under_one_point_five_or_path_stops()
|
||||
{
|
||||
var alert = new VerdictCalibrationAlert { Kind = VerdictCalibrationAlertKinds.ShareRegression, VerdictPath = "sweep:r6", Status = "Incomplete" };
|
||||
|
||||
var stillHigh = Horizon(w => new[] { ("sweep:r6", "Incomplete", w ? 4 : 1), ("agent:complete", "Succeeded", w ? 16 : 19) });
|
||||
Assert.False(VerdictCalibrationRadar.ShouldReArm(alert, stillHigh, Target));
|
||||
|
||||
var elevatedButUnderGate = Horizon(w => new[] { ("sweep:r6", "Incomplete", w ? 2 : 1), ("agent:complete", "Succeeded", w ? 18 : 19) }, sessionsPerDay: 20);
|
||||
// 10% vs 5% = 2.0× — still at/above 1.5× → keep
|
||||
Assert.False(VerdictCalibrationRadar.ShouldReArm(alert, elevatedButUnderGate, Target));
|
||||
|
||||
var back = Horizon(w => new[] { ("sweep:r6", "Incomplete", 1), ("agent:complete", "Succeeded", 19) });
|
||||
Assert.True(VerdictCalibrationRadar.ShouldReArm(alert, back, Target));
|
||||
|
||||
var gone = Horizon(w => new[] { ("sweep:r6", "Incomplete", w ? 0 : 1), ("agent:complete", "Succeeded", w ? 20 : 19) });
|
||||
Assert.True(VerdictCalibrationRadar.ShouldReArm(alert, gone, Target));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evidence_gap_rearms_under_fifteen_percent()
|
||||
{
|
||||
var alert = new VerdictCalibrationAlert { Kind = VerdictCalibrationAlertKinds.EvidenceGap, VerdictPath = "r6", Status = "*" };
|
||||
var still = Horizon(w => new[] { ("sweep:r6", "Incomplete", 4), ("sweep:r5_incomplete", "Incomplete", 16) });
|
||||
Assert.False(VerdictCalibrationRadar.ShouldReArm(alert, still, Target)); // 20%
|
||||
var between = Horizon(w => new[] { ("sweep:r6", "Incomplete", 17), ("sweep:r5_incomplete", "Incomplete", 83) }, sessionsPerDay: 100);
|
||||
Assert.False(VerdictCalibrationRadar.ShouldReArm(alert, between, Target)); // 17%
|
||||
var low = Horizon(w => new[] { ("sweep:r6", "Incomplete", 1), ("sweep:r5_incomplete", "Incomplete", 19) });
|
||||
Assert.True(VerdictCalibrationRadar.ShouldReArm(alert, low, Target)); // 5%
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CurrentSums_follow_the_alert_kind()
|
||||
{
|
||||
var rows = Horizon(w => new[] { ("sweep:r6", "Incomplete", 2), ("maxlife:r3", "Incomplete", 1), ("agent:complete", "Succeeded", 17) });
|
||||
var share = VerdictCalibrationRadar.CurrentSums(new VerdictCalibrationAlert { Kind = VerdictCalibrationAlertKinds.ShareRegression, VerdictPath = "sweep:r6", Status = "Incomplete" }, rows, Target);
|
||||
Assert.Equal((14, 140, 56, 560), share);
|
||||
var silence = VerdictCalibrationRadar.CurrentSums(new VerdictCalibrationAlert { Kind = VerdictCalibrationAlertKinds.SilenceShareRegression }, rows, Target);
|
||||
Assert.Equal((21, 140, 84, 560), silence);
|
||||
var gap = VerdictCalibrationRadar.CurrentSums(new VerdictCalibrationAlert { Kind = VerdictCalibrationAlertKinds.EvidenceGap }, rows, Target);
|
||||
Assert.Equal((14, 21, 56, 84), gap);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Rows_outside_the_horizon_are_ignored()
|
||||
{
|
||||
var rows = Horizon(w => new[] { ("agent:complete", "Succeeded", 20) });
|
||||
rows.Add(new VerdictCalibrationDailyAggregate { TenantId = "t", Date = "2026-08-23", SessionCount = 1000, Buckets = { new VerdictCalibrationBucket { VerdictPath = "sweep:r6", Status = "Incomplete", Count = 1000 } } });
|
||||
rows.Add(new VerdictCalibrationDailyAggregate { TenantId = "t", Date = "2026-07-01", SessionCount = 1000, Buckets = { new VerdictCalibrationBucket { VerdictPath = "sweep:r6", Status = "Incomplete", Count = 1000 } } });
|
||||
Assert.Empty(VerdictCalibrationRadar.Evaluate(rows, Target));
|
||||
}
|
||||
|
||||
// ---- tracker round-trip ----
|
||||
|
||||
[Fact]
|
||||
public void Alert_round_trips_through_the_tracker_entity()
|
||||
{
|
||||
var alert = new VerdictCalibrationAlert
|
||||
{
|
||||
TenantId = "11111111-1111-1111-1111-111111111111", Kind = VerdictCalibrationAlertKinds.ShareRegression,
|
||||
VerdictPath = "rule:ANALYZE-ESP-001", Status = "Failed",
|
||||
WindowHitCount = 7, WindowSessionCount = 40, BaselineHitCount = 3, BaselineSessionCount = 160,
|
||||
WindowRatePct = 17.5, BaselineRatePct = 1.9, Lift = 9.3, WindowStartDate = "2026-08-16", WindowEndDate = "2026-08-22",
|
||||
Dimension = new RuleRegressionDimension { Dimension = "osBuild", Value = "26200.1234", HitCount = 6, HitSharePct = 85.7, AllSharePct = 20.0, Lift = 4.3 },
|
||||
FirstNotifiedAt = new DateTime(2026, 8, 20, 6, 0, 0, DateTimeKind.Utc), LastEvaluatedAt = new DateTime(2026, 8, 23, 6, 0, 0, DateTimeKind.Utc),
|
||||
};
|
||||
var entity = TableHardwareRejectionNotificationTracker.BuildVerdictCalibrationEntity(alert.TenantId, alert);
|
||||
Assert.Equal("verdictcalibration|share_regression|rule:analyze-esp-001|failed", entity.RowKey);
|
||||
Assert.Equal(alert.TenantId, entity.PartitionKey);
|
||||
|
||||
var back = TableHardwareRejectionNotificationTracker.MapToVerdictCalibrationAlert(entity);
|
||||
Assert.Equal(JsonSerializer.Serialize(alert), JsonSerializer.Serialize(back));
|
||||
|
||||
// Tri-states: no lift / no dimension → absent columns → null on read, never 0 / a guessed dimension.
|
||||
var bare = new VerdictCalibrationAlert
|
||||
{
|
||||
TenantId = alert.TenantId, Kind = VerdictCalibrationAlertKinds.EvidenceGap, VerdictPath = "r6", Status = "*",
|
||||
FirstNotifiedAt = alert.FirstNotifiedAt, LastEvaluatedAt = alert.LastEvaluatedAt,
|
||||
};
|
||||
var bareBack = TableHardwareRejectionNotificationTracker.MapToVerdictCalibrationAlert(
|
||||
TableHardwareRejectionNotificationTracker.BuildVerdictCalibrationEntity(bare.TenantId, bare));
|
||||
Assert.Null(bareBack.Lift);
|
||||
Assert.Null(bareBack.Dimension);
|
||||
Assert.Equal("verdictcalibration|evidence_gap|r6|*", TableHardwareRejectionNotificationTracker.BuildVerdictCalibrationRowKey(bare.Kind, bare.VerdictPath, bare.Status));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Verdict calibration aggregate (docs/backend/verdict-calibration.md): the pure bucketing core,
|
||||
/// the persistence round-trip and the matrix arithmetic of the read endpoint.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationTests
|
||||
{
|
||||
private const string TenantA = "11111111-1111-1111-1111-111111111111";
|
||||
private const string TenantB = "22222222-2222-2222-2222-222222222222";
|
||||
private static readonly DateTime Now = new(2026, 8, 23, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static SessionSummary Session(string tenant, string id, SessionStatus status, DateTime started,
|
||||
string? path = null, string? priorPath = null, string? priorStatus = null, string? adminMarked = null,
|
||||
DateTime? completed = null, string serial = "SN-1", string? failureReason = null, string? reconcileReason = null) => new()
|
||||
{
|
||||
TenantId = tenant, SessionId = id, SerialNumber = serial, Status = status, StartedAt = started,
|
||||
CompletedAt = completed, VerdictPath = path, PriorVerdictPath = priorPath, PriorStatus = priorStatus,
|
||||
AdminMarkedAction = adminMarked, FailureReason = failureReason ?? string.Empty,
|
||||
ReconcileReason = reconcileReason ?? string.Empty, FailureSource = string.Empty,
|
||||
};
|
||||
|
||||
private static DeviceHistory History(string tenant, string serial, params (string Id, DateTime Started, DateTime? Completed, string Status)[] refs) => new()
|
||||
{
|
||||
TenantId = tenant, SerialKey = serial.ToLowerInvariant(), SerialNumber = serial,
|
||||
Chain = refs.Select(r => new DeviceSessionRef { SessionId = r.Id, StartedAt = r.Started, CompletedAt = r.Completed, Status = r.Status }).ToList(),
|
||||
};
|
||||
|
||||
private static VerdictCalibrationBucket Bucket(VerdictCalibrationDailyAggregate row, string path, string status)
|
||||
=> Assert.Single(row.Buckets, b => b.VerdictPath == path && b.Status == status);
|
||||
|
||||
// ---- BuildVerdictCalibrationAggregates ----
|
||||
|
||||
[Fact]
|
||||
public void Buckets_by_tenant_date_path_and_status_with_global_mirror()
|
||||
{
|
||||
var d1 = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
Session(TenantA, "a1", SessionStatus.Succeeded, d1, path: "agent:complete", completed: d1.AddHours(1)),
|
||||
Session(TenantA, "a2", SessionStatus.Succeeded, d1.AddHours(2), path: "agent:complete", completed: d1.AddHours(3)),
|
||||
Session(TenantA, "a3", SessionStatus.Incomplete, d1, path: "sweep:r6", completed: d1.AddHours(6)),
|
||||
Session(TenantA, "a4", SessionStatus.Stalled, d1.AddDays(1), path: "sweep:stalled"),
|
||||
Session(TenantB, "b1", SessionStatus.Failed, d1, path: "agent:failed", completed: d1.AddHours(1)),
|
||||
};
|
||||
var rows = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary<string, List<DeviceHistory>>(), Now);
|
||||
|
||||
var a10 = Assert.Single(rows, r => r.TenantId == TenantA && r.Date == "2026-08-10");
|
||||
Assert.Equal(3, a10.SessionCount);
|
||||
Assert.Equal(3, a10.TerminalSessionCount);
|
||||
Assert.Equal(2, Bucket(a10, "agent:complete", "Succeeded").Count);
|
||||
Assert.Equal(1, Bucket(a10, "sweep:r6", "Incomplete").Count);
|
||||
|
||||
var a11 = Assert.Single(rows, r => r.TenantId == TenantA && r.Date == "2026-08-11");
|
||||
Assert.Equal(1, a11.SessionCount);
|
||||
Assert.Equal(0, a11.TerminalSessionCount); // Stalled is not terminal
|
||||
Assert.Equal(1, Bucket(a11, "sweep:stalled", "Stalled").Count);
|
||||
|
||||
var g10 = Assert.Single(rows, r => r.TenantId == "global" && r.Date == "2026-08-10");
|
||||
Assert.Equal(4, g10.SessionCount);
|
||||
Assert.Equal(1, Bucket(g10, "agent:failed", "Failed").Count);
|
||||
Assert.Equal(VerdictCalibrationDailyAggregate.CurrentVersion, g10.Version);
|
||||
Assert.Equal(Now, g10.ComputedAt);
|
||||
|
||||
// Deterministic ordering: rows by tenant then date, buckets by path then status.
|
||||
Assert.Equal(rows.Select(r => (r.TenantId, r.Date)).OrderBy(k => k.TenantId, StringComparer.Ordinal).ThenBy(k => k.Date, StringComparer.Ordinal), rows.Select(r => (r.TenantId, r.Date)));
|
||||
Assert.Equal(a10.Buckets.Select(b => b.VerdictPath).OrderBy(p => p, StringComparer.Ordinal), a10.Buckets.Select(b => b.VerdictPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unstamped_rows_are_derived_and_counted_as_derived()
|
||||
{
|
||||
var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
Session(TenantA, "a1", SessionStatus.Succeeded, d, path: null, completed: d.AddHours(1)), // legacy agent completion
|
||||
Session(TenantA, "a2", SessionStatus.Incomplete, d, path: null, completed: d.AddHours(5),
|
||||
failureReason: "No Device Setup completion or explicit failure signal observed before timeout"),
|
||||
Session(TenantA, "a3", SessionStatus.Succeeded, d, path: "agent:complete", completed: d.AddHours(1)),
|
||||
};
|
||||
var row = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary<string, List<DeviceHistory>>(), Now)
|
||||
.Single(r => r.TenantId == TenantA);
|
||||
|
||||
var complete = Bucket(row, "agent:complete", "Succeeded");
|
||||
Assert.Equal(2, complete.Count);
|
||||
Assert.Equal(1, complete.DerivedCount);
|
||||
var r6 = Bucket(row, "legacy:r6", "Incomplete");
|
||||
Assert.Equal(1, r6.Count);
|
||||
Assert.Equal(1, r6.DerivedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Overrides_are_attributed_to_the_prior_path()
|
||||
{
|
||||
var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// sweep said Incomplete, agent later completed → late completion against sweep:r5_incomplete
|
||||
Session(TenantA, "a1", SessionStatus.Succeeded, d, path: "agent:complete", priorPath: "sweep:r5_incomplete", priorStatus: "Incomplete", completed: d.AddHours(1)),
|
||||
// admin flipped an agent success to Failed
|
||||
Session(TenantA, "a2", SessionStatus.Failed, d, path: "manual:failed", priorPath: "agent:complete", priorStatus: "Succeeded", adminMarked: "Failed", completed: d.AddHours(1)),
|
||||
// retro reclassification Failed → Incomplete
|
||||
Session(TenantA, "a3", SessionStatus.Incomplete, d, path: "retro:r6", priorPath: "agent:failed", priorStatus: "Failed", completed: d.AddHours(1)),
|
||||
// admin mark to Succeeded counts as admin, not late completion
|
||||
Session(TenantA, "a4", SessionStatus.Succeeded, d, path: "manual:succeeded", priorPath: "sweep:r6", priorStatus: "Incomplete", adminMarked: "Succeeded", completed: d.AddHours(1)),
|
||||
};
|
||||
var row = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, new Dictionary<string, List<DeviceHistory>>(), Now)
|
||||
.Single(r => r.TenantId == TenantA);
|
||||
|
||||
var priorSweep = Bucket(row, "sweep:r5_incomplete", "Incomplete");
|
||||
Assert.Equal(0, priorSweep.Count); // nobody is on it any more
|
||||
Assert.Equal(1, priorSweep.OverriddenByLateCompletion);
|
||||
|
||||
var priorAgent = Bucket(row, "agent:complete", "Succeeded");
|
||||
Assert.Equal(1, priorAgent.Count); // a1 is on it now
|
||||
Assert.Equal(1, priorAgent.OverriddenByAdmin); // a2 used to be
|
||||
|
||||
Assert.Equal(1, Bucket(row, "agent:failed", "Failed").OverriddenOther);
|
||||
Assert.Equal(1, Bucket(row, "sweep:r6", "Incomplete").OverriddenByAdmin);
|
||||
Assert.Equal(0, Bucket(row, "manual:succeeded", "Succeeded").OverriddenByAdmin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReEnrollment_proxy_counts_only_eligible_terminal_sessions()
|
||||
{
|
||||
var old = Now.AddDays(-20);
|
||||
var young = Now.AddDays(-2);
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// old, re-enrolled 3 days later → eligible + re-enrolled
|
||||
Session(TenantA, "a1", SessionStatus.Incomplete, old, path: "sweep:r6", completed: old.AddHours(5), serial: "SN-A"),
|
||||
// old, next attempt 10 days later → eligible, not re-enrolled
|
||||
Session(TenantA, "a2", SessionStatus.Failed, old, path: "agent:failed", completed: old.AddHours(1), serial: "SN-B"),
|
||||
// old, no next attempt → eligible, not re-enrolled
|
||||
Session(TenantA, "a3", SessionStatus.Succeeded, old, path: "agent:complete", completed: old.AddHours(1), serial: "SN-C"),
|
||||
// too young to judge → not eligible even though the chain shows a follow-up
|
||||
Session(TenantA, "a4", SessionStatus.Failed, young, path: "agent:failed", completed: young.AddHours(1), serial: "SN-D"),
|
||||
// non-terminal → never eligible
|
||||
Session(TenantA, "a5", SessionStatus.AwaitingUser, old, path: "sweep:r5_awaiting", serial: "SN-E"),
|
||||
};
|
||||
var histories = new Dictionary<string, List<DeviceHistory>>
|
||||
{
|
||||
[TenantA] = new()
|
||||
{
|
||||
History(TenantA, "SN-A", ("a1", old, old.AddHours(5), "Incomplete"), ("x1", old.AddDays(3), old.AddDays(3).AddHours(1), "Succeeded")),
|
||||
History(TenantA, "SN-B", ("a2", old, old.AddHours(1), "Failed"), ("x2", old.AddDays(10), null, "Succeeded")),
|
||||
History(TenantA, "SN-C", ("a3", old, old.AddHours(1), "Succeeded")),
|
||||
History(TenantA, "SN-D", ("a4", young, young.AddHours(1), "Failed"), ("x4", young.AddHours(2), null, "Succeeded")),
|
||||
},
|
||||
};
|
||||
var rows = MaintenanceService.BuildVerdictCalibrationAggregates(sessions, histories, Now);
|
||||
|
||||
var oldRow = rows.Single(r => r.TenantId == TenantA && r.Date == old.ToString("yyyy-MM-dd"));
|
||||
var r6 = Bucket(oldRow, "sweep:r6", "Incomplete");
|
||||
Assert.Equal((1, 1), (r6.Eligible7d, r6.ReEnrolled7d));
|
||||
var failed = Bucket(oldRow, "agent:failed", "Failed");
|
||||
Assert.Equal((1, 0), (failed.Eligible7d, failed.ReEnrolled7d));
|
||||
var complete = Bucket(oldRow, "agent:complete", "Succeeded");
|
||||
Assert.Equal((1, 0), (complete.Eligible7d, complete.ReEnrolled7d));
|
||||
var awaiting = Bucket(oldRow, "sweep:r5_awaiting", "AwaitingUser");
|
||||
Assert.Equal((0, 0), (awaiting.Eligible7d, awaiting.ReEnrolled7d));
|
||||
|
||||
var youngRow = rows.Single(r => r.TenantId == TenantA && r.Date == young.ToString("yyyy-MM-dd"));
|
||||
var youngFailed = Bucket(youngRow, "agent:failed", "Failed");
|
||||
Assert.Equal((0, 0), (youngFailed.Eligible7d, youngFailed.ReEnrolled7d));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sessions_inside_a_deletion_cascade_are_skipped()
|
||||
{
|
||||
var d = new DateTime(2026, 8, 10, 9, 0, 0, DateTimeKind.Utc);
|
||||
var deleting = Session(TenantA, "a1", SessionStatus.Succeeded, d, path: "agent:complete");
|
||||
deleting.DeletionState = "Pending";
|
||||
var rows = MaintenanceService.BuildVerdictCalibrationAggregates(new[] { deleting }, new Dictionary<string, List<DeviceHistory>>(), Now);
|
||||
Assert.Empty(rows);
|
||||
}
|
||||
|
||||
// ---- Persistence round-trip ----
|
||||
|
||||
[Fact]
|
||||
public void Aggregate_round_trips_through_the_table_entity()
|
||||
{
|
||||
var aggregate = new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = TenantA, Date = "2026-08-10", Version = 1, SessionCount = 5, TerminalSessionCount = 4, ComputedAt = Now,
|
||||
Buckets =
|
||||
{
|
||||
new VerdictCalibrationBucket { VerdictPath = "agent:complete", Status = "Succeeded", Count = 3, DerivedCount = 1, Eligible7d = 3, ReEnrolled7d = 1, OverriddenByAdmin = 1, OverriddenByLateCompletion = 0, OverriddenOther = 2 },
|
||||
new VerdictCalibrationBucket { VerdictPath = "sweep:r6", Status = "Incomplete", Count = 1 },
|
||||
},
|
||||
};
|
||||
var entity = TableStorageService.BuildVerdictCalibrationAggregateEntity(aggregate);
|
||||
Assert.Equal(TenantA, entity.PartitionKey);
|
||||
Assert.Equal("2026-08-10", entity.RowKey);
|
||||
|
||||
var back = TableStorageService.MapToVerdictCalibrationAggregate(entity);
|
||||
Assert.Equal(JsonSerializer.Serialize(aggregate), JsonSerializer.Serialize(back));
|
||||
}
|
||||
|
||||
// ---- Response builder ----
|
||||
|
||||
private static VerdictCalibrationDailyAggregate Day(string tenant, DateTime date, int sessions, params (string Path, string Status, int Count)[] buckets) => new()
|
||||
{
|
||||
TenantId = tenant, Date = date.ToString("yyyy-MM-dd"), SessionCount = sessions, TerminalSessionCount = sessions, Version = 1, ComputedAt = Now,
|
||||
Buckets = buckets.Select(b => new VerdictCalibrationBucket { VerdictPath = b.Path, Status = b.Status, Count = b.Count, Eligible7d = b.Count, ReEnrolled7d = b.Count / 5 }).ToList(),
|
||||
};
|
||||
|
||||
// Same naming policy as the worker host (Program.cs: camelCase) so the assertions pin the wire shape.
|
||||
private static readonly JsonSerializerOptions Wire = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
|
||||
|
||||
private static JsonElement BuildJson(IReadOnlyList<VerdictCalibrationDailyAggregate> daily, DateTime today, int days)
|
||||
=> JsonSerializer.SerializeToElement(VerdictCalibrationResponse.Build(daily, "global", today, days), Wire);
|
||||
|
||||
[Fact]
|
||||
public void Response_sums_the_window_and_computes_share_rate_and_trend()
|
||||
{
|
||||
var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc);
|
||||
var daily = new List<VerdictCalibrationDailyAggregate>();
|
||||
// 35 days: baseline days carry 10 sessions (1 sweep:r6), the last 7 days carry 10 sessions (3 sweep:r6) → share 10% → 30%, lift 3.0
|
||||
for (var i = 34; i >= 0; i--)
|
||||
{
|
||||
var date = today.AddDays(-i);
|
||||
var r6 = i < 7 ? 3 : 1;
|
||||
daily.Add(Day("global", date, 10, ("agent:complete", "Succeeded", 10 - r6), ("sweep:r6", "Incomplete", r6)));
|
||||
}
|
||||
var json = BuildJson(daily, today, days: 30);
|
||||
|
||||
Assert.True(json.GetProperty("success").GetBoolean());
|
||||
Assert.Equal(30, json.GetProperty("windowDays").GetInt32());
|
||||
Assert.Equal(300, json.GetProperty("totals").GetProperty("sessions").GetInt32());
|
||||
Assert.Equal(30, json.GetProperty("totals").GetProperty("days").GetInt32());
|
||||
|
||||
var paths = json.GetProperty("paths").EnumerateArray().ToList();
|
||||
var r6Row = paths.Single(p => p.GetProperty("verdictPath").GetString() == "sweep:r6");
|
||||
Assert.Equal(7 * 3 + 23 * 1, r6Row.GetProperty("count").GetInt32());
|
||||
Assert.Equal(Math.Round(100.0 * 44 / 300, 1), r6Row.GetProperty("sharePct").GetDouble());
|
||||
Assert.Equal(30.0, r6Row.GetProperty("window7").GetProperty("sharePct").GetDouble());
|
||||
Assert.Equal(10.0, r6Row.GetProperty("baseline28").GetProperty("sharePct").GetDouble());
|
||||
Assert.Equal(3.0, r6Row.GetProperty("lift").GetDouble());
|
||||
// Eligible 44 ≥ 20 → rate stated; ReEnrolled7d = count/5 per day → 7*0 + 23*0 = 0 → 0%
|
||||
Assert.Equal(0.0, r6Row.GetProperty("reEnrollRatePct").GetDouble());
|
||||
|
||||
// Sorted by count desc → agent:complete first.
|
||||
Assert.Equal("agent:complete", paths[0].GetProperty("verdictPath").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Response_withholds_rate_below_twenty_eligible_and_lift_without_baseline()
|
||||
{
|
||||
var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc);
|
||||
var daily = new List<VerdictCalibrationDailyAggregate>
|
||||
{
|
||||
Day("global", today, 5, ("maxlife:r3", "Incomplete", 5)),
|
||||
};
|
||||
var json = BuildJson(daily, today, days: 7);
|
||||
var row = json.GetProperty("paths").EnumerateArray().Single();
|
||||
Assert.Equal(JsonValueKind.Null, row.GetProperty("reEnrollRatePct").ValueKind);
|
||||
Assert.Equal(JsonValueKind.Null, row.GetProperty("lift").ValueKind); // no baseline rows → never invented
|
||||
Assert.Equal(100.0, row.GetProperty("sharePct").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Response_keeps_a_zero_count_bucket_that_carries_overrides()
|
||||
{
|
||||
var today = new DateTime(2026, 8, 23, 0, 0, 0, DateTimeKind.Utc);
|
||||
var day = Day("global", today, 1, ("agent:complete", "Succeeded", 1));
|
||||
day.Buckets.Add(new VerdictCalibrationBucket { VerdictPath = "sweep:r5_incomplete", Status = "Incomplete", Count = 0, OverriddenByLateCompletion = 1 });
|
||||
var json = BuildJson(new[] { day }, today, days: 7);
|
||||
var rows = json.GetProperty("paths").EnumerateArray().ToList();
|
||||
var prior = rows.Single(p => p.GetProperty("verdictPath").GetString() == "sweep:r5_incomplete");
|
||||
Assert.Equal(0, prior.GetProperty("count").GetInt32());
|
||||
Assert.Equal(1, prior.GetProperty("overriddenByLateCompletion").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Response_is_empty_but_well_formed_before_the_first_sweep()
|
||||
{
|
||||
var json = BuildJson(Array.Empty<VerdictCalibrationDailyAggregate>(), new DateTime(2026, 8, 23), 30);
|
||||
Assert.True(json.GetProperty("success").GetBoolean());
|
||||
Assert.Empty(json.GetProperty("paths").EnumerateArray());
|
||||
Assert.Equal(0, json.GetProperty("totals").GetProperty("sessions").GetInt32());
|
||||
Assert.Equal(JsonValueKind.Null, json.GetProperty("computedAt").ValueKind);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, 30)]
|
||||
[InlineData("abc", 30)]
|
||||
[InlineData("0", 1)]
|
||||
[InlineData("500", 180)]
|
||||
[InlineData("14", 14)]
|
||||
public void ClampDays_bounds_the_window(string? raw, int expected)
|
||||
=> Assert.Equal(expected, VerdictCalibrationResponse.ClampDays(raw));
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Verdict calibration instrumentation (docs/backend/verdict-calibration.md): every status write
|
||||
/// declares its origin, overriding a prior verdict preserves it, the classifier names the rule
|
||||
/// behind each verdict, and pre-instrumentation rows derive an honest path from their literals.
|
||||
/// </summary>
|
||||
public class VerdictPathTests
|
||||
{
|
||||
private static readonly DateTime Start = new(2026, 8, 20, 8, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// ---- ComputePriorVerdict: the correction stream ----
|
||||
|
||||
[Theory]
|
||||
[InlineData("Succeeded", "agent:complete", "Failed", true)] // admin mark over an agent success
|
||||
[InlineData("Incomplete", "sweep:r5_incomplete", "Succeeded", true)] // late agent completion upgrades a sweep verdict
|
||||
[InlineData("AwaitingUser", "sweep:r5_awaiting", "Incomplete", true)] // grace expired
|
||||
[InlineData("Failed", "agent:failed", "Incomplete", true)] // retro reclassification
|
||||
[InlineData("Stalled", "sweep:stalled", "InProgress", false)] // stall heal is not a correction
|
||||
[InlineData("Stalled", "sweep:stalled", "Succeeded", false)]
|
||||
[InlineData("InProgress", "register:new", "Succeeded", false)]
|
||||
[InlineData("Pending", "agent:whiteglove_pending", "InProgress", false)]
|
||||
[InlineData("AwaitingUser", "sweep:wg_awaiting", "AwaitingUser", false)] // same-status refresh keeps the original prior
|
||||
public void Prior_is_preserved_only_when_a_verdict_is_overridden(string existing, string existingPath, string incoming, bool expectPrior)
|
||||
{
|
||||
var prior = TableStorageService.ComputePriorVerdict(existing, existingPath, Enum.Parse<SessionStatus>(incoming));
|
||||
if (expectPrior)
|
||||
{
|
||||
Assert.NotNull(prior);
|
||||
Assert.Equal(existing, prior!.Value.PriorStatus);
|
||||
Assert.Equal(existingPath, prior.Value.PriorVerdictPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Null(prior);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Prior_needs_a_stamped_path_on_the_existing_row()
|
||||
{
|
||||
// Pre-instrumentation rows carry no path — there is nothing trustworthy to preserve.
|
||||
Assert.Null(TableStorageService.ComputePriorVerdict("Incomplete", null, SessionStatus.Succeeded));
|
||||
Assert.Null(TableStorageService.ComputePriorVerdict("Incomplete", "", SessionStatus.Succeeded));
|
||||
Assert.Null(TableStorageService.ComputePriorVerdict(null, "sweep:r6", SessionStatus.Succeeded));
|
||||
}
|
||||
|
||||
// ---- VerdictPaths vocabulary ----
|
||||
|
||||
[Fact]
|
||||
public void Compose_and_Origin_round_trip()
|
||||
{
|
||||
var path = VerdictPaths.Compose(VerdictPaths.OriginSweep, ClassifierRules.R5DeviceSetupIncomplete);
|
||||
Assert.Equal("sweep:r5_incomplete", path);
|
||||
Assert.Equal("sweep", VerdictPaths.Origin(path));
|
||||
Assert.True(VerdictPaths.IsClassifierPath(path));
|
||||
Assert.True(VerdictPaths.IsClassifierPath("maxlife:r1"));
|
||||
Assert.True(VerdictPaths.IsClassifierPath("late:r4"));
|
||||
Assert.True(VerdictPaths.IsClassifierPath("retro:r6"));
|
||||
Assert.False(VerdictPaths.IsClassifierPath(VerdictPaths.AgentComplete));
|
||||
Assert.False(VerdictPaths.IsClassifierPath(VerdictPaths.SweepStalled));
|
||||
Assert.False(VerdictPaths.IsClassifierPath(VerdictPaths.RetroSuperseded));
|
||||
Assert.False(VerdictPaths.IsClassifierPath("legacy:r6"));
|
||||
Assert.False(VerdictPaths.IsClassifierPath(VerdictPaths.ManualFailed));
|
||||
Assert.Throws<ArgumentException>(() => VerdictPaths.Compose("", "r1"));
|
||||
Assert.Throws<ArgumentException>(() => VerdictPaths.Compose("sweep", " "));
|
||||
}
|
||||
|
||||
// ---- Classifier: every return names its rule ----
|
||||
|
||||
private static EnrollmentEvent Evt(string type, string? message = null) => new()
|
||||
{
|
||||
EventType = type, Timestamp = Start, Source = "test", Message = message ?? type,
|
||||
};
|
||||
|
||||
private static EnrollmentEvent Esp(string message) => Evt("esp_provisioning_status", message);
|
||||
private const string DeviceSetup44 = "ESP provisioning status: DeviceSetup — 4 of 4 subcategories completed";
|
||||
private const string AccountSetup55 = "ESP provisioning status: AccountSetup — 5 of 5 subcategories completed";
|
||||
private const string AccountSetup15 = "ESP provisioning status: AccountSetup — 1 of 5 subcategories completed";
|
||||
|
||||
private static string RuleOf(IReadOnlyList<EnrollmentEvent> events, double hours = 6, int grace = 72,
|
||||
bool preProvisioned = false, DateTime? resumedAt = null, bool selfDeploying = false)
|
||||
{
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var (_, _, rule) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, Start, Start.AddHours(hours), grace, isPreProvisioned: preProvisioned,
|
||||
resumedAt: resumedAt, isSelfDeployingProfile: selfDeploying);
|
||||
return rule;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classifier_returns_the_rule_id_behind_each_verdict()
|
||||
{
|
||||
Assert.Equal(ClassifierRules.R1ExplicitFailure, RuleOf(new[] { Evt("enrollment_failed") }));
|
||||
Assert.Equal(ClassifierRules.R1bWhiteGloveAwaiting, RuleOf(
|
||||
new[] { Esp(DeviceSetup44), Evt("whiteglove_complete") }, hours: 6, grace: 51, preProvisioned: true, resumedAt: Start.AddHours(1)));
|
||||
Assert.Equal(ClassifierRules.R1bWhiteGloveSucceeded, RuleOf(
|
||||
new[] { Esp(DeviceSetup44), Evt("whiteglove_complete") }, hours: 60, grace: 51, preProvisioned: true, resumedAt: Start.AddHours(1)));
|
||||
Assert.Equal(ClassifierRules.R1cSelfDeploying, RuleOf(new[] { Esp(DeviceSetup44) }, selfDeploying: true));
|
||||
Assert.Equal(ClassifierRules.R2AccountSetupComplete, RuleOf(new[] { Esp(DeviceSetup44), Esp(AccountSetup55) }));
|
||||
Assert.Equal(ClassifierRules.R3EmergencyBreak, RuleOf(new[] { Esp(DeviceSetup44), Evt("agent_emergency_break") }));
|
||||
Assert.Equal(ClassifierRules.R4DesktopHello, RuleOf(new[] { Esp(DeviceSetup44), Evt("desktop_arrived"), Evt("hello_provisioning_completed") }));
|
||||
Assert.Equal(ClassifierRules.R5DeviceSetupAwaiting, RuleOf(new[] { Esp(DeviceSetup44), Esp(AccountSetup15) }, hours: 6, grace: 72));
|
||||
Assert.Equal(ClassifierRules.R5DeviceSetupIncomplete, RuleOf(new[] { Esp(DeviceSetup44), Esp(AccountSetup15) }, hours: 80, grace: 72));
|
||||
Assert.Equal(ClassifierRules.R6Fallthrough, RuleOf(Array.Empty<EnrollmentEvent>()));
|
||||
}
|
||||
|
||||
// ---- Derivation for pre-instrumentation rows ----
|
||||
|
||||
private static SessionSummary Row(SessionStatus status, string? failureReason = null, string? reconcileReason = null,
|
||||
string? failureSource = null, string? adminMarked = null, bool soft = false, string? verdictPath = null) => new()
|
||||
{
|
||||
SessionId = "s", TenantId = "t", Status = status,
|
||||
FailureReason = failureReason ?? string.Empty, ReconcileReason = reconcileReason ?? string.Empty,
|
||||
FailureSource = failureSource ?? string.Empty, AdminMarkedAction = adminMarked, EspSoftFailure = soft,
|
||||
VerdictPath = verdictPath,
|
||||
};
|
||||
|
||||
private static string Classified(IReadOnlyList<EnrollmentEvent> events, double hours = 80, int grace = 72, bool selfDeploying = false)
|
||||
{
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var (_, reason, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, Start, Start.AddHours(hours), grace, isSelfDeployingProfile: selfDeploying);
|
||||
return reason;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stamped_rows_are_returned_verbatim()
|
||||
{
|
||||
var (path, derived) = VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete, failureReason: "anything", verdictPath: "sweep:r6"));
|
||||
Assert.Equal("sweep:r6", path);
|
||||
Assert.False(derived);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Derivation_uses_the_real_classifier_literals()
|
||||
{
|
||||
// Reasons come straight from the classifier so a wording change there surfaces here.
|
||||
Assert.Equal(("legacy:r6", true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete,
|
||||
failureReason: Classified(Array.Empty<EnrollmentEvent>()))));
|
||||
Assert.Equal(("legacy:r5_incomplete", true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete,
|
||||
failureReason: Classified(new[] { Esp(DeviceSetup44), Esp(AccountSetup15) }))));
|
||||
Assert.Equal(("legacy:r5_awaiting", true), VerdictPathDerivation.Derive(Row(SessionStatus.AwaitingUser,
|
||||
failureReason: Classified(new[] { Esp(DeviceSetup44), Esp(AccountSetup15) }, hours: 6))));
|
||||
Assert.Equal(("legacy:r3", true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete,
|
||||
failureReason: Classified(new[] { Esp(DeviceSetup44), Evt("agent_emergency_break") }))));
|
||||
Assert.Equal(("legacy:r2", true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded,
|
||||
reconcileReason: Classified(new[] { Esp(DeviceSetup44), Esp(AccountSetup55) }))));
|
||||
Assert.Equal(("legacy:r4", true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded,
|
||||
reconcileReason: Classified(new[] { Esp(DeviceSetup44), Evt("desktop_arrived"), Evt("hello_provisioning_completed") }))));
|
||||
Assert.Equal(("legacy:r1c", true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded,
|
||||
reconcileReason: Classified(new[] { Esp(DeviceSetup44) }, selfDeploying: true))));
|
||||
Assert.Equal(("legacy:r1", true), VerdictPathDerivation.Derive(Row(SessionStatus.Failed,
|
||||
failureReason: Classified(new[] { Evt("enrollment_failed") }))));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Derivation_recovers_the_origin_from_writer_suffixes()
|
||||
{
|
||||
var r6 = Classified(Array.Empty<EnrollmentEvent>());
|
||||
Assert.Equal(("maxlife:r6", true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete,
|
||||
failureReason: r6 + " Verdict triggered by the agent's max-lifetime watchdog shutdown.")));
|
||||
Assert.Equal(("retro:r6", true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete,
|
||||
failureReason: r6 + " Retro-reclassified from the legacy blanket timeout verdict.")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Derivation_attributes_unambiguous_non_classifier_paths()
|
||||
{
|
||||
Assert.Equal((VerdictPaths.ManualFailed, true), VerdictPathDerivation.Derive(Row(SessionStatus.Failed,
|
||||
failureReason: "Manually marked as failed by administrator", failureSource: "manual", adminMarked: "Failed")));
|
||||
Assert.Equal((VerdictPaths.ManualSucceeded, true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded, adminMarked: "Succeeded")));
|
||||
Assert.Equal(("rule:ANALYZE-ESP-001", true), VerdictPathDerivation.Derive(Row(SessionStatus.Failed,
|
||||
failureReason: "Rule: ESP timeout", failureSource: "rule:ANALYZE-ESP-001")));
|
||||
Assert.Equal((VerdictPaths.AgentComplete, true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded)));
|
||||
Assert.Equal((VerdictPaths.AgentComplete, true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded,
|
||||
reconcileReason: "Late completion report received — upgraded prior 'Incomplete' verdict")));
|
||||
Assert.Equal((VerdictPaths.AgentCompleteSoft, true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded, soft: true)));
|
||||
Assert.Equal((VerdictPaths.AgentFailed, true), VerdictPathDerivation.Derive(Row(SessionStatus.Failed, failureReason: "ESP timeout (0x800705B4)")));
|
||||
Assert.Equal((VerdictPaths.AgentEspFailureFallback, true), VerdictPathDerivation.Derive(Row(SessionStatus.Failed, failureReason: "ESP failure (backend fallback)")));
|
||||
Assert.Equal((VerdictPaths.SweepStalled, true), VerdictPathDerivation.Derive(Row(SessionStatus.Stalled, failureReason: "Agent silent for 130min (detected by maintenance sweep)")));
|
||||
Assert.Equal((VerdictPaths.AgentStallProbe, true), VerdictPathDerivation.Derive(Row(SessionStatus.Stalled, failureReason: "Agent reported stall after 60min without progress (stall_probe)")));
|
||||
Assert.Equal(("legacy:superseded", true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete, failureReason: "Superseded by session abc: re-registered")));
|
||||
Assert.Equal((VerdictPaths.AgentWhiteGlovePending, true), VerdictPathDerivation.Derive(Row(SessionStatus.Pending)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Derivation_never_invents_a_path()
|
||||
{
|
||||
Assert.Equal((VerdictPaths.LegacyUnknown, true), VerdictPathDerivation.Derive(Row(SessionStatus.Failed, failureReason: "Session timed out after 5 hours (started at 2026-01-01 UTC)")));
|
||||
Assert.Equal((VerdictPaths.LegacyUnknown, true), VerdictPathDerivation.Derive(Row(SessionStatus.Incomplete)));
|
||||
Assert.Equal((VerdictPaths.LegacyUnknown, true), VerdictPathDerivation.Derive(Row(SessionStatus.InProgress)));
|
||||
Assert.Equal((VerdictPaths.LegacyUnknown, true), VerdictPathDerivation.Derive(Row(SessionStatus.Succeeded, reconcileReason: "Some future wording")));
|
||||
}
|
||||
}
|
||||
+146
@@ -422,6 +422,152 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
};
|
||||
}
|
||||
|
||||
// ===== VERDICT CALIBRATION DRIFT (docs/backend/verdict-calibration.md) =====
|
||||
|
||||
internal const string VerdictCalibrationRowKeyPrefix = "verdictcalibration|";
|
||||
|
||||
internal static string BuildVerdictCalibrationRowKey(string kind, string verdictPath, string status)
|
||||
{
|
||||
// Paths are a closed vocabulary plus rule ids (ANALYZE-...), statuses are enum names and
|
||||
// the group marker "*" - none contain table-key-hostile characters, but sanitize anyway.
|
||||
var k = SanitizeTableKey((kind ?? string.Empty).Trim().ToLowerInvariant());
|
||||
var p = SanitizeTableKey((verdictPath ?? string.Empty).Trim().ToLowerInvariant());
|
||||
var s = SanitizeTableKey((status ?? string.Empty).Trim().ToLowerInvariant());
|
||||
return $"{VerdictCalibrationRowKeyPrefix}{k}|{p}|{s}";
|
||||
}
|
||||
|
||||
public async Task<bool> TryRegisterVerdictCalibrationAlertAsync(string tenantId, VerdictCalibrationAlert alert)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(alert?.Kind) || string.IsNullOrWhiteSpace(alert?.VerdictPath))
|
||||
return false;
|
||||
try
|
||||
{
|
||||
await _table.AddEntityAsync(BuildVerdictCalibrationEntity(tenantId, alert!));
|
||||
_logger.LogInformation(
|
||||
"VerdictCalibration tracker registered: tenant={TenantId} kind={Kind} path={Path}", tenantId, alert!.Kind, alert.VerdictPath);
|
||||
return true;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
return false; // episode already active - one ops event per episode
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VerdictCalibration tracker failed: tenant={TenantId} kind={Kind} path={Path}", tenantId, alert!.Kind, alert.VerdictPath);
|
||||
return false; // fail closed - never double-fire
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RefreshVerdictCalibrationAlertAsync(string tenantId, VerdictCalibrationAlert alert)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(alert?.Kind) || string.IsNullOrWhiteSpace(alert?.VerdictPath))
|
||||
return;
|
||||
try
|
||||
{
|
||||
await _table.UpsertEntityAsync(BuildVerdictCalibrationEntity(tenantId, alert!), TableUpdateMode.Replace);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VerdictCalibration tracker refresh failed: tenant={TenantId} kind={Kind} path={Path}", tenantId, alert!.Kind, alert.VerdictPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteVerdictCalibrationAlertAsync(string tenantId, string kind, string verdictPath, string status)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenantId) || string.IsNullOrWhiteSpace(kind) || string.IsNullOrWhiteSpace(verdictPath))
|
||||
return;
|
||||
try
|
||||
{
|
||||
await _table.DeleteEntityAsync(tenantId.ToLowerInvariant(), BuildVerdictCalibrationRowKey(kind, verdictPath, status));
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// already gone - idempotent
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VerdictCalibration tracker delete failed: tenant={TenantId} kind={Kind} path={Path}", tenantId, kind, verdictPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<VerdictCalibrationAlert>> GetVerdictCalibrationAlertsAsync(string tenantId)
|
||||
{
|
||||
var results = new List<VerdictCalibrationAlert>();
|
||||
if (string.IsNullOrWhiteSpace(tenantId)) return results;
|
||||
try
|
||||
{
|
||||
var partitionKey = tenantId.ToLowerInvariant();
|
||||
// '}' sorts directly after '|' - prefix range over "verdictcalibration|...".
|
||||
var filter = $"PartitionKey eq '{partitionKey}' and RowKey ge '{VerdictCalibrationRowKeyPrefix}' and RowKey lt 'verdictcalibration}}'";
|
||||
await foreach (var entity in _table.QueryAsync<TableEntity>(filter: filter))
|
||||
{
|
||||
results.Add(MapToVerdictCalibrationAlert(entity));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "VerdictCalibration tracker list failed for tenant {TenantId}", tenantId);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// internal static: pinned by round-trip unit tests (table-storage-serialization rule).
|
||||
internal static TableEntity BuildVerdictCalibrationEntity(string tenantId, VerdictCalibrationAlert alert)
|
||||
{
|
||||
var entity = new TableEntity(tenantId.ToLowerInvariant(), BuildVerdictCalibrationRowKey(alert.Kind, alert.VerdictPath, alert.Status))
|
||||
{
|
||||
["TenantId"] = tenantId,
|
||||
["Kind"] = alert.Kind,
|
||||
["VerdictPath"] = alert.VerdictPath,
|
||||
["Status"] = alert.Status ?? string.Empty,
|
||||
["WindowHitCount"] = alert.WindowHitCount,
|
||||
["WindowSessionCount"] = alert.WindowSessionCount,
|
||||
["BaselineHitCount"] = alert.BaselineHitCount,
|
||||
["BaselineSessionCount"] = alert.BaselineSessionCount,
|
||||
["WindowRatePct"] = alert.WindowRatePct,
|
||||
["BaselineRatePct"] = alert.BaselineRatePct,
|
||||
["WindowStartDate"] = alert.WindowStartDate ?? string.Empty,
|
||||
["WindowEndDate"] = alert.WindowEndDate ?? string.Empty,
|
||||
["FirstNotifiedAt"] = alert.FirstNotifiedAt,
|
||||
["LastEvaluatedAt"] = alert.LastEvaluatedAt,
|
||||
};
|
||||
if (alert.Lift.HasValue)
|
||||
entity["Lift"] = alert.Lift.Value;
|
||||
if (alert.Dimension != null)
|
||||
entity["DimensionJson"] = JsonSerializer.Serialize(alert.Dimension);
|
||||
return entity;
|
||||
}
|
||||
|
||||
internal static VerdictCalibrationAlert MapToVerdictCalibrationAlert(TableEntity entity)
|
||||
{
|
||||
RuleRegressionDimension? dimension = null;
|
||||
var dimensionJson = entity.GetString("DimensionJson");
|
||||
if (!string.IsNullOrEmpty(dimensionJson))
|
||||
{
|
||||
try { dimension = JsonSerializer.Deserialize<RuleRegressionDimension>(dimensionJson!); }
|
||||
catch (JsonException) { dimension = null; }
|
||||
}
|
||||
return new VerdictCalibrationAlert
|
||||
{
|
||||
TenantId = entity.GetString("TenantId") ?? entity.PartitionKey,
|
||||
Kind = entity.GetString("Kind") ?? string.Empty,
|
||||
VerdictPath = entity.GetString("VerdictPath") ?? string.Empty,
|
||||
Status = entity.GetString("Status") ?? string.Empty,
|
||||
WindowHitCount = entity.GetInt32("WindowHitCount") ?? 0,
|
||||
WindowSessionCount = entity.GetInt32("WindowSessionCount") ?? 0,
|
||||
BaselineHitCount = entity.GetInt32("BaselineHitCount") ?? 0,
|
||||
BaselineSessionCount = entity.GetInt32("BaselineSessionCount") ?? 0,
|
||||
WindowRatePct = entity.GetDouble("WindowRatePct") ?? 0,
|
||||
BaselineRatePct = entity.GetDouble("BaselineRatePct") ?? 0,
|
||||
Lift = entity.GetDouble("Lift"),
|
||||
WindowStartDate = entity.GetString("WindowStartDate") ?? string.Empty,
|
||||
WindowEndDate = entity.GetString("WindowEndDate") ?? string.Empty,
|
||||
Dimension = dimension,
|
||||
FirstNotifiedAt = entity.GetDateTimeOffset("FirstNotifiedAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
LastEvaluatedAt = entity.GetDateTimeOffset("LastEvaluatedAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
};
|
||||
}
|
||||
|
||||
internal const string AppVersionRegressionRowKeyPrefix = "appversionregression|";
|
||||
|
||||
/// <summary>
|
||||
|
||||
+6
@@ -60,6 +60,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<List<SessionSummary>> GetUsageWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null)
|
||||
=> _storage.GetUsageWindowSessionsAsync(startDate, endDate, tenantId);
|
||||
|
||||
public Task<List<SessionSummary>> GetMaintenanceWindowSessionsAsync(DateTime startDate, DateTime endDate)
|
||||
=> _storage.GetMaintenanceWindowSessionsAsync(startDate, endDate);
|
||||
|
||||
public Task<List<SessionSummary>> GetGeoWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null)
|
||||
=> _storage.GetGeoWindowSessionsAsync(startDate, endDate, tenantId);
|
||||
|
||||
@@ -72,6 +75,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults)
|
||||
=> _storage.GetLegacyTimeoutFailedSessionsAsync(tenantId, maxResults);
|
||||
|
||||
public Task<List<SessionSummary>> GetSelfDeployingSilentSessionsAsync(string tenantId, int maxResults)
|
||||
=> _storage.GetSelfDeployingSilentSessionsAsync(tenantId, maxResults);
|
||||
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId)
|
||||
=> _storage.GetSessionsLeanAsync(tenantId);
|
||||
|
||||
|
||||
+12
@@ -159,5 +159,17 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
public Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteDeviceJourneyAggregatesOlderThanAsync(cutoffDate);
|
||||
|
||||
public Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate)
|
||||
=> _storage.SaveVerdictCalibrationAggregateAsync(aggregate);
|
||||
|
||||
public Task<List<VerdictCalibrationDailyAggregate>> GetVerdictCalibrationAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate)
|
||||
=> _storage.GetVerdictCalibrationAggregatesAsync(tenantId, startDate, endDate);
|
||||
|
||||
public Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey)
|
||||
=> _storage.DeleteVerdictCalibrationAggregateAsync(tenantId, dateKey);
|
||||
|
||||
public Task<int> DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
=> _storage.DeleteVerdictCalibrationAggregatesOlderThanAsync(cutoffDate);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -58,7 +58,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
=> _storage.GetAllSessionStatsAsync(tenantIdFilter, days, allowedTenantIds);
|
||||
|
||||
public async Task<bool> UpdateSessionStatusAsync(
|
||||
string tenantId, string sessionId, SessionStatus status,
|
||||
string tenantId, string sessionId, SessionStatus status, string verdictPath,
|
||||
EnrollmentPhase? currentPhase = null, string? failureReason = null,
|
||||
DateTime? completedAt = null, DateTime? earliestEventTimestamp = null,
|
||||
DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null,
|
||||
@@ -68,7 +68,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
string? failureSnapshotJson = null, bool allowTerminalReclassification = false,
|
||||
bool espSoftFailure = false, string? completionSource = null)
|
||||
{
|
||||
var transitioned = await _storage.UpdateSessionStatusAsync(tenantId, sessionId, status,
|
||||
var transitioned = await _storage.UpdateSessionStatusAsync(tenantId, sessionId, status, verdictPath,
|
||||
currentPhase, failureReason, completedAt, earliestEventTimestamp,
|
||||
latestEventTimestamp, isPreProvisioned, isUserDriven, resumedAt,
|
||||
stalledAt, clearStalledAt, clearFailureReason, failureSource, adminMarkedAction,
|
||||
@@ -142,8 +142,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
=> _storage.UpdateSessionDiagnosticsBlobAsync(tenantId, sessionId, blobName, destination);
|
||||
|
||||
public Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned,
|
||||
SessionStatus? status = null, bool? isUserDriven = null)
|
||||
=> _storage.SetSessionPreProvisionedAsync(tenantId, sessionId, isPreProvisioned, status, isUserDriven);
|
||||
SessionStatus? status = null, bool? isUserDriven = null, string? verdictPath = null)
|
||||
=> _storage.SetSessionPreProvisionedAsync(tenantId, sessionId, isPreProvisioned, status, isUserDriven, verdictPath);
|
||||
|
||||
public Task UpdateSessionGeoAsync(string tenantId, string sessionId,
|
||||
string? country, string? region, string? city, string? loc)
|
||||
|
||||
+3
@@ -18,5 +18,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
public Task InitializeAsync()
|
||||
=> _storage.InitializeTablesAsync();
|
||||
|
||||
public Task EnsureAllAsync()
|
||||
=> _storage.EnsureAllTablesAsync();
|
||||
}
|
||||
}
|
||||
|
||||
+5
-1
@@ -35,6 +35,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
/// Query parameters:
|
||||
/// - mode: "legacy_timeouts" (default) — re-classify pre-classifier "Session timed out" Failed rows;
|
||||
/// "pending_orphans" — resolve Pending rows superseded by a later session of the same device.
|
||||
/// "self_deploying_silent" — heal self-deploying (kiosk) rows parked Incomplete/AwaitingUser/Stalled
|
||||
/// after Device ESP provisioned to Succeeded (classifier rule 1c, 2026-08-23).
|
||||
/// - dryRun: "false" to actually write; anything else (or absent) reports only.
|
||||
/// - tenantId: optional single-tenant scope (GUID); absent = all tenants.
|
||||
/// - maxSessions: per-invocation examination cap (default 200, max 2000). Re-run to continue.
|
||||
@@ -82,6 +84,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
await _service.ReclassifyLegacyTimeoutsAsync(tenantIdScope, dryRun, maxSessions, userEmail),
|
||||
LegacyReclassificationService.ModePendingOrphans =>
|
||||
await _service.ResolvePendingOrphansAsync(tenantIdScope, dryRun, maxSessions, userEmail),
|
||||
LegacyReclassificationService.ModeSelfDeployingSilent =>
|
||||
await _service.ReconcileSelfDeployingSilentAsync(tenantIdScope, dryRun, maxSessions, userEmail),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
@@ -90,7 +94,7 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
var bad = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await bad.WriteAsJsonAsync(new
|
||||
{
|
||||
error = $"Unknown mode '{mode}'. Valid: {LegacyReclassificationService.ModeLegacyTimeouts}, {LegacyReclassificationService.ModePendingOrphans}"
|
||||
error = $"Unknown mode '{mode}'. Valid: {LegacyReclassificationService.ModeLegacyTimeouts}, {LegacyReclassificationService.ModePendingOrphans}, {LegacyReclassificationService.ModeSelfDeployingSilent}"
|
||||
});
|
||||
return bad;
|
||||
}
|
||||
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Metrics
|
||||
{
|
||||
/// <summary>
|
||||
/// Operator-only verdict calibration matrix (docs/backend/verdict-calibration.md): per verdict
|
||||
/// path, how many sessions it produced in the window, its share, how often it was overridden
|
||||
/// (admin / late completion / other), the 7-day re-enrollment proxy, and a 7d-vs-28d trend.
|
||||
/// Platform scope ("global" partition) or one tenant via ?tenantId=. Serves the admin
|
||||
/// metrics page and the <c>get_verdict_calibration</c> MCP tool. Never customer-facing —
|
||||
/// these are platform-internal classifier diagnostics, not product features.
|
||||
/// </summary>
|
||||
public class GetVerdictCalibrationFunction
|
||||
{
|
||||
private readonly ILogger<GetVerdictCalibrationFunction> _logger;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly IHardwareRejectionNotificationTracker _tracker;
|
||||
|
||||
public GetVerdictCalibrationFunction(
|
||||
ILogger<GetVerdictCalibrationFunction> logger,
|
||||
IMetricsRepository metricsRepo,
|
||||
IHardwareRejectionNotificationTracker tracker)
|
||||
{
|
||||
_logger = logger;
|
||||
_metricsRepo = metricsRepo;
|
||||
_tracker = tracker;
|
||||
}
|
||||
|
||||
[Function("GetVerdictCalibration")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/metrics/verdict-calibration")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Authentication + GlobalReadOrAdmin authorization enforced by PolicyEnforcementMiddleware
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var tenantIdFilter = query["tenantId"];
|
||||
var days = VerdictCalibrationResponse.ClampDays(query["days"]);
|
||||
var partition = "global";
|
||||
if (!string.IsNullOrWhiteSpace(tenantIdFilter))
|
||||
{
|
||||
if (!Guid.TryParse(tenantIdFilter, out var parsed))
|
||||
{
|
||||
var bad = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await bad.WriteAsJsonAsync(new { success = false, message = "tenantId must be a GUID" });
|
||||
return bad;
|
||||
}
|
||||
partition = parsed.ToString("D");
|
||||
}
|
||||
|
||||
var today = DateTime.UtcNow.Date;
|
||||
// One read covers both the requested window and the trend horizon (7d window + 28d baseline).
|
||||
var readDays = Math.Max(days, VerdictCalibrationResponse.TrendHorizonDays);
|
||||
var daily = await _metricsRepo.GetVerdictCalibrationAggregatesAsync(
|
||||
partition, VerdictCalibrationResponse.InclusiveWindowStart(today, readDays), today);
|
||||
// Active drift episodes of this partition (tracker keyspace; fail-soft empty).
|
||||
var alerts = await _tracker.GetVerdictCalibrationAlertsAsync(partition);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(VerdictCalibrationResponse.Build(daily, partition, today, days, alerts));
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error fetching verdict calibration");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Pure response builder — internal static so the matrix arithmetic is pinned by tests.</summary>
|
||||
internal static class VerdictCalibrationResponse
|
||||
{
|
||||
internal const int DefaultWindowDays = 30;
|
||||
internal const int MaxWindowDays = 180; // aggregate retention
|
||||
internal const int TrendWindowDays = 7;
|
||||
internal const int TrendBaselineDays = 28;
|
||||
internal const int TrendHorizonDays = TrendWindowDays + TrendBaselineDays;
|
||||
|
||||
internal static int ClampDays(string? raw)
|
||||
{
|
||||
if (!int.TryParse(raw, out var days)) return DefaultWindowDays;
|
||||
return Math.Clamp(days, 1, MaxWindowDays);
|
||||
}
|
||||
|
||||
/// <summary>"Last N days" = exactly N calendar day keys including today (both range ends inclusive).</summary>
|
||||
internal static DateTime InclusiveWindowStart(DateTime today, int days) => today.AddDays(-(days - 1));
|
||||
|
||||
internal sealed class PathRow
|
||||
{
|
||||
public string VerdictPath { get; set; } = string.Empty;
|
||||
public string Status { get; set; } = string.Empty;
|
||||
public int Count { get; set; }
|
||||
public double SharePct { get; set; }
|
||||
public int DerivedCount { get; set; }
|
||||
public int Eligible7d { get; set; }
|
||||
public int ReEnrolled7d { get; set; }
|
||||
/// <summary>Null below the minimum eligible count — never a rate on a handful of sessions.</summary>
|
||||
public double? ReEnrollRatePct { get; set; }
|
||||
public int OverriddenByAdmin { get; set; }
|
||||
public int OverriddenByLateCompletion { get; set; }
|
||||
public int OverriddenOther { get; set; }
|
||||
public TrendWindow Window7 { get; set; } = new();
|
||||
public TrendWindow Baseline28 { get; set; } = new();
|
||||
/// <summary>Window share ÷ baseline share; null when the baseline share is 0 (a new path has no finite lift — never invented).</summary>
|
||||
public double? Lift { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class TrendWindow
|
||||
{
|
||||
public int Count { get; set; }
|
||||
public int Sessions { get; set; }
|
||||
public double SharePct { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Minimum eligible sessions before a re-enrollment rate is stated (truthfulness: no rate on n<20).</summary>
|
||||
internal const int MinEligibleForRate = 20;
|
||||
|
||||
internal static object Build(
|
||||
IReadOnlyList<VerdictCalibrationDailyAggregate> daily, string partition, DateTime today, int days,
|
||||
IReadOnlyList<VerdictCalibrationAlert>? alerts = null)
|
||||
{
|
||||
var windowStart = Key(InclusiveWindowStart(today, days));
|
||||
var trendWindowStart = Key(InclusiveWindowStart(today, TrendWindowDays));
|
||||
var baselineStart = Key(InclusiveWindowStart(today, TrendHorizonDays));
|
||||
|
||||
var rows = new Dictionary<(string Path, string Status), PathRow>();
|
||||
int sessions = 0, terminal = 0, derived = 0;
|
||||
int trendSessions = 0, baselineSessions = 0;
|
||||
DateTime? computedAt = null;
|
||||
var versions = new HashSet<int>();
|
||||
|
||||
PathRow Row(VerdictCalibrationBucket b)
|
||||
{
|
||||
var key = (b.VerdictPath, b.Status);
|
||||
if (!rows.TryGetValue(key, out var row))
|
||||
{
|
||||
row = new PathRow { VerdictPath = b.VerdictPath, Status = b.Status };
|
||||
rows[key] = row;
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
foreach (var day in daily)
|
||||
{
|
||||
var inWindow = string.CompareOrdinal(day.Date, windowStart) >= 0;
|
||||
var inTrendWindow = string.CompareOrdinal(day.Date, trendWindowStart) >= 0;
|
||||
var inBaseline = !inTrendWindow && string.CompareOrdinal(day.Date, baselineStart) >= 0;
|
||||
|
||||
if (inWindow)
|
||||
{
|
||||
sessions += day.SessionCount;
|
||||
terminal += day.TerminalSessionCount;
|
||||
versions.Add(day.Version);
|
||||
if (computedAt == null || day.ComputedAt > computedAt) computedAt = day.ComputedAt;
|
||||
}
|
||||
if (inTrendWindow) trendSessions += day.SessionCount;
|
||||
if (inBaseline) baselineSessions += day.SessionCount;
|
||||
|
||||
foreach (var b in day.Buckets)
|
||||
{
|
||||
// Overrides are attributed to the prior path and can exist on a bucket with Count=0.
|
||||
var row = Row(b);
|
||||
if (inWindow)
|
||||
{
|
||||
row.Count += b.Count;
|
||||
row.DerivedCount += b.DerivedCount;
|
||||
row.Eligible7d += b.Eligible7d;
|
||||
row.ReEnrolled7d += b.ReEnrolled7d;
|
||||
row.OverriddenByAdmin += b.OverriddenByAdmin;
|
||||
row.OverriddenByLateCompletion += b.OverriddenByLateCompletion;
|
||||
row.OverriddenOther += b.OverriddenOther;
|
||||
derived += b.DerivedCount;
|
||||
}
|
||||
if (inTrendWindow) row.Window7.Count += b.Count;
|
||||
if (inBaseline) row.Baseline28.Count += b.Count;
|
||||
}
|
||||
}
|
||||
|
||||
var list = rows.Values
|
||||
.Where(r => r.Count > 0 || r.OverriddenByAdmin + r.OverriddenByLateCompletion + r.OverriddenOther > 0)
|
||||
.OrderByDescending(r => r.Count)
|
||||
.ThenBy(r => r.VerdictPath, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.Status, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
foreach (var r in list)
|
||||
{
|
||||
r.SharePct = Pct(r.Count, sessions);
|
||||
r.ReEnrollRatePct = r.Eligible7d >= MinEligibleForRate ? Pct(r.ReEnrolled7d, r.Eligible7d) : null;
|
||||
r.Window7.Sessions = trendSessions;
|
||||
r.Window7.SharePct = Pct(r.Window7.Count, trendSessions);
|
||||
r.Baseline28.Sessions = baselineSessions;
|
||||
r.Baseline28.SharePct = Pct(r.Baseline28.Count, baselineSessions);
|
||||
r.Lift = r.Baseline28.SharePct > 0 && trendSessions > 0
|
||||
? Math.Round(r.Window7.SharePct / r.Baseline28.SharePct, 2)
|
||||
: null;
|
||||
}
|
||||
|
||||
return new
|
||||
{
|
||||
success = true,
|
||||
tenantId = partition,
|
||||
windowDays = days,
|
||||
windowStart,
|
||||
windowEnd = Key(today),
|
||||
computedAt,
|
||||
versions = versions.OrderBy(v => v).ToArray(),
|
||||
totals = new { sessions, terminal, derived, days = daily.Count(d => string.CompareOrdinal(d.Date, windowStart) >= 0) },
|
||||
trend = new { windowDays = TrendWindowDays, baselineDays = TrendBaselineDays, windowSessions = trendSessions, baselineSessions },
|
||||
paths = list,
|
||||
// Wording contract shared with the rule radar: a dimension is correlated, not causal.
|
||||
alerts = (alerts ?? Array.Empty<VerdictCalibrationAlert>())
|
||||
.OrderByDescending(a => a.FirstNotifiedAt)
|
||||
.ToList(),
|
||||
};
|
||||
}
|
||||
|
||||
private static string Key(DateTime d) => d.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
|
||||
|
||||
private static double Pct(int part, int whole) => whole <= 0 ? 0 : Math.Round(100.0 * part / whole, 1);
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
tenantId,
|
||||
sessionId,
|
||||
SessionStatus.Failed,
|
||||
VerdictPaths.ManualFailed,
|
||||
currentPhase: null, // Keep current phase
|
||||
failureReason: "Manually marked as failed by administrator",
|
||||
failureSource: "manual",
|
||||
|
||||
+1
@@ -49,6 +49,7 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
tenantId,
|
||||
sessionId,
|
||||
SessionStatus.Succeeded,
|
||||
VerdictPaths.ManualSucceeded,
|
||||
currentPhase: null, // Keep current phase
|
||||
failureReason: "Manually marked as succeeded by administrator",
|
||||
adminMarkedAction: "Succeeded"
|
||||
|
||||
+1
-1
@@ -305,7 +305,7 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
$"enrollment session at {registration.StartedAt:yyyy-MM-dd HH:mm} UTC before this one " +
|
||||
"reached a terminal state.";
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
registration.TenantId, stale.SessionId, SessionStatus.Incomplete,
|
||||
registration.TenantId, stale.SessionId, SessionStatus.Incomplete, VerdictPaths.RegisterSuperseded,
|
||||
failureReason: reason, failureSource: "superseded_by_reregistration");
|
||||
if (!transitioned)
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Helpers;
|
||||
|
||||
/// <summary>
|
||||
/// One drift finding of the verdict-calibration radar (pure output; the maintenance partial
|
||||
/// turns it into a tracker episode + ops event).
|
||||
/// </summary>
|
||||
public class VerdictCalibrationFinding
|
||||
{
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
public string VerdictPath { get; set; } = string.Empty;
|
||||
public string Status { get; set; } = string.Empty;
|
||||
public int WindowHitCount { get; set; }
|
||||
public int WindowSessionCount { get; set; }
|
||||
public int BaselineHitCount { get; set; }
|
||||
public int BaselineSessionCount { get; set; }
|
||||
public double WindowRatePct { get; set; }
|
||||
public double BaselineRatePct { get; set; }
|
||||
public double? Lift { get; set; }
|
||||
public string WindowStartDate { get; set; } = string.Empty;
|
||||
public string WindowEndDate { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure drift detection over one tenant's verdict-calibration daily rows
|
||||
/// (docs/backend/verdict-calibration.md) — the same statistics as the rule-regression radar
|
||||
/// (7-day window vs the prior 28 days, lift ≥2×, Wilson-separated via
|
||||
/// <see cref="MetricsMath.RateIncreaseSeparated"/>, small-n gates) applied to three questions:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>share_regression</b> — one verdict path's share of all sessions doubled. Points at a rule
|
||||
/// (or an agent path) whose conditions suddenly match much more often.</item>
|
||||
/// <item><b>silence_share_regression</b> — the share of sessions the backend had to decide because
|
||||
/// the agent went silent (sweep:* + maxlife:*) doubled. An agent-liveness signal, not a classifier one.</item>
|
||||
/// <item><b>evidence_gap</b> — absolute gate: the pure fallthrough rule (r6) decides ≥20 % of the
|
||||
/// window's classifier verdicts. The evidence the other rules need is not arriving.</item>
|
||||
/// </list>
|
||||
/// Derived (<c>legacy:*</c>) paths are excluded from all three — pre-instrumentation attribution
|
||||
/// is too weak to alert on. Agent-declared paths (<c>agent:*</c>, <c>register:*</c>) are excluded
|
||||
/// from the per-path share regression: their share tracks the customer's workflow mix (a
|
||||
/// WhiteGlove rollout week doubles <c>agent:whiteglove_pending</c>; a Continue-Anyway tenant
|
||||
/// lives on <c>agent:complete_soft</c>) — real signals, but not about the BACKEND classifier
|
||||
/// this radar calibrates. First production pass (2026-08-23) fired exactly those three. Re-arm
|
||||
/// mirrors the rule radar: share back under 1.5× baseline or the path stops occurring; the
|
||||
/// evidence gap re-arms under 15 %.
|
||||
/// </summary>
|
||||
public static class VerdictCalibrationRadar
|
||||
{
|
||||
public const int WindowDays = RuleRegressionRadar.WindowDays;
|
||||
public const int BaselineDays = RuleRegressionRadar.BaselineDays;
|
||||
public const int MinWindowHits = RuleRegressionRadar.MinWindowHitSessions;
|
||||
public const int MinWindowSessions = RuleRegressionRadar.MinWindowSessions;
|
||||
public const double MinLift = RuleRegressionRadar.MinLift;
|
||||
public const double ReArmLiftFactor = RuleRegressionRadar.ReArmLiftFactor;
|
||||
|
||||
/// <summary>Evidence gap fires at ≥20 % r6 share of the window's classifier verdicts (≥20 verdicts) and re-arms under 15 %.</summary>
|
||||
public const double EvidenceGapFirePct = 20.0;
|
||||
public const double EvidenceGapReArmPct = 15.0;
|
||||
public const int EvidenceGapMinVerdicts = 20;
|
||||
|
||||
public const string SilenceGroupPath = "sweep+maxlife";
|
||||
public const string EvidenceGapPath = "r6";
|
||||
public const string GroupStatus = "*";
|
||||
|
||||
private const string LegacyOrigin = "legacy";
|
||||
|
||||
/// <summary>
|
||||
/// True when a path is eligible for the per-path share regression: backend-decided paths only
|
||||
/// (sweep / maxlife / late / retro / rule / manual / ingest). Agent-declared and registration
|
||||
/// paths mirror customer workflow, legacy paths are derived.
|
||||
/// </summary>
|
||||
public static bool IsShareRegressionEligible(string path)
|
||||
{
|
||||
var origin = VerdictPaths.Origin(path);
|
||||
return origin != LegacyOrigin
|
||||
&& origin != "agent"
|
||||
&& origin != "register";
|
||||
}
|
||||
|
||||
public static List<VerdictCalibrationFinding> Evaluate(IReadOnlyList<VerdictCalibrationDailyAggregate> tenantRows, DateTime targetDateUtc)
|
||||
{
|
||||
var findings = new List<VerdictCalibrationFinding>();
|
||||
var horizon = Summarize(tenantRows, targetDateUtc);
|
||||
var windowStart = targetDateUtc.Date.AddDays(-(WindowDays - 1)).ToString("yyyy-MM-dd");
|
||||
var windowEnd = targetDateUtc.Date.ToString("yyyy-MM-dd");
|
||||
|
||||
// 1. Per-path share regression.
|
||||
foreach (var pair in horizon.Paths)
|
||||
{
|
||||
if (!IsShareRegressionEligible(pair.Key.Path)) continue;
|
||||
var f = TryShareRegression(VerdictCalibrationAlertKinds.ShareRegression, pair.Key.Path, pair.Key.Status,
|
||||
pair.Value.WindowHits, horizon.WindowSessions, pair.Value.BaselineHits, horizon.BaselineSessions, windowStart, windowEnd);
|
||||
if (f != null) findings.Add(f);
|
||||
}
|
||||
|
||||
// 2. Silence-share regression (group).
|
||||
{
|
||||
var f = TryShareRegression(VerdictCalibrationAlertKinds.SilenceShareRegression, SilenceGroupPath, GroupStatus,
|
||||
horizon.SilenceWindowHits, horizon.WindowSessions, horizon.SilenceBaselineHits, horizon.BaselineSessions, windowStart, windowEnd);
|
||||
if (f != null) findings.Add(f);
|
||||
}
|
||||
|
||||
// 3. Evidence gap (absolute).
|
||||
if (horizon.ClassifierWindowVerdicts >= EvidenceGapMinVerdicts)
|
||||
{
|
||||
var pct = 100.0 * horizon.R6WindowHits / horizon.ClassifierWindowVerdicts;
|
||||
if (pct >= EvidenceGapFirePct)
|
||||
{
|
||||
findings.Add(new VerdictCalibrationFinding
|
||||
{
|
||||
Kind = VerdictCalibrationAlertKinds.EvidenceGap,
|
||||
VerdictPath = EvidenceGapPath,
|
||||
Status = GroupStatus,
|
||||
WindowHitCount = horizon.R6WindowHits,
|
||||
WindowSessionCount = horizon.ClassifierWindowVerdicts,
|
||||
BaselineHitCount = horizon.R6BaselineHits,
|
||||
BaselineSessionCount = horizon.ClassifierBaselineVerdicts,
|
||||
WindowRatePct = Math.Round(pct, 1),
|
||||
BaselineRatePct = horizon.ClassifierBaselineVerdicts > 0
|
||||
? Math.Round(100.0 * horizon.R6BaselineHits / horizon.ClassifierBaselineVerdicts, 1) : 0,
|
||||
Lift = null,
|
||||
WindowStartDate = windowStart,
|
||||
WindowEndDate = windowEnd,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return findings
|
||||
.OrderByDescending(f => f.WindowRatePct)
|
||||
.ThenBy(f => f.Kind, StringComparer.Ordinal)
|
||||
.ThenBy(f => f.VerdictPath, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>True when an ACTIVE alert may re-arm given the current horizon sums (or when its path is no longer eligible — episodes from an earlier, broader gate clear themselves).</summary>
|
||||
public static bool ShouldReArm(VerdictCalibrationAlert alert, IReadOnlyList<VerdictCalibrationDailyAggregate> tenantRows, DateTime targetDateUtc)
|
||||
{
|
||||
if (alert.Kind == VerdictCalibrationAlertKinds.ShareRegression && !IsShareRegressionEligible(alert.VerdictPath))
|
||||
return true;
|
||||
|
||||
var horizon = Summarize(tenantRows, targetDateUtc);
|
||||
switch (alert.Kind)
|
||||
{
|
||||
case VerdictCalibrationAlertKinds.EvidenceGap:
|
||||
if (horizon.ClassifierWindowVerdicts < EvidenceGapMinVerdicts) return true;
|
||||
return 100.0 * horizon.R6WindowHits / horizon.ClassifierWindowVerdicts < EvidenceGapReArmPct;
|
||||
|
||||
case VerdictCalibrationAlertKinds.SilenceShareRegression:
|
||||
return ReArmByRate(horizon.SilenceWindowHits, horizon.WindowSessions, horizon.SilenceBaselineHits, horizon.BaselineSessions);
|
||||
|
||||
default:
|
||||
horizon.Paths.TryGetValue((alert.VerdictPath, alert.Status), out var sums);
|
||||
return ReArmByRate(sums.WindowHits, horizon.WindowSessions, sums.BaselineHits, horizon.BaselineSessions);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Current numbers for an ongoing episode (refresh without re-firing).</summary>
|
||||
public static (int WindowHits, int WindowSessions, int BaselineHits, int BaselineSessions) CurrentSums(
|
||||
VerdictCalibrationAlert alert, IReadOnlyList<VerdictCalibrationDailyAggregate> tenantRows, DateTime targetDateUtc)
|
||||
{
|
||||
var horizon = Summarize(tenantRows, targetDateUtc);
|
||||
switch (alert.Kind)
|
||||
{
|
||||
case VerdictCalibrationAlertKinds.EvidenceGap:
|
||||
return (horizon.R6WindowHits, horizon.ClassifierWindowVerdicts, horizon.R6BaselineHits, horizon.ClassifierBaselineVerdicts);
|
||||
case VerdictCalibrationAlertKinds.SilenceShareRegression:
|
||||
return (horizon.SilenceWindowHits, horizon.WindowSessions, horizon.SilenceBaselineHits, horizon.BaselineSessions);
|
||||
default:
|
||||
horizon.Paths.TryGetValue((alert.VerdictPath, alert.Status), out var sums);
|
||||
return (sums.WindowHits, horizon.WindowSessions, sums.BaselineHits, horizon.BaselineSessions);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ReArmByRate(int windowHits, int windowSessions, int baselineHits, int baselineSessions)
|
||||
{
|
||||
if (windowHits == 0 || windowSessions == 0) return true;
|
||||
if (baselineSessions == 0 || baselineHits == 0) return false; // zero-baseline episode re-arms only via hits-stopped
|
||||
var windowRate = (double)windowHits / windowSessions;
|
||||
var baselineRate = (double)baselineHits / baselineSessions;
|
||||
return windowRate < ReArmLiftFactor * baselineRate;
|
||||
}
|
||||
|
||||
private static VerdictCalibrationFinding? TryShareRegression(string kind, string path, string status,
|
||||
int windowHits, int windowSessions, int baselineHits, int baselineSessions, string windowStart, string windowEnd)
|
||||
{
|
||||
if (windowHits < MinWindowHits) return null;
|
||||
if (windowSessions < MinWindowSessions) return null;
|
||||
if (baselineSessions <= 0) return null;
|
||||
|
||||
var windowRate = (double)windowHits / windowSessions;
|
||||
var baselineRate = (double)baselineHits / baselineSessions;
|
||||
if (windowRate < MinLift * baselineRate) return null;
|
||||
if (!MetricsMath.RateIncreaseSeparated(windowHits, windowSessions, baselineHits, baselineSessions)) return null;
|
||||
|
||||
return new VerdictCalibrationFinding
|
||||
{
|
||||
Kind = kind,
|
||||
VerdictPath = path,
|
||||
Status = status,
|
||||
WindowHitCount = windowHits,
|
||||
WindowSessionCount = windowSessions,
|
||||
BaselineHitCount = baselineHits,
|
||||
BaselineSessionCount = baselineSessions,
|
||||
WindowRatePct = Math.Round(windowRate * 100, 1),
|
||||
BaselineRatePct = Math.Round(baselineRate * 100, 1),
|
||||
Lift = baselineRate > 0 ? Math.Round(windowRate / baselineRate, 1) : null,
|
||||
WindowStartDate = windowStart,
|
||||
WindowEndDate = windowEnd,
|
||||
};
|
||||
}
|
||||
|
||||
internal readonly record struct PathSums(int WindowHits, int BaselineHits);
|
||||
|
||||
internal sealed class Horizon
|
||||
{
|
||||
public int WindowSessions;
|
||||
public int BaselineSessions;
|
||||
public int SilenceWindowHits;
|
||||
public int SilenceBaselineHits;
|
||||
public int R6WindowHits;
|
||||
public int R6BaselineHits;
|
||||
public int ClassifierWindowVerdicts;
|
||||
public int ClassifierBaselineVerdicts;
|
||||
public readonly Dictionary<(string Path, string Status), PathSums> Paths = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sums the daily rows into the trailing window [target−6d, target] and the prior baseline
|
||||
/// [target−34d, target−7d] (ISO-string date compare, rows outside the horizon ignored).
|
||||
/// Silence group = sweep:* + maxlife:* (incl. non-rule sweep paths like sweep:stalled — the
|
||||
/// question is "how often did the backend have to decide", not which rule did). Classifier
|
||||
/// verdicts = stamped rule paths (<see cref="VerdictPaths.IsClassifierPath"/>); r6 = their
|
||||
/// fallthrough subset.
|
||||
/// </summary>
|
||||
internal static Horizon Summarize(IReadOnlyList<VerdictCalibrationDailyAggregate> rows, DateTime targetDateUtc)
|
||||
{
|
||||
var windowStart = targetDateUtc.Date.AddDays(-(WindowDays - 1)).ToString("yyyy-MM-dd");
|
||||
var baselineStart = targetDateUtc.Date.AddDays(-(WindowDays - 1 + BaselineDays)).ToString("yyyy-MM-dd");
|
||||
var target = targetDateUtc.Date.ToString("yyyy-MM-dd");
|
||||
|
||||
var h = new Horizon();
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (string.CompareOrdinal(row.Date, target) > 0) continue;
|
||||
bool inWindow;
|
||||
if (string.CompareOrdinal(row.Date, windowStart) >= 0) inWindow = true;
|
||||
else if (string.CompareOrdinal(row.Date, baselineStart) >= 0) inWindow = false;
|
||||
else continue;
|
||||
|
||||
if (inWindow) h.WindowSessions += row.SessionCount; else h.BaselineSessions += row.SessionCount;
|
||||
|
||||
foreach (var b in row.Buckets)
|
||||
{
|
||||
if (b.Count == 0) continue;
|
||||
var key = (b.VerdictPath, b.Status);
|
||||
h.Paths.TryGetValue(key, out var sums);
|
||||
h.Paths[key] = inWindow
|
||||
? new PathSums(sums.WindowHits + b.Count, sums.BaselineHits)
|
||||
: new PathSums(sums.WindowHits, sums.BaselineHits + b.Count);
|
||||
|
||||
var origin = VerdictPaths.Origin(b.VerdictPath);
|
||||
if (origin == VerdictPaths.OriginSweep || origin == VerdictPaths.OriginMaxLifetime)
|
||||
{
|
||||
if (inWindow) h.SilenceWindowHits += b.Count; else h.SilenceBaselineHits += b.Count;
|
||||
}
|
||||
if (VerdictPaths.IsClassifierPath(b.VerdictPath))
|
||||
{
|
||||
if (inWindow) h.ClassifierWindowVerdicts += b.Count; else h.ClassifierBaselineVerdicts += b.Count;
|
||||
if (b.VerdictPath.EndsWith(":" + ClassifierRules.R6Fallthrough, StringComparison.Ordinal))
|
||||
{
|
||||
if (inWindow) h.R6WindowHits += b.Count; else h.R6BaselineHits += b.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return h;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Helpers
|
||||
{
|
||||
/// <summary>
|
||||
/// Read-side attribution for sessions written BEFORE <see cref="SessionSummary.VerdictPath"/>
|
||||
/// was stamped. No mass backfill: the calibration aggregate calls <see cref="Derive"/> and
|
||||
/// counts the result with <c>Derived=true</c>, so pre-instrumentation history still shows up
|
||||
/// in the matrix and is visibly weaker. The derivation reads the fixed reason literals the
|
||||
/// writers emit (classifier rule prefixes, the max-lifetime / retro suffixes, the manual and
|
||||
/// rule attributions) — when the origin is ambiguous but the rule is known the path is
|
||||
/// <c>legacy:{rule}</c>; when nothing is attributable it is <see cref="VerdictPaths.LegacyUnknown"/>.
|
||||
/// A stamped row is returned verbatim with <c>Derived=false</c>.
|
||||
/// </summary>
|
||||
public static class VerdictPathDerivation
|
||||
{
|
||||
private const string OriginLegacy = "legacy";
|
||||
|
||||
// Literals are owned by EnrollmentTimeoutClassifier / the writers; pinned by
|
||||
// VerdictPathDerivationTests so a wording change there fails here, not silently.
|
||||
private const string R1Prefix = "Enrollment reported an explicit failure before timeout";
|
||||
private const string R1bAwaitingPrefix = "Pre-provisioning (WhiteGlove) completed; no user sign-in since Part 2 resumed";
|
||||
private const string R1bSucceededPrefix = "Reconciled at timeout: pre-provisioning (WhiteGlove Part 1) completed";
|
||||
private const string R1cPrefix = "Reconciled: self-deploying profile";
|
||||
private const string R2Prefix = "Reconciled at timeout: Account Setup completed";
|
||||
private const string R3Prefix = "Agent emergency break fired";
|
||||
private const string R4Prefix = "Reconciled at timeout: user completed setup";
|
||||
private const string R5AwaitingPrefix = "Device Setup completed; awaiting user / Account Setup phase";
|
||||
private const string R5IncompletePrefix = "No completion signal within";
|
||||
private const string R6Prefix = "No Device Setup completion or explicit failure signal observed before timeout";
|
||||
private const string SupersededPrefix = "Superseded by session";
|
||||
private const string SweepStalledPrefix = "Agent silent for";
|
||||
private const string AgentStallProbePrefix = "Agent reported stall after";
|
||||
private const string EspFallbackLiteral = "ESP failure (backend fallback)";
|
||||
/// <summary>Pre-classifier blanket timeout verdict (LegacyReclassificationService) — never agent-reported.</summary>
|
||||
private const string LegacyBlanketTimeoutPrefix = "Session timed out";
|
||||
private const string MaxLifetimeSuffix = "max-lifetime watchdog";
|
||||
private const string RetroSuffix = "Retro-reclassified";
|
||||
|
||||
public static (string Path, bool Derived) Derive(SessionSummary s)
|
||||
{
|
||||
if (s == null) throw new ArgumentNullException(nameof(s));
|
||||
if (!string.IsNullOrWhiteSpace(s.VerdictPath))
|
||||
return (s.VerdictPath!, false);
|
||||
|
||||
return (DeriveLegacy(s), true);
|
||||
}
|
||||
|
||||
private static string DeriveLegacy(SessionSummary s)
|
||||
{
|
||||
var failureReason = s.FailureReason ?? string.Empty;
|
||||
var reconcileReason = s.ReconcileReason ?? string.Empty;
|
||||
var failureSource = s.FailureSource ?? string.Empty;
|
||||
|
||||
// Unambiguous attributions first — independent of status text.
|
||||
if (string.Equals(s.AdminMarkedAction, "Succeeded", StringComparison.OrdinalIgnoreCase))
|
||||
return VerdictPaths.ManualSucceeded;
|
||||
if (string.Equals(s.AdminMarkedAction, "Failed", StringComparison.OrdinalIgnoreCase))
|
||||
return VerdictPaths.ManualFailed;
|
||||
if (failureSource.StartsWith("rule:", StringComparison.OrdinalIgnoreCase))
|
||||
return VerdictPaths.Compose(VerdictPaths.OriginRule, failureSource.Substring(5));
|
||||
|
||||
switch (s.Status)
|
||||
{
|
||||
case SessionStatus.Succeeded:
|
||||
if (reconcileReason.Length == 0
|
||||
|| reconcileReason.StartsWith("Late completion report received", StringComparison.Ordinal))
|
||||
return s.EspSoftFailure ? VerdictPaths.AgentCompleteSoft : VerdictPaths.AgentComplete;
|
||||
if (Starts(reconcileReason, R1bSucceededPrefix)) return Classifier(reconcileReason, ClassifierRules.R1bWhiteGloveSucceeded);
|
||||
if (Starts(reconcileReason, R1cPrefix)) return Classifier(reconcileReason, ClassifierRules.R1cSelfDeploying);
|
||||
if (Starts(reconcileReason, R2Prefix)) return Classifier(reconcileReason, ClassifierRules.R2AccountSetupComplete);
|
||||
if (Starts(reconcileReason, R4Prefix)) return Classifier(reconcileReason, ClassifierRules.R4DesktopHello);
|
||||
return VerdictPaths.LegacyUnknown;
|
||||
|
||||
case SessionStatus.Failed:
|
||||
if (Starts(failureReason, R1Prefix)) return Classifier(failureReason, ClassifierRules.R1ExplicitFailure);
|
||||
if (string.Equals(failureSource, "max_lifetime_watchdog", StringComparison.OrdinalIgnoreCase))
|
||||
return VerdictPaths.Compose(VerdictPaths.OriginMaxLifetime, ClassifierRules.R1ExplicitFailure);
|
||||
if (failureReason.Contains(EspFallbackLiteral, StringComparison.Ordinal)) return VerdictPaths.AgentEspFailureFallback;
|
||||
if (Starts(failureReason, LegacyBlanketTimeoutPrefix)) return VerdictPaths.LegacyUnknown;
|
||||
if (failureReason.Length > 0 && failureSource.Length == 0) return VerdictPaths.AgentFailed;
|
||||
return VerdictPaths.LegacyUnknown;
|
||||
|
||||
case SessionStatus.Incomplete:
|
||||
if (Starts(failureReason, R3Prefix)) return Classifier(failureReason, ClassifierRules.R3EmergencyBreak);
|
||||
if (Starts(failureReason, R5IncompletePrefix)) return Classifier(failureReason, ClassifierRules.R5DeviceSetupIncomplete);
|
||||
if (Starts(failureReason, R6Prefix)) return Classifier(failureReason, ClassifierRules.R6Fallthrough);
|
||||
if (Starts(failureReason, SupersededPrefix)) return VerdictPaths.Compose(OriginLegacy, "superseded");
|
||||
return VerdictPaths.LegacyUnknown;
|
||||
|
||||
case SessionStatus.AwaitingUser:
|
||||
if (Starts(failureReason, R1bAwaitingPrefix)) return VerdictPaths.Compose(OriginLegacy, "wg_awaiting");
|
||||
if (Starts(failureReason, R5AwaitingPrefix)) return Classifier(failureReason, ClassifierRules.R5DeviceSetupAwaiting);
|
||||
return VerdictPaths.LegacyUnknown;
|
||||
|
||||
case SessionStatus.Stalled:
|
||||
if (Starts(failureReason, SweepStalledPrefix)) return VerdictPaths.SweepStalled;
|
||||
if (Starts(failureReason, AgentStallProbePrefix)) return VerdictPaths.AgentStallProbe;
|
||||
return VerdictPaths.LegacyUnknown;
|
||||
|
||||
case SessionStatus.Pending:
|
||||
return VerdictPaths.AgentWhiteGlovePending;
|
||||
|
||||
default:
|
||||
return VerdictPaths.LegacyUnknown;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Classifier verdict with a known rule: the origin is recoverable only from the suffixes
|
||||
/// the max-lifetime and retro writers append; sweep / late-reconcile share the bare text
|
||||
/// and stay <c>legacy:{rule}</c>.
|
||||
/// </summary>
|
||||
private static string Classifier(string reason, string rule)
|
||||
{
|
||||
if (reason.Contains(MaxLifetimeSuffix, StringComparison.Ordinal))
|
||||
return VerdictPaths.Compose(VerdictPaths.OriginMaxLifetime, rule);
|
||||
if (reason.Contains(RetroSuffix, StringComparison.Ordinal))
|
||||
return VerdictPaths.Compose(VerdictPaths.OriginRetro, rule);
|
||||
return VerdictPaths.Compose(OriginLegacy, rule);
|
||||
}
|
||||
|
||||
private static bool Starts(string text, string prefix)
|
||||
=> text.StartsWith(prefix, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
@@ -226,6 +226,7 @@ builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Offboardin
|
||||
// no AddHostedService call needed. DI for its dependencies comes from the singletons above.
|
||||
|
||||
builder.Services.AddHostedService<TableInitializerService>(); // Initialize all tables at startup
|
||||
builder.Services.AddHostedService<StartupTelemetryService>(); // BackendStartupMs / BackendTableInitMs metrics on ApplicationStarted
|
||||
|
||||
// Data Access Layer — repository interfaces backed by Table Storage.
|
||||
// To switch to Cosmos DB: replace AddTableStorageDataAccess() with AddCosmosDataAccess().
|
||||
|
||||
@@ -464,6 +464,7 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("GET", "global/metrics/fleet-health", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/metrics/time-attribution", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/metrics/device-journeys", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/metrics/verdict-calibration", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/apps/list", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/apps/{appName}/analytics", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/apps/{appName}/sessions", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
@@ -21,8 +20,15 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly ILogger<DistressRateLimitService> _logger;
|
||||
|
||||
// Per-key locks (same proven pattern as RateLimitService)
|
||||
private readonly ConcurrentDictionary<string, object> _locks = new();
|
||||
// History + lock in one cache entry (same pattern as RateLimitService): same list ⇒ same lock.
|
||||
private sealed class RequestBucket
|
||||
{
|
||||
public readonly List<DateTime> History = new();
|
||||
public readonly object Sync = new();
|
||||
}
|
||||
|
||||
// Serializes bucket creation (GetOrCreate is not atomic); hot path is a lock-free TryGetValue.
|
||||
private readonly object _createLock = new();
|
||||
|
||||
// Layer 1: Per-IP
|
||||
private const int MaxPerIp = 5;
|
||||
@@ -91,17 +97,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// </summary>
|
||||
private bool CheckSlidingWindow(string cacheKey, int maxRequests, TimeSpan window, DateTime now)
|
||||
{
|
||||
var lockObj = _locks.GetOrAdd(cacheKey, _ => new object());
|
||||
var history = _cache.GetOrCreate(cacheKey, entry =>
|
||||
{
|
||||
entry.SlidingExpiration = window;
|
||||
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(2);
|
||||
entry.RegisterPostEvictionCallback((key, _, _, _) => _locks.TryRemove((string)key, out _));
|
||||
return new List<DateTime>();
|
||||
})!;
|
||||
var bucket = GetOrCreateBucket(cacheKey, window);
|
||||
|
||||
lock (lockObj)
|
||||
lock (bucket.Sync)
|
||||
{
|
||||
var history = bucket.History;
|
||||
|
||||
// Remove entries outside the window
|
||||
var windowStart = now.Subtract(window);
|
||||
history.RemoveAll(ts => ts < windowStart);
|
||||
@@ -113,16 +114,26 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
history.Add(now);
|
||||
|
||||
_cache.Set(cacheKey, history, new MemoryCacheEntryOptions
|
||||
{
|
||||
SlidingExpiration = window,
|
||||
AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(2)
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private RequestBucket GetOrCreateBucket(string cacheKey, TimeSpan window)
|
||||
{
|
||||
if (_cache.TryGetValue(cacheKey, out RequestBucket? existing) && existing != null)
|
||||
return existing;
|
||||
|
||||
lock (_createLock)
|
||||
{
|
||||
return _cache.GetOrCreate(cacheKey, entry =>
|
||||
{
|
||||
entry.SlidingExpiration = window;
|
||||
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(2);
|
||||
return new RequestBucket();
|
||||
})!;
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsCircuitOpen(DateTime now)
|
||||
{
|
||||
lock (_globalLock)
|
||||
|
||||
@@ -273,6 +273,34 @@ namespace AutopilotMonitor.Functions.Services
|
||||
$"device presumably sealed or powered off awaiting the end user{acct}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Self-deploying (kiosk / shared-device) profile gate. Such a profile has NO user phase:
|
||||
/// Device ESP reaching all-succeeded IS the enrollment's end, and the agent's own
|
||||
/// SelfDeploying terminal only adds a 5-min confirmation window on top of it. When the
|
||||
/// agent goes silent after that point (reboot into the kiosk autologon, powered off and
|
||||
/// boxed for the school term, WiFi never re-associating for the relaunch — tenant
|
||||
/// aebdce78, 836 sessions 2026-08-17..21), "awaiting user / Account Setup" is a factual
|
||||
/// error and the later Incomplete("Account Setup 0/5") is worse: nobody will ever sign
|
||||
/// in, the devices are finished and in service. Demands the registration-time
|
||||
/// registry-confirmed profile flag (CloudAssignedOobeConfig 0x20|0x40), Device Setup
|
||||
/// all-succeeded and no explicit failure. Shared by the sweep (stage 1 + 2), the
|
||||
/// max-lifetime ingest verdict and the retro-reclassification so one evidence rule applies.
|
||||
/// </summary>
|
||||
public static bool IsSelfDeployingProvisioned(in EspProvisioningRollup rollup, bool isSelfDeployingProfile)
|
||||
=> isSelfDeployingProfile
|
||||
&& rollup.DeviceSetupAllSucceeded
|
||||
&& !rollup.HasExplicitFailure;
|
||||
|
||||
/// <summary>
|
||||
/// Customer-facing Succeeded-reconcile reason for <see cref="IsSelfDeployingProvisioned"/>.
|
||||
/// Carries the silence-transparency clause like every other reconcile reason.
|
||||
/// </summary>
|
||||
public static string SelfDeployingReconcileReason(DateTime startedAtUtc, DateTime nowUtc, DateTime? lastEventAtUtc)
|
||||
=> AppendReconcileTiming(
|
||||
"Reconciled: self-deploying profile — Device Setup fully provisioned (this profile has no user / Account Setup phase); " +
|
||||
"agent went silent before confirming completion",
|
||||
startedAtUtc, nowUtc, lastEventAtUtc);
|
||||
|
||||
/// <summary>
|
||||
/// Decide the honest terminal (or non-terminal) state for a timed-out session. See the
|
||||
/// decision table in the design note. Returns the target status and a human-readable
|
||||
@@ -291,14 +319,16 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// </param>
|
||||
/// <param name="isPreProvisioned">Session's IsPreProvisioned flag (WhiteGlove Part-2 gate input).</param>
|
||||
/// <param name="resumedAt">Session's ResumedAt — set exclusively by the whiteglove_resumed ingest.</param>
|
||||
public static (SessionStatus Status, string Reason) ClassifyTimedOutSession(
|
||||
/// <param name="isSelfDeployingProfile">Session's IsSelfDeployingProfile flag (rule 1c gate input).</param>
|
||||
public static (SessionStatus Status, string Reason, string Rule) ClassifyTimedOutSession(
|
||||
EspProvisioningRollup rollup, DateTime startedAtUtc, DateTime nowUtc, int graceHours,
|
||||
DateTime? lastEventAtUtc = null, bool isPreProvisioned = false, DateTime? resumedAt = null)
|
||||
DateTime? lastEventAtUtc = null, bool isPreProvisioned = false, DateTime? resumedAt = null,
|
||||
bool isSelfDeployingProfile = false)
|
||||
{
|
||||
// 1. An explicit terminal failure event is a real failure (defensive — such a session
|
||||
// would normally already be terminal via ingest and not reach the sweep).
|
||||
if (rollup.HasExplicitFailure)
|
||||
return (SessionStatus.Failed, "Enrollment reported an explicit failure before timeout");
|
||||
return (SessionStatus.Failed, "Enrollment reported an explicit failure before timeout", ClassifierRules.R1ExplicitFailure);
|
||||
|
||||
// 1b. WhiteGlove Part 2 with zero user evidence since the resume: the device is parked
|
||||
// between technician and end user (sealed box / powered off at the logon screen).
|
||||
@@ -311,13 +341,22 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (IsWhiteGloveAwaitingUser(rollup, isPreProvisioned, resumedAt))
|
||||
{
|
||||
if ((nowUtc - startedAtUtc).TotalHours < graceHours)
|
||||
return (SessionStatus.AwaitingUser, WhiteGloveAwaitingUserReason(rollup));
|
||||
return (SessionStatus.AwaitingUser, WhiteGloveAwaitingUserReason(rollup), ClassifierRules.R1bWhiteGloveAwaiting);
|
||||
|
||||
return (SessionStatus.Succeeded, AppendReconcileTiming(
|
||||
$"Reconciled at timeout: pre-provisioning (WhiteGlove Part 1) completed — no user sign-in observed within {graceHours}h grace",
|
||||
startedAtUtc, nowUtc, lastEventAtUtc));
|
||||
startedAtUtc, nowUtc, lastEventAtUtc), ClassifierRules.R1bWhiteGloveSucceeded);
|
||||
}
|
||||
|
||||
// 1c. Self-deploying profile with Device Setup fully provisioned: there is no user phase
|
||||
// to wait for, so neither AwaitingUser nor the grace window apply — the device is
|
||||
// done (see IsSelfDeployingProvisioned). Runs before rule 2 so the reason names the
|
||||
// profile instead of claiming "Account Setup completed"; real user evidence cannot
|
||||
// contradict it (a kiosk autologon is not a user) and an explicit failure already
|
||||
// returned in rule 1.
|
||||
if (IsSelfDeployingProvisioned(rollup, isSelfDeployingProfile))
|
||||
return (SessionStatus.Succeeded, SelfDeployingReconcileReason(startedAtUtc, nowUtc, lastEventAtUtc), ClassifierRules.R1cSelfDeploying);
|
||||
|
||||
// 2. Account Setup fully succeeded (or a terminal completion event) → reconcile to success.
|
||||
// Once Part 2 has resumed, the Part-1 whiteglove_complete in the same stream no longer
|
||||
// proves the session's outcome — only a real enrollment_complete does. Without this,
|
||||
@@ -329,14 +368,15 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (rollup.AccountSetupAllSucceeded || terminalCompleteProvesOutcome)
|
||||
return (SessionStatus.Succeeded, AppendReconcileTiming(
|
||||
"Reconciled at timeout: Account Setup completed (all subcategories succeeded / enrollment_complete observed)",
|
||||
startedAtUtc, nowUtc, lastEventAtUtc));
|
||||
startedAtUtc, nowUtc, lastEventAtUtc), ClassifierRules.R2AccountSetupComplete);
|
||||
|
||||
// 3. The agent's absolute-age emergency break fired — the agent has cleaned up and exited, so
|
||||
// nothing more will ever arrive for this session. Terminalize NOW (skip the AwaitingUser grace):
|
||||
// it did not complete (2) and did not explicitly fail (1), so the honest verdict is Incomplete.
|
||||
if (rollup.HasAgentEmergencyBreak)
|
||||
return (SessionStatus.Incomplete,
|
||||
"Agent emergency break fired (absolute session-age cap) — agent gone without completion");
|
||||
"Agent emergency break fired (absolute session-age cap) — agent gone without completion",
|
||||
ClassifierRules.R3EmergencyBreak);
|
||||
|
||||
// 4. The user demonstrably finished setup: a real-user desktop was observed AND
|
||||
// Windows Hello reached a positive terminal (provisioned or skipped). Those are
|
||||
@@ -371,7 +411,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
: "agent went silent before reporting completion";
|
||||
return (SessionStatus.Succeeded, AppendReconcileTiming(
|
||||
$"Reconciled at timeout: user completed setup ({evidence}) — {detail}",
|
||||
startedAtUtc, nowUtc, lastEventAtUtc));
|
||||
startedAtUtc, nowUtc, lastEventAtUtc), ClassifierRules.R4DesktopHello);
|
||||
}
|
||||
|
||||
// 5. Device Setup fully provisioned (device is AADJ + MDM-enrolled), user phase pending.
|
||||
@@ -384,7 +424,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
? $" (Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal})"
|
||||
: " (Account Setup not yet started)";
|
||||
return (SessionStatus.AwaitingUser,
|
||||
$"Device Setup completed; awaiting user / Account Setup phase — agent silent, within {graceHours}h grace{acct}");
|
||||
$"Device Setup completed; awaiting user / Account Setup phase — agent silent, within {graceHours}h grace{acct}",
|
||||
ClassifierRules.R5DeviceSetupAwaiting);
|
||||
}
|
||||
|
||||
// "0/0" would suggest a parsed rollup that never existed — when no Account Setup
|
||||
@@ -393,12 +434,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
? $"last Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal}"
|
||||
: "Account Setup progress never observed";
|
||||
return (SessionStatus.Incomplete,
|
||||
$"No completion signal within {graceHours}h grace after Device Setup completed ({acctDetail})");
|
||||
$"No completion signal within {graceHours}h grace after Device Setup completed ({acctDetail})",
|
||||
ClassifierRules.R5DeviceSetupIncomplete);
|
||||
}
|
||||
|
||||
// 6. Silent before Device Setup completed, with no explicit failure → unknown, not a failure.
|
||||
return (SessionStatus.Incomplete,
|
||||
"No Device Setup completion or explicit failure signal observed before timeout");
|
||||
"No Device Setup completion or explicit failure signal observed before timeout",
|
||||
ClassifierRules.R6Fallthrough);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+26
-15
@@ -286,9 +286,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
var effectiveStart = session?.ResumedAt ?? session?.StartedAt ?? triggerEvent.Timestamp;
|
||||
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents);
|
||||
var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (targetStatus, reason, rule) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session?.LastEventAt ?? c.LatestEventTimestamp,
|
||||
isPreProvisioned: session?.IsPreProvisioned == true, resumedAt: session?.ResumedAt);
|
||||
isPreProvisioned: session?.IsPreProvisioned == true, resumedAt: session?.ResumedAt,
|
||||
isSelfDeployingProfile: session?.IsSelfDeployingProfile == true);
|
||||
|
||||
// Keep the max-lifetime trigger visible: the Succeeded reasons already carry their own
|
||||
// silence-transparency clause, everything else gets the watchdog context appended.
|
||||
@@ -314,7 +315,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, targetStatus, triggerEvent.Phase, reason,
|
||||
request.TenantId, request.SessionId, targetStatus, VerdictPaths.Compose(VerdictPaths.OriginMaxLifetime, rule),
|
||||
triggerEvent.Phase, reason,
|
||||
completedAt: isTerminalNonSuccess ? triggerEvent.Timestamp : (DateTime?)null,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
failureSource: isTerminalNonSuccess ? "max_lifetime_watchdog" : null,
|
||||
@@ -356,15 +358,17 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var effectiveStart = session.ResumedAt ?? session.StartedAt;
|
||||
var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (targetStatus, reason, rule) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt ?? c.LatestEventTimestamp,
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt);
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt,
|
||||
isSelfDeployingProfile: session.IsSelfDeployingProfile);
|
||||
|
||||
if (targetStatus != SessionStatus.Succeeded)
|
||||
return;
|
||||
|
||||
var healed = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Succeeded,
|
||||
VerdictPaths.Compose(VerdictPaths.OriginLateReconcile, rule),
|
||||
failureReason: reason,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
if (healed)
|
||||
@@ -404,7 +408,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return false;
|
||||
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.AwaitingUser,
|
||||
request.TenantId, request.SessionId, SessionStatus.AwaitingUser, VerdictPaths.IngestWhiteGloveAwaiting,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
failureReason: EnrollmentTimeoutClassifier.WhiteGloveAwaitingUserReason(rollup));
|
||||
if (transitioned)
|
||||
@@ -463,7 +467,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
? c.CompletionEvent.Data["completionSource"]?.ToString() : null;
|
||||
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Succeeded, c.CompletionEvent.Phase,
|
||||
request.TenantId, request.SessionId, SessionStatus.Succeeded,
|
||||
espSoftFailure ? VerdictPaths.AgentCompleteSoft : VerdictPaths.AgentComplete,
|
||||
c.CompletionEvent.Phase,
|
||||
completedAt: c.CompletionEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
espSoftFailure: espSoftFailure, completionSource: completionSource);
|
||||
@@ -490,7 +496,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
: c.FailureEvent.Message;
|
||||
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.FailureEvent.Phase, failureReason,
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, VerdictPaths.AgentFailed,
|
||||
c.FailureEvent.Phase, failureReason,
|
||||
completedAt: c.FailureEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned);
|
||||
@@ -500,7 +507,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
failureReason = c.EspFailureEvent.Message ?? "ESP failure (backend fallback)";
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.EspFailureEvent.Phase, failureReason,
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, VerdictPaths.AgentEspFailureFallback,
|
||||
c.EspFailureEvent.Phase, failureReason,
|
||||
completedAt: c.EspFailureEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed via esp_failure fallback - {FailureReason} (transitioned={Transitioned})",
|
||||
@@ -509,7 +517,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
else if (c.GatherCompletionEvent != null)
|
||||
{
|
||||
await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Succeeded, c.GatherCompletionEvent.Phase,
|
||||
request.TenantId, request.SessionId, SessionStatus.Succeeded, VerdictPaths.AgentGatherComplete,
|
||||
c.GatherCompletionEvent.Phase,
|
||||
completedAt: c.GatherCompletionEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
_logger.LogInformation("{SessionPrefix} Status: Succeeded (gather_rules)", sessionPrefix);
|
||||
@@ -517,7 +526,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
else if (c.WhiteGloveEvent != null)
|
||||
{
|
||||
whiteGloveStatusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Pending, EnrollmentPhase.AppsDevice,
|
||||
request.TenantId, request.SessionId, SessionStatus.Pending, VerdictPaths.AgentWhiteGlovePending,
|
||||
EnrollmentPhase.AppsDevice,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
isPreProvisioned: true, isUserDriven: false);
|
||||
|
||||
@@ -538,7 +548,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_logger.LogWarning("{SessionPrefix} WhiteGlove UpdateSessionStatusAsync failed, attempting unconditional fallback for IsPreProvisioned + Status", sessionPrefix);
|
||||
try
|
||||
{
|
||||
await _sessionRepo.SetSessionPreProvisionedAsync(request.TenantId, request.SessionId, true, SessionStatus.Pending, isUserDriven: false);
|
||||
await _sessionRepo.SetSessionPreProvisionedAsync(request.TenantId, request.SessionId, true, SessionStatus.Pending, isUserDriven: false, verdictPath: VerdictPaths.AgentWhiteGlovePending);
|
||||
whiteGloveStatusTransitioned = true;
|
||||
_logger.LogInformation("{SessionPrefix} WhiteGlove fallback succeeded: IsPreProvisioned + Status=Pending set via unconditional merge", sessionPrefix);
|
||||
}
|
||||
@@ -578,7 +588,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (currentSession?.Status == SessionStatus.Pending)
|
||||
{
|
||||
await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.InProgress, c.WhiteGloveResumedEvent.Phase,
|
||||
request.TenantId, request.SessionId, SessionStatus.InProgress, VerdictPaths.AgentWhiteGloveResumed,
|
||||
c.WhiteGloveResumedEvent.Phase,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
isUserDriven: true, resumedAt: c.WhiteGloveResumedEvent.Timestamp);
|
||||
_logger.LogInformation("{SessionPrefix} Status: InProgress (WhiteGlove Part 2 resumed, IsUserDriven=true)", sessionPrefix);
|
||||
@@ -624,7 +635,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
var stalledReason = "Agent reported stall after 60min without progress (stall_probe)";
|
||||
var stalledTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Stalled,
|
||||
request.TenantId, request.SessionId, SessionStatus.Stalled, VerdictPaths.AgentStallProbe,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
stalledAt: c.SessionStalledEvent.Timestamp, failureReason: stalledReason);
|
||||
if (stalledTransitioned)
|
||||
@@ -647,7 +658,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (currentStatus == SessionStatus.Stalled)
|
||||
{
|
||||
var healed = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.InProgress,
|
||||
request.TenantId, request.SessionId, SessionStatus.InProgress, VerdictPaths.AgentStallHeal,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
clearStalledAt: true, clearFailureReason: true);
|
||||
if (healed)
|
||||
|
||||
@@ -18,6 +18,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// registered a LATER session (Part 2 under a fresh session id, or re-provisioning) are
|
||||
/// resolved as Incomplete("Superseded by ..."). Forward-looking coverage lives in the
|
||||
/// registration supersede pass; this mode clears the existing backlog.</item>
|
||||
/// <item><b>self_deploying_silent</b> — self-deploying-profile rows parked as
|
||||
/// Incomplete / AwaitingUser / Stalled by the pre-2026-08-23 classifier (which had no
|
||||
/// notion of a profile without a user phase) are re-run through the classifier; only a
|
||||
/// Succeeded verdict (<see cref="EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned"/>)
|
||||
/// is applied, everything else keeps its verdict.</item>
|
||||
/// </list>
|
||||
/// Hand-marked sessions (AdminMarkedAction) are never touched.
|
||||
/// </summary>
|
||||
@@ -30,6 +35,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
public const string ModeLegacyTimeouts = "legacy_timeouts";
|
||||
public const string ModePendingOrphans = "pending_orphans";
|
||||
public const string ModeSelfDeployingSilent = "self_deploying_silent";
|
||||
private const int MaxSamplesInResponse = 100;
|
||||
|
||||
public LegacyReclassificationService(
|
||||
@@ -108,9 +114,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
var events = await _sessionRepo.GetSessionEventsAsync(tenantId, session.SessionId, maxResults: 1000);
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var effectiveStart = session.ResumedAt ?? session.StartedAt;
|
||||
var (target, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (target, reason, rule) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt,
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt);
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt,
|
||||
isSelfDeployingProfile: session.IsSelfDeployingProfile);
|
||||
|
||||
if (target == SessionStatus.Failed)
|
||||
{
|
||||
@@ -138,7 +145,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
tenantId, session.SessionId, target,
|
||||
tenantId, session.SessionId, target, VerdictPaths.Compose(VerdictPaths.OriginRetro, rule),
|
||||
failureReason: reason,
|
||||
failureSource: target == SessionStatus.Incomplete ? "legacy_timeout_retro_reconcile" : null,
|
||||
allowTerminalReclassification: true);
|
||||
@@ -165,6 +172,99 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<ReclassificationResult> ReconcileSelfDeployingSilentAsync(
|
||||
string? tenantIdScope, bool dryRun, int maxSessions, string triggeredBy)
|
||||
{
|
||||
var result = new ReclassificationResult { Mode = ModeSelfDeployingSilent, DryRun = dryRun };
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
foreach (var tenantId in await ResolveTenantsAsync(tenantIdScope))
|
||||
{
|
||||
if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; }
|
||||
result.TenantsExamined++;
|
||||
|
||||
int? tenantGraceHours = null, absoluteMaxHours = null;
|
||||
try
|
||||
{
|
||||
var config = await _configService.GetConfigurationAsync(tenantId);
|
||||
tenantGraceHours = config?.SessionGraceHours;
|
||||
absoluteMaxHours = config?.AbsoluteMaxSessionHours;
|
||||
}
|
||||
catch { /* defaults */ }
|
||||
var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours);
|
||||
|
||||
var candidates = await _maintenanceRepo.GetSelfDeployingSilentSessionsAsync(
|
||||
tenantId, maxSessions - result.SessionsExamined + 1);
|
||||
if (candidates.Count > maxSessions - result.SessionsExamined)
|
||||
{
|
||||
result.CapReached = true;
|
||||
candidates = candidates.Take(maxSessions - result.SessionsExamined).ToList();
|
||||
}
|
||||
|
||||
foreach (var session in candidates)
|
||||
{
|
||||
result.SessionsExamined++;
|
||||
try
|
||||
{
|
||||
if (!string.IsNullOrEmpty(session.AdminMarkedAction))
|
||||
{
|
||||
result.Skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var events = await _sessionRepo.GetSessionEventsAsync(tenantId, session.SessionId, maxResults: 1000);
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var effectiveStart = session.ResumedAt ?? session.StartedAt;
|
||||
var (target, reason, _) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt,
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt,
|
||||
isSelfDeployingProfile: session.IsSelfDeployingProfile);
|
||||
|
||||
// Heal-only: a verdict other than Succeeded means the device never reached
|
||||
// Device ESP all-succeeded (or failed explicitly) — the existing label stands.
|
||||
if (target != SessionStatus.Succeeded)
|
||||
{
|
||||
result.Skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
RecordSample(result, tenantId, session, target, reason, oldStatus: session.Status.ToString());
|
||||
|
||||
if (dryRun)
|
||||
{
|
||||
result.WouldChange++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Succeeded may upgrade Incomplete / AwaitingUser / Stalled through the normal
|
||||
// reconcile gate (no terminal-reclassification override needed); the storage
|
||||
// layer additionally refuses admin-marked rows.
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
tenantId, session.SessionId, SessionStatus.Succeeded, VerdictPaths.RetroSelfDeployingReconcile,
|
||||
failureReason: reason);
|
||||
if (transitioned)
|
||||
{
|
||||
result.Changed++;
|
||||
result.ToSucceeded++;
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Skipped++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors++;
|
||||
_logger.LogWarning(ex, "Self-deploying retro-reconcile failed for session {SessionId} (tenant {TenantId})",
|
||||
session.SessionId, tenantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await WriteAuditAsync(result, tenantIdScope, triggeredBy);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<ReclassificationResult> ResolvePendingOrphansAsync(
|
||||
string? tenantIdScope, bool dryRun, int maxSessions, string triggeredBy)
|
||||
{
|
||||
@@ -214,7 +314,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
// Pending is non-terminal — the standard transition guard covers this write.
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
tenantId, pending.SessionId, SessionStatus.Incomplete,
|
||||
tenantId, pending.SessionId, SessionStatus.Incomplete, VerdictPaths.RetroSuperseded,
|
||||
failureReason: reason, failureSource: "superseded_by_reregistration");
|
||||
if (transitioned) { result.Changed++; result.ToIncomplete++; }
|
||||
else result.Skipped++;
|
||||
|
||||
@@ -289,14 +289,13 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// Sessions with a deletion cascade in flight are skipped so the sweep cannot write a
|
||||
/// breakdown row after the session's deletion manifest was snapshotted.
|
||||
/// </summary>
|
||||
private async Task SweepTimeAttributionAsync()
|
||||
private async Task SweepTimeAttributionAsync(IReadOnlyList<SessionSummary> sweepWindow)
|
||||
{
|
||||
try
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var today = DateTime.UtcNow.Date;
|
||||
var sessions = await _maintenanceRepo.GetSessionsByDateRangeAsync(
|
||||
today.AddDays(-TimeAttributionSweepDays), today.AddDays(1));
|
||||
var sessions = SliceSweepWindow(sweepWindow, today.AddDays(-TimeAttributionSweepDays));
|
||||
|
||||
var pairs = new List<(SessionSummary Session, SessionTimeBreakdown Breakdown)>();
|
||||
var missing = new List<SessionSummary>();
|
||||
@@ -1102,6 +1101,19 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to cleanup old device-journey aggregates");
|
||||
}
|
||||
|
||||
// Verdict-calibration daily aggregates: same 180d window (regenerable; the radar
|
||||
// needs 35d, the matrix serves up to 180d).
|
||||
try
|
||||
{
|
||||
var deleted = await _metricsRepo.DeleteVerdictCalibrationAggregatesOlderThanAsync(now.AddDays(-usageMetricsRetentionDays));
|
||||
if (deleted > 0)
|
||||
_logger.LogInformation("Verdict-calibration aggregate cleanup: deleted {Count} rows older than {Days} days", deleted, usageMetricsRetentionDays);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to cleanup old verdict-calibration aggregates");
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Detects and cleans up orphaned events — events stored in the Events table
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// per window device per tick — same order as the F1 sweep; negligible at current fleet
|
||||
/// size (the scaling lever is gating the per-device RMW on new terminal sessions).
|
||||
/// </summary>
|
||||
private async Task SweepDeviceJourneysAsync()
|
||||
private async Task SweepDeviceJourneysAsync(IReadOnlyList<SessionSummary> sweepWindow)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -84,7 +84,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
// ---- Phase 2: merge the window's terminal sessions into their device chains. ----
|
||||
var sessions = await _maintenanceRepo.GetSessionsByDateRangeAsync(windowStart, windowEnd);
|
||||
var sessions = SliceSweepWindow(sweepWindow, windowStart);
|
||||
var excludedSessions = new List<SessionSummary>();
|
||||
var byDevice = new Dictionary<(string TenantId, string SerialKey), List<SessionSummary>>();
|
||||
|
||||
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Partial: verdict-calibration daily aggregates (docs/backend/verdict-calibration.md) — the
|
||||
/// operator thermometer for the rule classifier: which verdict path produced how many
|
||||
/// sessions per day, how many of those were later overridden (admin / late completion /
|
||||
/// other), and how many devices re-enrolled within 7 days.
|
||||
/// </summary>
|
||||
public partial class MaintenanceService
|
||||
{
|
||||
/// <summary>App setting kill switch: "true" skips the sweep. Fail-open — the sweep only writes its own regenerable table.</summary>
|
||||
internal const string VerdictCalibrationKillSwitchSetting = "VerdictCalibrationSweepDisabled";
|
||||
|
||||
/// <summary>Rolling window: sessions started in the last 30 days (same as the F1/F2 sweeps).</summary>
|
||||
private const int VerdictCalibrationSweepDays = 30;
|
||||
|
||||
/// <summary>Re-enrollment proxy horizon: another terminal session of the same device within this many days.</summary>
|
||||
internal const int VerdictCalibrationReEnrollDays = 7;
|
||||
|
||||
/// <summary>
|
||||
/// Recomputes the verdict-calibration rows for the rolling window — idempotent per
|
||||
/// (tenant × StartedAt date) plus "global" mirror rows. Slices the tick's shared window
|
||||
/// scan (no own drain) and reads each window tenant's device histories (one partition
|
||||
/// scan per tenant) for the re-enrollment proxy. Stale date rows a run did not regenerate
|
||||
/// are deleted so window sums never serve ghost counts.
|
||||
/// </summary>
|
||||
private async Task SweepVerdictCalibrationAsync(IReadOnlyList<SessionSummary> sweepWindow)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.Equals(_configuration[VerdictCalibrationKillSwitchSetting], "true", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("Verdict-calibration sweep skipped ({Setting}=true)", VerdictCalibrationKillSwitchSetting);
|
||||
return;
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var now = DateTime.UtcNow;
|
||||
var today = now.Date;
|
||||
var windowStart = today.AddDays(-VerdictCalibrationSweepDays);
|
||||
var windowEnd = today.AddDays(1);
|
||||
|
||||
var sessions = SliceSweepWindow(sweepWindow, windowStart);
|
||||
|
||||
var historiesByTenant = new Dictionary<string, List<DeviceHistory>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var tenantId in sessions.Select(s => s.TenantId).Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
historiesByTenant[tenantId] = await _metricsRepo.GetDeviceHistoriesByTenantAsync(tenantId);
|
||||
}
|
||||
|
||||
var aggregates = BuildVerdictCalibrationAggregates(sessions, historiesByTenant, now);
|
||||
var saved = 0;
|
||||
foreach (var aggregate in aggregates)
|
||||
{
|
||||
if (await _metricsRepo.SaveVerdictCalibrationAggregateAsync(aggregate))
|
||||
saved++;
|
||||
}
|
||||
|
||||
var generatedKeys = new HashSet<(string TenantId, string Date)>(aggregates.Select(a => (a.TenantId, a.Date)));
|
||||
var removedStale = 0;
|
||||
foreach (var partition in aggregates.Select(a => a.TenantId).Distinct().ToList())
|
||||
{
|
||||
var existingRows = await _metricsRepo.GetVerdictCalibrationAggregatesAsync(partition, windowStart, windowEnd.AddDays(-1));
|
||||
foreach (var row in existingRows)
|
||||
{
|
||||
if (generatedKeys.Contains((partition, row.Date))) continue;
|
||||
await _metricsRepo.DeleteVerdictCalibrationAggregateAsync(partition, row.Date);
|
||||
removedStale++;
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation(
|
||||
"Verdict-calibration sweep: {Sessions} window sessions across {Tenants} tenants, {Rows} rows written, {Removed} stale rows removed in {Ms}ms",
|
||||
sessions.Count, historiesByTenant.Count, saved, removedStale, sw.ElapsedMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Verdict-calibration sweep failed (non-fatal)");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure aggregation core behind <see cref="SweepVerdictCalibrationAsync"/> — internal
|
||||
/// static so the contract is pinned by unit tests. Per session: the current
|
||||
/// (VerdictPath, Status) bucket gets <c>Count</c> (+ <c>DerivedCount</c> when the path was
|
||||
/// derived read-side), terminal sessions old enough get <c>Eligible7d</c> and, if the
|
||||
/// device's history chain shows another terminal session starting within 7 days of this
|
||||
/// one's end, <c>ReEnrolled7d</c>; a session carrying a prior verdict additionally
|
||||
/// increments the PRIOR bucket's <c>Overridden*</c> counter. Sessions inside a deletion
|
||||
/// cascade are skipped. Rows are emitted per (tenant, StartedAt date) and mirrored into
|
||||
/// the "global" partition; buckets are sorted for deterministic output.
|
||||
/// </summary>
|
||||
internal static List<VerdictCalibrationDailyAggregate> BuildVerdictCalibrationAggregates(
|
||||
IReadOnlyList<SessionSummary> sessions,
|
||||
IReadOnlyDictionary<string, List<DeviceHistory>> historiesByTenant,
|
||||
DateTime nowUtc)
|
||||
{
|
||||
var nextStartBySession = BuildNextStartLookup(historiesByTenant);
|
||||
var eligibleCutoff = nowUtc.AddDays(-VerdictCalibrationReEnrollDays);
|
||||
|
||||
var rows = new Dictionary<(string TenantId, string Date), RowAccumulator>();
|
||||
|
||||
RowAccumulator Row(string tenantId, string date)
|
||||
{
|
||||
var key = (tenantId, date);
|
||||
if (!rows.TryGetValue(key, out var acc))
|
||||
{
|
||||
acc = new RowAccumulator();
|
||||
rows[key] = acc;
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
foreach (var session in sessions)
|
||||
{
|
||||
if (session == null) continue;
|
||||
if (!string.IsNullOrEmpty(session.DeletionState) && session.DeletionState != "None")
|
||||
continue;
|
||||
|
||||
var date = session.StartedAt.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
|
||||
var (path, derived) = VerdictPathDerivation.Derive(session);
|
||||
var statusName = session.Status.ToString();
|
||||
var isTerminal = DeviceJourneyCalculator.IsTerminal(session.Status);
|
||||
|
||||
var eligible = false;
|
||||
var reEnrolled = false;
|
||||
if (isTerminal)
|
||||
{
|
||||
var end = session.CompletedAt ?? session.LastEventAt ?? session.StartedAt;
|
||||
eligible = end <= eligibleCutoff;
|
||||
if (eligible
|
||||
&& nextStartBySession.TryGetValue((session.TenantId, session.SessionId), out var next)
|
||||
&& next.NextStart.HasValue
|
||||
&& (next.NextStart.Value - next.End) < TimeSpan.FromDays(VerdictCalibrationReEnrollDays))
|
||||
{
|
||||
reEnrolled = true;
|
||||
}
|
||||
}
|
||||
|
||||
var overridden = ClassifyOverride(session);
|
||||
|
||||
foreach (var partition in new[] { session.TenantId, "global" })
|
||||
{
|
||||
var acc = Row(partition, date);
|
||||
acc.SessionCount++;
|
||||
if (isTerminal) acc.TerminalSessionCount++;
|
||||
|
||||
var bucket = acc.Bucket(path, statusName);
|
||||
bucket.Count++;
|
||||
if (derived) bucket.DerivedCount++;
|
||||
if (eligible) bucket.Eligible7d++;
|
||||
if (reEnrolled) bucket.ReEnrolled7d++;
|
||||
|
||||
if (overridden.HasValue)
|
||||
{
|
||||
var prior = acc.Bucket(session.PriorVerdictPath!, session.PriorStatus!);
|
||||
switch (overridden.Value)
|
||||
{
|
||||
case OverrideKind.Admin: prior.OverriddenByAdmin++; break;
|
||||
case OverrideKind.LateCompletion: prior.OverriddenByLateCompletion++; break;
|
||||
default: prior.OverriddenOther++; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rows
|
||||
.OrderBy(r => r.Key.TenantId, StringComparer.Ordinal)
|
||||
.ThenBy(r => r.Key.Date, StringComparer.Ordinal)
|
||||
.Select(r => new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = r.Key.TenantId,
|
||||
Date = r.Key.Date,
|
||||
Version = VerdictCalibrationDailyAggregate.CurrentVersion,
|
||||
SessionCount = r.Value.SessionCount,
|
||||
TerminalSessionCount = r.Value.TerminalSessionCount,
|
||||
Buckets = r.Value.Buckets.Values
|
||||
.OrderBy(b => b.VerdictPath, StringComparer.Ordinal)
|
||||
.ThenBy(b => b.Status, StringComparer.Ordinal)
|
||||
.ToList(),
|
||||
ComputedAt = nowUtc,
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private enum OverrideKind { Admin, LateCompletion, Other }
|
||||
|
||||
/// <summary>
|
||||
/// A session whose current status overrode a prior verdict names the writer that did it:
|
||||
/// an admin mark (AdminMarkedAction set), a late agent completion (now Succeeded without
|
||||
/// admin involvement), or anything else (retro-reclassification, grace expiry, supersede).
|
||||
/// </summary>
|
||||
private static OverrideKind? ClassifyOverride(SessionSummary session)
|
||||
{
|
||||
if (string.IsNullOrEmpty(session.PriorVerdictPath) || string.IsNullOrEmpty(session.PriorStatus))
|
||||
return null;
|
||||
if (!string.IsNullOrEmpty(session.AdminMarkedAction))
|
||||
return OverrideKind.Admin;
|
||||
if (session.Status == SessionStatus.Succeeded)
|
||||
return OverrideKind.LateCompletion;
|
||||
return OverrideKind.Other;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// (tenant, sessionId) → this session's chain end and the next chain entry's StartedAt —
|
||||
/// the re-enrollment proxy source. Chains are ascending by StartedAt and capped at 20, so
|
||||
/// a window session older than the cap simply has no entry (counts as not re-enrolled
|
||||
/// only when eligible — the cap is a known, small bias toward under-counting).
|
||||
/// </summary>
|
||||
internal static Dictionary<(string TenantId, string SessionId), (DateTime End, DateTime? NextStart)> BuildNextStartLookup(
|
||||
IReadOnlyDictionary<string, List<DeviceHistory>> historiesByTenant)
|
||||
{
|
||||
var lookup = new Dictionary<(string, string), (DateTime, DateTime?)>();
|
||||
foreach (var pair in historiesByTenant)
|
||||
{
|
||||
foreach (var history in pair.Value)
|
||||
{
|
||||
var chain = history.Chain;
|
||||
for (var i = 0; i < chain.Count; i++)
|
||||
{
|
||||
var end = chain[i].CompletedAt ?? chain[i].StartedAt;
|
||||
DateTime? nextStart = i + 1 < chain.Count ? chain[i + 1].StartedAt : null;
|
||||
lookup[(pair.Key, chain[i].SessionId)] = (end, nextStart);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lookup;
|
||||
}
|
||||
|
||||
private sealed class RowAccumulator
|
||||
{
|
||||
public int SessionCount;
|
||||
public int TerminalSessionCount;
|
||||
public readonly Dictionary<(string Path, string Status), VerdictCalibrationBucket> Buckets = new();
|
||||
|
||||
public VerdictCalibrationBucket Bucket(string path, string status)
|
||||
{
|
||||
var key = (path, status);
|
||||
if (!Buckets.TryGetValue(key, out var bucket))
|
||||
{
|
||||
bucket = new VerdictCalibrationBucket { VerdictPath = path, Status = status };
|
||||
Buckets[key] = bucket;
|
||||
}
|
||||
return bucket;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Partial: verdict-calibration drift radar (docs/backend/verdict-calibration.md). Same
|
||||
/// episode/tracker pattern as the rule-regression radar, over the verdict-calibration daily
|
||||
/// rows (<see cref="VerdictCalibrationRadar"/>). Operator-only: fires a VerdictCalibrationDrift
|
||||
/// ops event once per episode and surfaces the episode in the calibration endpoint's
|
||||
/// <c>alerts[]</c>; there is deliberately no tenant bell.
|
||||
/// </summary>
|
||||
public partial class MaintenanceService
|
||||
{
|
||||
/// <summary>App setting kill switch: "true" skips the radar. Fail-open — it only notifies.</summary>
|
||||
internal const string VerdictCalibrationRadarKillSwitchSetting = "VerdictCalibrationRadarDisabled";
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates every partition that has calibration rows in the horizon — each tenant plus
|
||||
/// the "global" mirror (a platform-wide drift is its own signal; per-tenant alerts catch
|
||||
/// the local ones). Anchored on <paramref name="targetDate"/> (timer path: yesterday —
|
||||
/// whole days only). Idempotent per anchor via the tracker dedup; fail-soft per partition.
|
||||
/// </summary>
|
||||
private async Task RunVerdictCalibrationRadarAsync(DateTime targetDate, IReadOnlyList<SessionSummary> sweepWindow)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.Equals(_configuration[VerdictCalibrationRadarKillSwitchSetting], "true", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("Verdict-calibration radar skipped ({Setting}=true)", VerdictCalibrationRadarKillSwitchSetting);
|
||||
return;
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var horizonStart = targetDate.Date.AddDays(-(VerdictCalibrationRadar.WindowDays - 1 + VerdictCalibrationRadar.BaselineDays));
|
||||
|
||||
// Tenants are discovered from the tick's shared window scan (the calibration
|
||||
// sweep wrote a row per tenant it saw) — no own drain.
|
||||
var tenantIds = sweepWindow
|
||||
.Select(s => s.TenantId)
|
||||
.Where(t => !string.IsNullOrEmpty(t))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
tenantIds.Add("global");
|
||||
|
||||
int fired = 0, refreshed = 0, rearmed = 0;
|
||||
foreach (var partition in tenantIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
var rows = await _metricsRepo.GetVerdictCalibrationAggregatesAsync(partition, horizonStart, targetDate.Date);
|
||||
if (rows.Count == 0) continue;
|
||||
var (f, r, a) = await EvaluateVerdictCalibrationPartitionAsync(partition, rows, targetDate);
|
||||
fired += f; refreshed += r; rearmed += a;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Verdict-calibration radar failed for partition {Partition} (non-fatal)", partition);
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation(
|
||||
"Verdict-calibration radar: {Partitions} partitions, {Fired} fired, {Refreshed} refreshed, {Rearmed} re-armed in {Ms}ms (anchor {Anchor})",
|
||||
tenantIds.Count, fired, refreshed, rearmed, sw.ElapsedMilliseconds, targetDate.ToString("yyyy-MM-dd"));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Verdict-calibration radar failed (non-fatal)");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(int Fired, int Refreshed, int Rearmed)> EvaluateVerdictCalibrationPartitionAsync(
|
||||
string partition, List<VerdictCalibrationDailyAggregate> rows, DateTime targetDate)
|
||||
{
|
||||
var findings = VerdictCalibrationRadar.Evaluate(rows, targetDate);
|
||||
var active = await _hardwareRejectionTracker.GetVerdictCalibrationAlertsAsync(partition);
|
||||
var activeByKey = active.ToDictionary(a => AlertKey(a.Kind, a.VerdictPath, a.Status), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
int fired = 0, refreshed = 0, rearmed = 0;
|
||||
var firedKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var finding in findings)
|
||||
{
|
||||
var key = AlertKey(finding.Kind, finding.VerdictPath, finding.Status);
|
||||
firedKeys.Add(key);
|
||||
|
||||
if (activeByKey.TryGetValue(key, out var existing))
|
||||
{
|
||||
await _hardwareRejectionTracker.RefreshVerdictCalibrationAlertAsync(
|
||||
partition, BuildVerdictAlert(partition, finding, existing.Dimension, existing.FirstNotifiedAt));
|
||||
refreshed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var dimension = await TryCorrelateVerdictDimensionsAsync(partition, finding);
|
||||
var alert = BuildVerdictAlert(partition, finding, dimension, DateTime.UtcNow);
|
||||
if (!await _hardwareRejectionTracker.TryRegisterVerdictCalibrationAlertAsync(partition, alert))
|
||||
continue;
|
||||
|
||||
fired++;
|
||||
await _opsEventService.RecordVerdictCalibrationDriftAsync(
|
||||
partition, finding.Kind, finding.VerdictPath, finding.Status,
|
||||
finding.WindowHitCount, finding.WindowSessionCount, finding.WindowRatePct,
|
||||
finding.BaselineHitCount, finding.BaselineSessionCount, finding.BaselineRatePct,
|
||||
finding.Lift, DescribeDimension(dimension));
|
||||
}
|
||||
|
||||
foreach (var alert in active)
|
||||
{
|
||||
if (firedKeys.Contains(AlertKey(alert.Kind, alert.VerdictPath, alert.Status)))
|
||||
continue;
|
||||
|
||||
if (VerdictCalibrationRadar.ShouldReArm(alert, rows, targetDate))
|
||||
{
|
||||
await _hardwareRejectionTracker.DeleteVerdictCalibrationAlertAsync(partition, alert.Kind, alert.VerdictPath, alert.Status);
|
||||
rearmed++;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Elevated but no longer fully separated: keep the episode, refresh the numbers.
|
||||
var (wh, ws, bh, bs) = VerdictCalibrationRadar.CurrentSums(alert, rows, targetDate);
|
||||
alert.WindowHitCount = wh;
|
||||
alert.WindowSessionCount = ws;
|
||||
alert.BaselineHitCount = bh;
|
||||
alert.BaselineSessionCount = bs;
|
||||
alert.WindowRatePct = ws > 0 ? Math.Round(100.0 * wh / ws, 1) : 0;
|
||||
alert.BaselineRatePct = bs > 0 ? Math.Round(100.0 * bh / bs, 1) : 0;
|
||||
alert.Lift = alert.Kind == VerdictCalibrationAlertKinds.EvidenceGap || bh == 0 || bs == 0 || ws == 0
|
||||
? null
|
||||
: Math.Round(((double)wh / ws) / ((double)bh / bs), 1);
|
||||
alert.WindowStartDate = targetDate.Date.AddDays(-(VerdictCalibrationRadar.WindowDays - 1)).ToString("yyyy-MM-dd");
|
||||
alert.WindowEndDate = targetDate.Date.ToString("yyyy-MM-dd");
|
||||
alert.LastEvaluatedAt = DateTime.UtcNow;
|
||||
await _hardwareRejectionTracker.RefreshVerdictCalibrationAlertAsync(partition, alert);
|
||||
refreshed++;
|
||||
}
|
||||
}
|
||||
|
||||
return (fired, refreshed, rearmed);
|
||||
}
|
||||
|
||||
private static string AlertKey(string kind, string path, string status) => $"{kind}|{path}|{status}";
|
||||
|
||||
internal static VerdictCalibrationAlert BuildVerdictAlert(
|
||||
string partition, VerdictCalibrationFinding finding, RuleRegressionDimension? dimension, DateTime firstNotifiedAt) => new()
|
||||
{
|
||||
TenantId = partition,
|
||||
Kind = finding.Kind,
|
||||
VerdictPath = finding.VerdictPath,
|
||||
Status = finding.Status,
|
||||
WindowHitCount = finding.WindowHitCount,
|
||||
WindowSessionCount = finding.WindowSessionCount,
|
||||
BaselineHitCount = finding.BaselineHitCount,
|
||||
BaselineSessionCount = finding.BaselineSessionCount,
|
||||
WindowRatePct = finding.WindowRatePct,
|
||||
BaselineRatePct = finding.BaselineRatePct,
|
||||
Lift = finding.Lift,
|
||||
WindowStartDate = finding.WindowStartDate,
|
||||
WindowEndDate = finding.WindowEndDate,
|
||||
Dimension = dimension,
|
||||
FirstNotifiedAt = firstNotifiedAt,
|
||||
LastEvaluatedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// On-fire dimension correlation for a per-path finding: the window's sessions whose
|
||||
/// (derived) verdict path matches vs all window sessions. Group kinds and the "global"
|
||||
/// partition (a cross-tenant scan would dwarf the signal) carry no dimension claim.
|
||||
/// Fail-soft: any error yields null — never a guessed dimension.
|
||||
/// </summary>
|
||||
private async Task<RuleRegressionDimension?> TryCorrelateVerdictDimensionsAsync(string partition, VerdictCalibrationFinding finding)
|
||||
{
|
||||
if (partition == "global" || finding.Kind != VerdictCalibrationAlertKinds.ShareRegression)
|
||||
return null;
|
||||
try
|
||||
{
|
||||
var windowStart = DateTime.ParseExact(finding.WindowStartDate, "yyyy-MM-dd", CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
|
||||
var windowEndExclusive = DateTime.ParseExact(finding.WindowEndDate, "yyyy-MM-dd", CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal).AddDays(1);
|
||||
|
||||
var allSessions = await _maintenanceRepo.GetSessionsByDateRangeAsync(windowStart, windowEndExclusive, partition);
|
||||
if (allSessions.Count == 0) return null;
|
||||
|
||||
var hitSessions = allSessions
|
||||
.Where(s => s.Status.ToString() == finding.Status
|
||||
&& VerdictPathDerivation.Derive(s).Path == finding.VerdictPath)
|
||||
.ToList();
|
||||
return RuleRegressionRadar.ComputeDimensionConcentration(hitSessions, allSessions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Verdict-calibration dimension correlation failed for {Path} in {Partition}", finding.VerdictPath, partition);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly PreviewWhitelistService _previewWhitelistService;
|
||||
private readonly IStorageInitializer _storageInitializer;
|
||||
private readonly ILogger<MaintenanceService> _logger;
|
||||
|
||||
private const string PlatformStatsAliasFileName = "platform-stats.json";
|
||||
@@ -101,6 +102,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
IConfiguration configuration,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
PreviewWhitelistService previewWhitelistService,
|
||||
IStorageInitializer storageInitializer,
|
||||
ILogger<MaintenanceService> logger)
|
||||
{
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
@@ -128,9 +130,22 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_configuration = configuration;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_previewWhitelistService = previewWhitelistService;
|
||||
_storageInitializer = storageInitializer;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
private async Task EnsureAllTablesAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await _storageInitializer.EnsureAllAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Daily table existence pass failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs all maintenance tasks (used by the daily timer trigger)
|
||||
/// </summary>
|
||||
@@ -141,6 +156,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
try
|
||||
{
|
||||
// Startup only point-reads the table schema sentinel (see
|
||||
// TableStorageService.InitializeTablesAsync); this daily full pass is what
|
||||
// recreates a table that was deleted out-of-band. Cheap (idempotent creates).
|
||||
await EnsureAllTablesAsync();
|
||||
await MarkStalledSessionsAsTimedOutAsync();
|
||||
await DetectExcessiveEventSessionsAsync();
|
||||
await AggregateMetricsWithCatchUpAsync();
|
||||
@@ -151,13 +170,24 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// App-version duration regression radar: same episode/tracker pattern over the
|
||||
// install summaries (trailing 35d horizon loaded internally). Fail-soft.
|
||||
await RunAppVersionRegressionRadarAsync();
|
||||
// ONE projected cross-tenant window scan feeds every rolling sweep below (the
|
||||
// StartedAt-only filter is a full-table drain in Table Storage — four of them per
|
||||
// tick was four times the same read). Each sweep slices its own window.
|
||||
var sweepWindow = await LoadSweepWindowSessionsAsync();
|
||||
// F1 PR2: rolling 30d breakdown backfill + daily attribution aggregates. Owns
|
||||
// its own window (NOT the snapshot-gated catch-up above) so late-terminating
|
||||
// sessions still reach their StartedAt-date's aggregate. Fail-soft internally.
|
||||
await SweepTimeAttributionAsync();
|
||||
await SweepTimeAttributionAsync(sweepWindow);
|
||||
// F2 PR4: device-history chain heal (incl. deleted-session ref cleanup) + daily
|
||||
// FTR aggregates over the same rolling window. Fail-soft internally.
|
||||
await SweepDeviceJourneysAsync();
|
||||
await SweepDeviceJourneysAsync(sweepWindow);
|
||||
// Verdict calibration: per-verdict-path daily buckets over the same rolling
|
||||
// window, AFTER the device-journey sweep so the re-enrollment proxy reads
|
||||
// freshly merged chains. Fail-soft internally.
|
||||
await SweepVerdictCalibrationAsync(sweepWindow);
|
||||
// Verdict-calibration drift radar over the rows the sweep just refreshed; anchored
|
||||
// on yesterday like the rule radar (whole days only). Fail-soft.
|
||||
await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow);
|
||||
// Plan §5 PR6 / §16 R14: session retention fanout extracted out of the 2h timer
|
||||
// into the dedicated 12h SessionDeletionMaintenanceFunction so cascade-lifecycle
|
||||
// work has independent cadence + kill-switch + OpsEvent watchdogs. The non-session
|
||||
@@ -199,6 +229,33 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shared sweep window: the longest horizon any rolling sweep needs — the calibration
|
||||
/// radar's 7d window + 28d baseline anchored on YESTERDAY (= 35 days back from today) —
|
||||
/// drained once per tick with the maintenance projection. Sweeps with a shorter window
|
||||
/// slice it by StartedAt (<see cref="SliceSweepWindow"/>). Fail-soft: an empty list makes
|
||||
/// every sweep a no-op rather than failing the tick.
|
||||
/// </summary>
|
||||
internal const int SweepWindowDays = Helpers.VerdictCalibrationRadar.WindowDays + Helpers.VerdictCalibrationRadar.BaselineDays;
|
||||
|
||||
private async Task<List<SessionSummary>> LoadSweepWindowSessionsAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var today = DateTime.UtcNow.Date;
|
||||
return await _maintenanceRepo.GetMaintenanceWindowSessionsAsync(today.AddDays(-SweepWindowDays), today.AddDays(1));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Sweep window load failed — rolling sweeps skip this tick (non-fatal)");
|
||||
return new List<SessionSummary>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Sessions of the shared window that started at or after <paramref name="windowStart"/> (UTC).</summary>
|
||||
internal static List<SessionSummary> SliceSweepWindow(IReadOnlyList<SessionSummary> window, DateTime windowStart)
|
||||
=> window.Where(s => s.StartedAt >= windowStart).ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Manually triggered maintenance with flexible date selection
|
||||
/// </summary>
|
||||
@@ -230,8 +287,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// Timer-path parity: manual maintenance also refreshes the attribution
|
||||
// breakdowns + daily aggregates and the device-history/FTR rollups
|
||||
// (rolling 30d windows, cheap once converged).
|
||||
await SweepTimeAttributionAsync();
|
||||
await SweepDeviceJourneysAsync();
|
||||
var sweepWindow = await LoadSweepWindowSessionsAsync();
|
||||
await SweepTimeAttributionAsync(sweepWindow);
|
||||
await SweepDeviceJourneysAsync(sweepWindow);
|
||||
await SweepVerdictCalibrationAsync(sweepWindow);
|
||||
await RunVerdictCalibrationRadarAsync(dateToAggregate, sweepWindow);
|
||||
|
||||
if (!aggregateOnly)
|
||||
{
|
||||
@@ -350,6 +410,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
int silentMarked = 0;
|
||||
int silentAwaitingUser = 0;
|
||||
int silentSelfDeployingReconciled = 0;
|
||||
foreach (var silent in silentSessions)
|
||||
{
|
||||
// The query is a permissive two-frame pre-filter; the decision is made
|
||||
@@ -374,30 +435,44 @@ namespace AutopilotMonitor.Functions.Services
|
||||
continue;
|
||||
}
|
||||
|
||||
// Self-deploying profile gate (kiosk tenant aebdce78, 2026-08-23): a
|
||||
// silent agent after Device ESP all-succeeded is a finished device
|
||||
// (rebooted into the kiosk autologon / boxed), not a stall and never
|
||||
// "awaiting user" — reconcile to Succeeded right here instead of
|
||||
// routing it through Stalled → AwaitingUser → Incomplete.
|
||||
if (silent.IsSelfDeployingProfile
|
||||
&& await TryReconcileSelfDeployingAsync(silent, lastContactAt, now))
|
||||
{
|
||||
silentSelfDeployingReconciled++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var silentMinutes = (int)(now - lastContactAt).TotalMinutes;
|
||||
await _sessionRepo.UpdateSessionStatusAsync(
|
||||
silent.TenantId,
|
||||
silent.SessionId,
|
||||
SessionStatus.Stalled,
|
||||
VerdictPaths.SweepStalled,
|
||||
stalledAt: now,
|
||||
failureReason: $"Agent silent for {silentMinutes}min (detected by maintenance sweep)");
|
||||
silentMarked++;
|
||||
}
|
||||
|
||||
if (silentMarked > 0 || silentAwaitingUser > 0)
|
||||
if (silentMarked > 0 || silentAwaitingUser > 0 || silentSelfDeployingReconciled > 0)
|
||||
{
|
||||
totalSessionsMarkedStalled += silentMarked;
|
||||
_logger.LogInformation($"Tenant {tenantId}: Marked {silentMarked} agent-silent sessions as Stalled, {silentAwaitingUser} as AwaitingUser (WhiteGlove Part 2; silence threshold: {agentSilenceHours}h)");
|
||||
_logger.LogInformation($"Tenant {tenantId}: Marked {silentMarked} agent-silent sessions as Stalled, {silentAwaitingUser} as AwaitingUser (WhiteGlove Part 2), {silentSelfDeployingReconciled} reconciled to Succeeded (self-deploying; silence threshold: {agentSilenceHours}h)");
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
"SessionStalled",
|
||||
"Session",
|
||||
$"{silentMarked + silentAwaitingUser} sessions",
|
||||
$"{silentMarked + silentAwaitingUser + silentSelfDeployingReconciled} sessions",
|
||||
"System.Maintenance",
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "SessionsMarkedStalled", silentMarked.ToString() },
|
||||
{ "SessionsAwaitingUser", silentAwaitingUser.ToString() },
|
||||
{ "SessionsSelfDeployingReconciled", silentSelfDeployingReconciled.ToString() },
|
||||
{ "AgentSilenceHours", agentSilenceHours.ToString() },
|
||||
{ "SilenceCutoff", silenceCutoff.ToString("yyyy-MM-ddTHH:mm:ss") }
|
||||
});
|
||||
@@ -468,9 +543,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// Device Setup done + within grace → AwaitingUser, else Incomplete; silent before
|
||||
// Device Setup → Incomplete. Never Failed without an explicit failure signal.
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents);
|
||||
var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
var (targetStatus, reason, rule) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt,
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt);
|
||||
isPreProvisioned: session.IsPreProvisioned, resumedAt: session.ResumedAt,
|
||||
isSelfDeployingProfile: session.IsSelfDeployingProfile);
|
||||
|
||||
// No-op if the verdict equals the current (non-terminal) state.
|
||||
if (targetStatus == session.Status)
|
||||
@@ -480,6 +556,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
session.TenantId,
|
||||
session.SessionId,
|
||||
targetStatus,
|
||||
VerdictPaths.Compose(VerdictPaths.OriginSweep, rule),
|
||||
failureReason: reason,
|
||||
failureSnapshotJson: snapshotJson
|
||||
);
|
||||
@@ -609,6 +686,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
silent.TenantId,
|
||||
silent.SessionId,
|
||||
SessionStatus.AwaitingUser,
|
||||
VerdictPaths.SweepWhiteGloveAwaiting,
|
||||
failureReason: EnrollmentTimeoutClassifier.WhiteGloveAwaitingUserReason(rollup));
|
||||
if (transitioned)
|
||||
_logger.LogInformation(
|
||||
@@ -623,6 +701,44 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies the self-deploying gate to one agent-silent candidate (the caller pre-filters on
|
||||
/// <see cref="SessionSummary.IsSelfDeployingProfile"/> so only those pay the event read).
|
||||
/// When <see cref="EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned"/> confirms Device
|
||||
/// ESP all-succeeded with no explicit failure, reconciles the session to Succeeded with the
|
||||
/// shared honest reason. Returns true when the write happened; false (fail-soft, also on
|
||||
/// read errors) lets the caller fall through to the normal Stalled marker.
|
||||
/// </summary>
|
||||
private async Task<bool> TryReconcileSelfDeployingAsync(SessionSummary silent, DateTime lastContactAt, DateTime now)
|
||||
{
|
||||
try
|
||||
{
|
||||
var events = await _sessionRepo.GetSessionEventsAsync(
|
||||
silent.TenantId, silent.SessionId, maxResults: 1000);
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
if (!EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned(rollup, silent.IsSelfDeployingProfile))
|
||||
return false;
|
||||
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
silent.TenantId,
|
||||
silent.SessionId,
|
||||
SessionStatus.Succeeded,
|
||||
VerdictPaths.SweepSelfDeployingReconcile,
|
||||
failureReason: EnrollmentTimeoutClassifier.SelfDeployingReconcileReason(
|
||||
silent.ResumedAt ?? silent.StartedAt, now, lastContactAt));
|
||||
if (transitioned)
|
||||
_logger.LogInformation(
|
||||
$"Session {silent.SessionId}: self-deploying profile silent after Device Setup provisioned — reconciled to Succeeded instead of Stalled");
|
||||
return transitioned;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
$"Self-deploying reconcile gate failed for session {silent.SessionId}; falling back to Stalled marker");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the server-authored <c>session_timeout</c> event injected into the stream when the
|
||||
/// maintenance sweep graduates a stalled session to terminal Failed. Static + pure so the field
|
||||
|
||||
+2
@@ -83,6 +83,8 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
// whole-row deletion besides chain-emptying.
|
||||
Constants.TableNames.DeviceHistories,
|
||||
Constants.TableNames.DeviceJourneyAggregates,
|
||||
// Verdict calibration daily buckets (PK=TenantId) — regenerable from Sessions, wiped with the tenant.
|
||||
Constants.TableNames.VerdictCalibrationAggregates,
|
||||
// Pre-write config snapshots (PK=tenantId). Wiped in the bulk phase; the
|
||||
// TenantConfiguration row itself is deleted LAST via SafeWipe (a delete, not a
|
||||
// save), so the wipe cannot re-create a snapshot afterwards. The admin-config
|
||||
|
||||
@@ -641,6 +641,40 @@ namespace AutopilotMonitor.Functions.Services
|
||||
lift,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// Verdict-calibration drift radar (docs/backend/verdict-calibration.md): a verdict
|
||||
/// path's share of sessions doubled over its 28d baseline, the silence share
|
||||
/// (sweep+maxlife — agent went quiet, backend had to decide) doubled, or the pure
|
||||
/// fallthrough rule r6 decides ≥20 % of classifier verdicts. Operator-only diagnostic —
|
||||
/// there is deliberately NO tenant bell. Fired ONCE per (kind, path, status) episode
|
||||
/// (tracker-deduped). Dual-registered in the web OPS_EVENT_TYPES catalog
|
||||
/// (memory: feedback_ops_event_types_dual_register).
|
||||
/// </summary>
|
||||
public Task RecordVerdictCalibrationDriftAsync(
|
||||
string tenantId, string kind, string verdictPath, string status,
|
||||
int windowHitCount, int windowSessionCount, double windowRatePct,
|
||||
int baselineHitCount, int baselineSessionCount, double baselineRatePct,
|
||||
double? lift, string? dimensionSummary)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "VerdictCalibrationDrift", OpsEventSeverity.Warning,
|
||||
$"Verdict calibration [{kind}] {verdictPath}/{status}: {windowHitCount}/{windowSessionCount} ({windowRatePct}%) over 7d " +
|
||||
$"vs {baselineRatePct}% baseline ({baselineHitCount}/{baselineSessionCount} over 28d)" +
|
||||
(lift.HasValue ? $" — lift {lift.Value}x" : string.Empty) +
|
||||
(string.IsNullOrEmpty(dimensionSummary) ? string.Empty : $". {dimensionSummary}"),
|
||||
tenantId, "System.Maintenance",
|
||||
new
|
||||
{
|
||||
kind,
|
||||
verdictPath,
|
||||
status,
|
||||
windowHitCount,
|
||||
windowSessionCount,
|
||||
windowRatePct,
|
||||
baselineHitCount,
|
||||
baselineSessionCount,
|
||||
baselineRatePct,
|
||||
lift,
|
||||
});
|
||||
|
||||
// ── Agent ──────────────────────────────────────────────────────────────
|
||||
|
||||
public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
@@ -17,10 +16,37 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly ILogger<RateLimitService> _logger;
|
||||
private readonly TimeSpan _windowDuration;
|
||||
|
||||
// Dedicated lock objects per device to avoid race conditions when cache evicts the request history list.
|
||||
// Using the list itself as a lock target is unsafe because cache eviction can create a new list instance,
|
||||
// causing different threads to lock on different objects.
|
||||
private readonly ConcurrentDictionary<string, object> _locks = new();
|
||||
// The request history and its lock travel together in one cache entry. Whoever observes the
|
||||
// same list observes the same lock, so two threads can never mutate one list under different
|
||||
// locks. A thread holding an already-evicted bucket only mutates a discarded list, which is
|
||||
// harmless at the expiry boundary. A separate per-key lock dictionary is deliberately NOT
|
||||
// used: its eviction-callback cleanup raced with lock re-creation and leaked on replaced entries.
|
||||
private sealed class RequestBucket
|
||||
{
|
||||
public readonly List<DateTime> History = new();
|
||||
public readonly object Sync = new();
|
||||
}
|
||||
|
||||
// IMemoryCache.GetOrCreate is not atomic: two first-time callers can each create a bucket and
|
||||
// the loser would count its request into a discarded list. Creation is serialized here
|
||||
// (cheap: only cache misses take it); the hot path is a lock-free TryGetValue.
|
||||
private readonly object _createLock = new();
|
||||
|
||||
private RequestBucket GetOrCreateBucket(string cacheKey)
|
||||
{
|
||||
if (_cache.TryGetValue(cacheKey, out RequestBucket? existing) && existing != null)
|
||||
return existing;
|
||||
|
||||
lock (_createLock)
|
||||
{
|
||||
return _cache.GetOrCreate(cacheKey, entry =>
|
||||
{
|
||||
entry.SlidingExpiration = _windowDuration;
|
||||
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1);
|
||||
return new RequestBucket();
|
||||
})!;
|
||||
}
|
||||
}
|
||||
|
||||
public RateLimitService(IMemoryCache cache, ILogger<RateLimitService> logger)
|
||||
{
|
||||
@@ -56,18 +82,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
var cacheKey = $"ratelimit_{deviceThumbprint}";
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
// Get or create request history for this device
|
||||
var lockObj = _locks.GetOrAdd(cacheKey, _ => new object());
|
||||
var requestHistory = _cache.GetOrCreate(cacheKey, entry =>
|
||||
{
|
||||
entry.SlidingExpiration = _windowDuration;
|
||||
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1);
|
||||
entry.RegisterPostEvictionCallback((key, _, _, _) => _locks.TryRemove((string)key, out _));
|
||||
return new List<DateTime>();
|
||||
})!;
|
||||
// Get or create request history for this device. The bucket is mutated in place; the
|
||||
// cache lookup itself refreshes the sliding expiration, so no re-Set is needed.
|
||||
var bucket = GetOrCreateBucket(cacheKey);
|
||||
|
||||
lock (lockObj)
|
||||
lock (bucket.Sync)
|
||||
{
|
||||
var requestHistory = bucket.History;
|
||||
|
||||
// Remove requests outside the sliding window
|
||||
var windowStart = now.Subtract(_windowDuration);
|
||||
requestHistory.RemoveAll(timestamp => timestamp < windowStart);
|
||||
@@ -104,13 +126,6 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// Add current request to history
|
||||
requestHistory.Add(now);
|
||||
|
||||
// Update cache
|
||||
_cache.Set(cacheKey, requestHistory, new MemoryCacheEntryOptions
|
||||
{
|
||||
SlidingExpiration = _windowDuration,
|
||||
AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1)
|
||||
});
|
||||
|
||||
return new RateLimitResult
|
||||
{
|
||||
IsAllowed = true,
|
||||
|
||||
@@ -311,7 +311,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// DurationSeconds. Letting UpdateSessionStatusAsync fall back to LastEventAt
|
||||
// anchors duration on when the session actually went silent.
|
||||
await _sessionRepo.UpdateSessionStatusAsync(
|
||||
tenantId, sessionId, SessionStatus.Failed,
|
||||
tenantId, sessionId, SessionStatus.Failed, VerdictPaths.Compose(VerdictPaths.OriginRule, rule.RuleId),
|
||||
failureReason: failureReason,
|
||||
failureSource: failureSource);
|
||||
|
||||
|
||||
@@ -74,6 +74,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
"ValidatedBy",
|
||||
"FailureSnapshotJson",
|
||||
"AdminMarkedAction",
|
||||
"VerdictPath",
|
||||
"PriorStatus",
|
||||
"PriorVerdictPath",
|
||||
"DurationSeconds",
|
||||
"DiagnosticsBlobName",
|
||||
"DiagnosticsBlobDestination",
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Microsoft.ApplicationInsights.DataContracts;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Emits one App Insights metric per process start so startup duration is a trend, not an
|
||||
/// anecdote: <c>BackendStartupMs</c> = process start → host ApplicationStarted (includes
|
||||
/// runtime + DI + every hosted service), plus <c>BackendTableInitMs</c> for the table
|
||||
/// initialization slice (see TableStorageService.InitializeTablesAsync / schema sentinel).
|
||||
/// Metrics are not subject to adaptive sampling and reach App Insights regardless of the
|
||||
/// worker log level, unlike Information traces.
|
||||
/// Query: <c>customMetrics | where name == "BackendStartupMs" | project timestamp, value, customDimensions</c>.
|
||||
/// </summary>
|
||||
public sealed class StartupTelemetryService : IHostedService
|
||||
{
|
||||
public const string StartupMetricName = "BackendStartupMs";
|
||||
public const string TableInitMetricName = "BackendTableInitMs";
|
||||
|
||||
private readonly IHostApplicationLifetime _lifetime;
|
||||
private readonly TelemetryClient _telemetry;
|
||||
private readonly TableStorageService _tableStorage;
|
||||
private readonly BackendBuildInfo _buildInfo;
|
||||
private readonly ILogger<StartupTelemetryService> _logger;
|
||||
|
||||
public StartupTelemetryService(
|
||||
IHostApplicationLifetime lifetime,
|
||||
TelemetryClient telemetry,
|
||||
TableStorageService tableStorage,
|
||||
BackendBuildInfo buildInfo,
|
||||
ILogger<StartupTelemetryService> logger)
|
||||
{
|
||||
_lifetime = lifetime;
|
||||
_telemetry = telemetry;
|
||||
_tableStorage = tableStorage;
|
||||
_buildInfo = buildInfo;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
// ApplicationStarted fires after ALL hosted services completed StartAsync — that is
|
||||
// the moment the host is ready to serve, independent of registration order.
|
||||
_lifetime.ApplicationStarted.Register(Emit);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
private void Emit()
|
||||
{
|
||||
try
|
||||
{
|
||||
var startupMs = (DateTime.UtcNow - Process.GetCurrentProcess().StartTime.ToUniversalTime()).TotalMilliseconds;
|
||||
var init = _tableStorage.LastInitialization;
|
||||
|
||||
var props = new Dictionary<string, string>
|
||||
{
|
||||
["version"] = _buildInfo.Version,
|
||||
["commit"] = _buildInfo.CommitHash,
|
||||
["tableInitFullPass"] = init.FullPassRan.ToString(),
|
||||
["instance"] = Environment.MachineName
|
||||
};
|
||||
|
||||
_telemetry.TrackMetric(new MetricTelemetry(StartupMetricName, Math.Round(startupMs)).WithProperties(props));
|
||||
_telemetry.TrackMetric(new MetricTelemetry(TableInitMetricName, Math.Round(init.DurationMs)).WithProperties(props));
|
||||
|
||||
_logger.LogInformation("Backend startup took {StartupMs}ms (table init {TableInitMs}ms, fullPass={FullPass})",
|
||||
Math.Round(startupMs), Math.Round(init.DurationMs), init.FullPassRan);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to emit startup telemetry");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static class MetricTelemetryExtensions
|
||||
{
|
||||
public static MetricTelemetry WithProperties(this MetricTelemetry metric, IReadOnlyDictionary<string, string> props)
|
||||
{
|
||||
foreach (var (key, value) in props)
|
||||
metric.Properties[key] = value;
|
||||
return metric;
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ namespace AutopilotMonitor.Functions.Services;
|
||||
/// <summary>
|
||||
/// Hosted service that initializes all Azure Table Storage tables at application startup.
|
||||
/// This ensures all tables exist before any requests are processed.
|
||||
/// Also performs a one-time backfill of the SessionsIndex table if it is empty.
|
||||
/// Also performs a one-time backfill of the SessionsIndex table on fresh storage (see StartAsync).
|
||||
/// </summary>
|
||||
public class TableInitializerService : IHostedService
|
||||
{
|
||||
@@ -23,12 +23,13 @@ public class TableInitializerService : IHostedService
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("TableInitializerService starting - initializing all Azure Table Storage tables");
|
||||
_logger.LogInformation("TableInitializerService starting");
|
||||
|
||||
var fullPassRan = false;
|
||||
try
|
||||
{
|
||||
await _tableStorageService.InitializeTablesAsync();
|
||||
_logger.LogInformation("TableInitializerService completed - all tables ready");
|
||||
fullPassRan = await _tableStorageService.InitializeTablesAsync();
|
||||
_logger.LogInformation("TableInitializerService completed - all tables ready (fullPass={FullPass})", fullPassRan);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -37,10 +38,15 @@ public class TableInitializerService : IHostedService
|
||||
// Individual operations will fail gracefully with appropriate error messages
|
||||
}
|
||||
|
||||
// One-time backfill: if SessionsIndex is empty, copy all sessions from Sessions table
|
||||
// One-time backfill: SessionsIndex can only be empty on fresh storage, which is exactly
|
||||
// when the schema sentinel was missing and the full pass ran. Skipped on the fast path,
|
||||
// and guarded by a conditional-insert claim so only one scaled-out instance scans.
|
||||
if (!fullPassRan) return;
|
||||
|
||||
try
|
||||
{
|
||||
if (await _tableStorageService.IsSessionIndexEmptyAsync())
|
||||
if (await _tableStorageService.IsSessionIndexEmptyAsync()
|
||||
&& await _tableStorageService.TryClaimSessionIndexBackfillAsync())
|
||||
{
|
||||
_logger.LogInformation("SessionsIndex table is empty — starting startup backfill");
|
||||
var count = await _tableStorageService.BackfillSessionIndexAsync();
|
||||
|
||||
@@ -595,6 +595,36 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public Task<List<SessionSummary>> GetGeoWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null)
|
||||
=> QuerySessionsByDateRangeAsync(startDate, endDate, tenantId, GeoMetricsSessionProjection);
|
||||
|
||||
/// <summary>
|
||||
/// Columns the rolling maintenance sweeps consume from the SHARED window scan (one
|
||||
/// cross-tenant drain per tick feeds the time-attribution, device-journey and
|
||||
/// verdict-calibration sweeps plus the calibration radar's tenant discovery — see
|
||||
/// MaintenanceService.LoadSweepWindowSessionsAsync). Union of the three sweeps' reads:
|
||||
/// identity/device (SerialNumber, Manufacturer, Model), status + timestamps, the
|
||||
/// enrollment-class inputs, DurationSeconds/EventCount, the deletion-cascade marker,
|
||||
/// the verdict attribution (VerdictPath/Prior*) and the legacy-derivation inputs
|
||||
/// (FailureReason/ReconcileReason/FailureSource/AdminMarkedAction/EspSoftFailure).
|
||||
/// Drops the wide row's FailureSnapshotJson, geo, OS and diagnostics columns. internal so
|
||||
/// MaintenanceSweepProjectionEquivalenceTests derives its keep-set from this exact array.
|
||||
/// </summary>
|
||||
internal static readonly string[] MaintenanceSweepSessionProjection =
|
||||
{
|
||||
"PartitionKey", "RowKey", "StartedAt", "CompletedAt", "LastEventAt", "Status",
|
||||
"DurationSeconds", "EventCount", "DeletionState",
|
||||
"SerialNumber", "Manufacturer", "Model",
|
||||
"EnrollmentType", "IsPreProvisioned", "IsSelfDeployingProfile", "IsUserDriven", "ResumedAt",
|
||||
"AdminMarkedAction", "VerdictPath", "PriorStatus", "PriorVerdictPath",
|
||||
"FailureReason", "ReconcileReason", "FailureSource", "EspSoftFailure",
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Column-projected cross-tenant date-range query backing the shared maintenance sweep
|
||||
/// window. Same filter semantics as <see cref="GetSessionsByDateRangeAsync"/>; only
|
||||
/// <see cref="MaintenanceSweepSessionProjection"/> is transferred.
|
||||
/// </summary>
|
||||
public Task<List<SessionSummary>> GetMaintenanceWindowSessionsAsync(DateTime startDate, DateTime endDate)
|
||||
=> QuerySessionsByDateRangeAsync(startDate, endDate, tenantId: null, MaintenanceSweepSessionProjection);
|
||||
|
||||
private async Task<List<SessionSummary>> QuerySessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId, string[]? select)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(tenantId))
|
||||
@@ -767,6 +797,39 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Self-deploying-profile sessions sitting in Incomplete / AwaitingUser / Stalled — the
|
||||
/// shapes the pre-2026-08-23 classifier produced for a finished kiosk device whose agent
|
||||
/// went silent after Device ESP. Candidates for the retro-reconcile; the classifier
|
||||
/// decides per session, this is only the cheap server-side pre-filter.
|
||||
/// </summary>
|
||||
public async Task<List<SessionSummary>> GetSelfDeployingSilentSessionsAsync(string tenantId, int maxResults)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
var filter = $"PartitionKey eq '{tenantId}' " +
|
||||
$"and IsSelfDeployingProfile eq true " +
|
||||
$"and (Status eq 'Incomplete' or Status eq 'AwaitingUser' or Status eq 'Stalled')";
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(filter: filter))
|
||||
{
|
||||
sessions.Add(MapToSessionSummary(entity));
|
||||
if (sessions.Count >= maxResults)
|
||||
break;
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, $"Failed to get self-deploying silent sessions for tenant {tenantId}");
|
||||
return new List<SessionSummary>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Narrow projection of every session row in the tenant partition — one scan feeding the
|
||||
/// in-memory Pending-orphan matching (misclassification audit 2026-07-16). Fields outside
|
||||
@@ -1085,8 +1148,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to check if session index is empty");
|
||||
return true; // Assume empty on error → trigger backfill
|
||||
// A transient read error must NOT trigger a full Sessions scan with serial upserts.
|
||||
// The manual maintenance backfill remains the safety net.
|
||||
_logger.LogWarning(ex, "Failed to check if session index is empty — skipping startup backfill");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -209,6 +209,16 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (!string.IsNullOrEmpty(adminMarkedAction))
|
||||
indexEntity["AdminMarkedAction"] = adminMarkedAction;
|
||||
|
||||
// Verdict calibration attribution (docs/backend/verdict-calibration.md): the aggregate
|
||||
// sweep reads these off the session row, and the index mirror must not drop them on a
|
||||
// StartedAt-shift full upsert.
|
||||
foreach (var col in new[] { "VerdictPath", "PriorStatus", "PriorVerdictPath" })
|
||||
{
|
||||
var val = sessionEntity.GetString(col);
|
||||
if (!string.IsNullOrEmpty(val))
|
||||
indexEntity[col] = val;
|
||||
}
|
||||
|
||||
var durationSeconds = sessionEntity.GetInt32("DurationSeconds");
|
||||
if (durationSeconds.HasValue)
|
||||
indexEntity["DurationSeconds"] = durationSeconds.Value;
|
||||
@@ -492,6 +502,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// race the in-flight cascade. Empty/null is fine on first registration.
|
||||
string? existingDeletionState = null;
|
||||
string? existingPendingDeletionManifestId = null;
|
||||
// Verdict calibration: a fresh row's status comes from registration itself; a
|
||||
// re-registration preserves the path that produced the (non-terminal) status it
|
||||
// keeps, and the prior-verdict pair, so the Replace never blanks the attribution.
|
||||
string verdictPath = VerdictPaths.RegisterNew;
|
||||
string? priorStatus = null;
|
||||
string? priorVerdictPath = null;
|
||||
|
||||
try
|
||||
{
|
||||
@@ -544,6 +560,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
existingIndexRowKey = existingEntity.GetString("IndexRowKey");
|
||||
existingDeletionState = existingEntity.GetString("DeletionState");
|
||||
existingPendingDeletionManifestId = existingEntity.GetString("PendingDeletionManifestId");
|
||||
verdictPath = existingEntity.GetString("VerdictPath") ?? verdictPath;
|
||||
priorStatus = existingEntity.GetString("PriorStatus");
|
||||
priorVerdictPath = existingEntity.GetString("PriorVerdictPath");
|
||||
|
||||
// Guard: never regress a terminal status (Succeeded/Failed) back to InProgress.
|
||||
// StoreSessionAsync uses UpsertEntity (Replace mode) which overwrites all fields.
|
||||
@@ -561,6 +580,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
_logger.LogInformation($"Session {registration.SessionId} resuming from Pending (WhiteGlove Part 2)");
|
||||
status = SessionStatus.InProgress.ToString();
|
||||
verdictPath = VerdictPaths.RegisterWhiteGloveResume;
|
||||
// Store ResumedAt as fallback if whiteglove_resumed event hasn't set it yet
|
||||
if (!resumedAt.HasValue)
|
||||
resumedAt = registration.StartedAt;
|
||||
@@ -618,6 +638,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (!string.IsNullOrWhiteSpace(reconcileReason))
|
||||
entity["ReconcileReason"] = reconcileReason;
|
||||
|
||||
entity["VerdictPath"] = verdictPath;
|
||||
if (!string.IsNullOrWhiteSpace(priorStatus))
|
||||
entity["PriorStatus"] = priorStatus;
|
||||
if (!string.IsNullOrWhiteSpace(priorVerdictPath))
|
||||
entity["PriorVerdictPath"] = priorVerdictPath;
|
||||
|
||||
// Latest non-Unknown validation wins: a Bootstrap-registered session that later
|
||||
// re-registers under cert auth upgrades to the cert-path validator; an Unknown
|
||||
// result (device validation disabled for the tenant) preserves whatever an
|
||||
@@ -691,7 +717,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// fresh-read window isn't reverted by the Replace below (see guard).
|
||||
"Status", "CurrentPhase", "CompletedAt", "FailureReason", "ReconcileReason",
|
||||
"FailureSnapshotJson", "FailureSource", "AdminMarkedAction", "DurationSeconds",
|
||||
"EspSoftFailure", "CompletionSource",
|
||||
"EspSoftFailure", "CompletionSource", "VerdictPath", "PriorStatus", "PriorVerdictPath",
|
||||
});
|
||||
freshEtag = freshResponse.Value.ETag;
|
||||
freshEntity = freshResponse.Value;
|
||||
@@ -731,7 +757,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|| freshStatus == SessionStatus.Incomplete.ToString())
|
||||
{
|
||||
entity["Status"] = freshStatus;
|
||||
foreach (var col in new[] { "CurrentPhase", "CompletedAt", "FailureReason", "ReconcileReason", "FailureSnapshotJson", "FailureSource", "AdminMarkedAction", "DurationSeconds", "EspSoftFailure", "CompletionSource" })
|
||||
foreach (var col in new[] { "CurrentPhase", "CompletedAt", "FailureReason", "ReconcileReason", "FailureSnapshotJson", "FailureSource", "AdminMarkedAction", "DurationSeconds", "EspSoftFailure", "CompletionSource", "VerdictPath", "PriorStatus", "PriorVerdictPath" })
|
||||
{
|
||||
if (freshEntity!.TryGetValue(col, out var val) && val is not null)
|
||||
entity[col] = val;
|
||||
@@ -1709,10 +1735,53 @@ namespace AutopilotMonitor.Functions.Services
|
||||
: null;
|
||||
}
|
||||
|
||||
public async Task<bool> UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null, bool allowTerminalReclassification = false, bool espSoftFailure = false, string? completionSource = null)
|
||||
/// <summary>
|
||||
/// Decides whether a status write overrides a prior VERDICT and must therefore preserve
|
||||
/// it in <c>PriorStatus</c>/<c>PriorVerdictPath</c> — the correction stream the verdict
|
||||
/// calibration aggregate counts (admin mark, late agent completion upgrading a sweep
|
||||
/// Incomplete, retro-reclassification). Only a prior Succeeded/Failed/Incomplete/
|
||||
/// AwaitingUser counts as a verdict; a stall marker healing back to InProgress is not a
|
||||
/// correction. Same-status writes (AwaitingUser refresh) keep the original prior. Null =
|
||||
/// nothing to preserve. Static + pure so the matrix is unit-testable without Table Storage.
|
||||
/// </summary>
|
||||
internal static (string PriorStatus, string PriorVerdictPath)? ComputePriorVerdict(
|
||||
string? existingStatus, string? existingVerdictPath, SessionStatus incoming)
|
||||
{
|
||||
if (string.IsNullOrEmpty(existingStatus) || string.IsNullOrEmpty(existingVerdictPath))
|
||||
return null;
|
||||
if (existingStatus == incoming.ToString())
|
||||
return null;
|
||||
|
||||
bool existingIsVerdict = existingStatus == SessionStatus.Succeeded.ToString()
|
||||
|| existingStatus == SessionStatus.Failed.ToString()
|
||||
|| existingStatus == SessionStatus.Incomplete.ToString()
|
||||
|| existingStatus == SessionStatus.AwaitingUser.ToString();
|
||||
return existingIsVerdict ? (existingStatus!, existingVerdictPath!) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stamps <c>VerdictPath</c> (always — every status write declares its origin, unlike
|
||||
/// FailureSource which is Failed-only) and the prior-verdict pair when
|
||||
/// <see cref="ComputePriorVerdict"/> says the write overrides one. Shared by the normal and
|
||||
/// the ETag-exhaustion force path so both stay in lock-step.
|
||||
/// </summary>
|
||||
private static void ApplyVerdictPath(TableEntity update, TableEntity existing, SessionStatus status, string verdictPath)
|
||||
{
|
||||
update["VerdictPath"] = verdictPath;
|
||||
var prior = ComputePriorVerdict(existing.GetString("Status"), existing.GetString("VerdictPath"), status);
|
||||
if (prior.HasValue)
|
||||
{
|
||||
update["PriorStatus"] = prior.Value.PriorStatus;
|
||||
update["PriorVerdictPath"] = prior.Value.PriorVerdictPath;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, string verdictPath, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null, bool allowTerminalReclassification = false, bool espSoftFailure = false, string? completionSource = null)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
if (string.IsNullOrWhiteSpace(verdictPath))
|
||||
throw new ArgumentException("Every status write must declare its VerdictPath (see VerdictPaths)", nameof(verdictPath));
|
||||
|
||||
const int maxRetries = 5;
|
||||
int retryCount = 0;
|
||||
@@ -1792,6 +1861,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
update["Status"] = status.ToString();
|
||||
ApplyVerdictPath(update, session, status, verdictPath);
|
||||
|
||||
// Update current phase if provided.
|
||||
if (currentPhase.HasValue)
|
||||
@@ -2080,6 +2150,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
var forceUpdate = new TableEntity(tenantId, sessionId);
|
||||
|
||||
forceUpdate["Status"] = status.ToString();
|
||||
ApplyVerdictPath(forceUpdate, freshSession, status, verdictPath);
|
||||
|
||||
if (currentPhase.HasValue)
|
||||
forceUpdate["CurrentPhase"] = (int)currentPhase.Value;
|
||||
@@ -2833,10 +2904,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// unconditional Merge-mode write. Uses ETag.All to bypass optimistic-concurrency conflicts,
|
||||
/// making this suitable as a last-resort fallback when ETag-based updates have been exhausted.
|
||||
/// </summary>
|
||||
public async Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned, SessionStatus? status = null, bool? isUserDriven = null)
|
||||
public async Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned, SessionStatus? status = null, bool? isUserDriven = null, string? verdictPath = null)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
if (status.HasValue && string.IsNullOrWhiteSpace(verdictPath))
|
||||
throw new ArgumentException("A status write must declare its VerdictPath (see VerdictPaths)", nameof(verdictPath));
|
||||
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
|
||||
@@ -2848,6 +2921,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (status.HasValue)
|
||||
{
|
||||
update["Status"] = status.Value.ToString();
|
||||
update["VerdictPath"] = verdictPath!;
|
||||
}
|
||||
|
||||
if (isUserDriven.HasValue)
|
||||
@@ -3162,6 +3236,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
EspSoftFailure = entity.GetBoolean("EspSoftFailure") ?? false,
|
||||
CompletionSource = entity.GetString("CompletionSource") ?? string.Empty,
|
||||
AdminMarkedAction = entity.GetString("AdminMarkedAction"),
|
||||
VerdictPath = entity.GetString("VerdictPath"),
|
||||
PriorStatus = entity.GetString("PriorStatus"),
|
||||
PriorVerdictPath = entity.GetString("PriorVerdictPath"),
|
||||
ValidatedBy = entity.GetString("ValidatedBy") ?? string.Empty,
|
||||
PendingActionsJson = entity.GetString("PendingActionsJson") ?? string.Empty,
|
||||
PendingActionsQueuedAt = SafeGetDateTime(entity, "PendingActionsQueuedAt"),
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Partial: verdict-calibration daily aggregate persistence (docs/backend/verdict-calibration.md).
|
||||
/// VerdictCalibrationAggregates: PK=TenantId ("global" for the cross-tenant row),
|
||||
/// RK="{yyyy-MM-dd}" — one row per (tenant, StartedAt date) holding the per-verdict-path
|
||||
/// buckets as a JSON column. Recomputed whole (Replace) by the maintenance sweep, 180d
|
||||
/// retention, regenerable from Sessions + DeviceHistories.
|
||||
/// </summary>
|
||||
public partial class TableStorageService
|
||||
{
|
||||
public async Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
await tableClient.UpsertEntityAsync(BuildVerdictCalibrationAggregateEntity(aggregate), TableUpdateMode.Replace);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to save verdict-calibration aggregate {Date} for tenant {TenantId}",
|
||||
aggregate.Date, aggregate.TenantId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<VerdictCalibrationDailyAggregate>> GetVerdictCalibrationAggregatesAsync(
|
||||
string tenantId, DateTime startDate, DateTime endDate)
|
||||
{
|
||||
if (tenantId != "global")
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
var filter = $"PartitionKey eq '{tenantId}' and RowKey ge '{startDate:yyyy-MM-dd}' and RowKey le '{endDate:yyyy-MM-dd}'";
|
||||
var results = new List<VerdictCalibrationDailyAggregate>();
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(filter: filter))
|
||||
{
|
||||
results.Add(MapToVerdictCalibrationAggregate(entity));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to query verdict-calibration aggregates for tenant {TenantId}", tenantId);
|
||||
return new List<VerdictCalibrationDailyAggregate>();
|
||||
}
|
||||
}
|
||||
|
||||
// internal static: pinned by round-trip tests together with MapToVerdictCalibrationAggregate.
|
||||
internal static TableEntity BuildVerdictCalibrationAggregateEntity(VerdictCalibrationDailyAggregate aggregate)
|
||||
{
|
||||
return new TableEntity(aggregate.TenantId, aggregate.Date)
|
||||
{
|
||||
["Date"] = aggregate.Date,
|
||||
["Version"] = aggregate.Version,
|
||||
["SessionCount"] = aggregate.SessionCount,
|
||||
["TerminalSessionCount"] = aggregate.TerminalSessionCount,
|
||||
["BucketsJson"] = JsonSerializer.Serialize(aggregate.Buckets),
|
||||
["ComputedAt"] = EnsureUtc(aggregate.ComputedAt),
|
||||
};
|
||||
}
|
||||
|
||||
internal static VerdictCalibrationDailyAggregate MapToVerdictCalibrationAggregate(TableEntity entity)
|
||||
{
|
||||
return new VerdictCalibrationDailyAggregate
|
||||
{
|
||||
TenantId = entity.PartitionKey,
|
||||
Date = entity.GetString("Date") ?? entity.RowKey,
|
||||
Version = entity.GetInt32("Version") ?? 0,
|
||||
SessionCount = entity.GetInt32("SessionCount") ?? 0,
|
||||
TerminalSessionCount = entity.GetInt32("TerminalSessionCount") ?? 0,
|
||||
Buckets = DeserializeJsonColumn<VerdictCalibrationBucket>(entity.GetString("BucketsJson")),
|
||||
ComputedAt = entity.GetDateTimeOffset("ComputedAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
};
|
||||
}
|
||||
|
||||
public async Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
await tableClient.DeleteEntityAsync(tenantId, dateKey);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete stale verdict-calibration aggregate {Date} for tenant {TenantId}",
|
||||
dateKey, tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<int> DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VerdictCalibrationAggregates);
|
||||
var filter = $"RowKey lt '{cutoffDate:yyyy-MM-dd}'";
|
||||
var deleted = 0;
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
|
||||
filter: filter, select: new[] { "PartitionKey", "RowKey" }))
|
||||
{
|
||||
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
|
||||
deleted++;
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete old verdict-calibration aggregates");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,25 +78,92 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return _tableServiceClient.GetTableClient(tableName);
|
||||
}
|
||||
|
||||
// ===== TABLE SCHEMA SENTINEL =====
|
||||
//
|
||||
// A cold start used to issue one CreateTableIfNotExists per table (each a Create POST
|
||||
// whose 409 is swallowed). The sentinel row in AdminConfiguration stores a hash derived
|
||||
// from Constants.TableNames.All; when it matches, startup is a single point-read.
|
||||
// Adding a table to TableNames.All changes the hash, so the next start runs the full
|
||||
// pass once and rewrites the sentinel — no manually maintained version to forget.
|
||||
// Gap: a table deleted by hand is not noticed until the daily maintenance full pass
|
||||
// (EnsureAllTablesAsync) recreates it.
|
||||
|
||||
internal const string SchemaSentinelPartitionKey = "SchemaSentinel";
|
||||
internal const string SchemaSentinelRowKey = "tables";
|
||||
internal const string SchemaSentinelHashProperty = "TableSchemaHash";
|
||||
internal const string SessionIndexBackfillClaimRowKey = "sessionIndexBackfill";
|
||||
|
||||
/// <summary>Outcome of the last <see cref="InitializeTablesAsync"/> call, read by StartupTelemetryService.</summary>
|
||||
public readonly record struct InitializationSnapshot(double DurationMs, bool FullPassRan);
|
||||
public InitializationSnapshot LastInitialization { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic hash over the table registry (ordinal-sorted, so declaration order is irrelevant).
|
||||
/// </summary>
|
||||
internal static string ComputeTableSchemaHash(IEnumerable<string> tableNames)
|
||||
{
|
||||
var joined = string.Join("\n", tableNames.OrderBy(n => n, StringComparer.Ordinal));
|
||||
var bytes = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(joined));
|
||||
return Convert.ToHexString(bytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes all Azure Table Storage tables.
|
||||
/// This method is idempotent and safe to call multiple times.
|
||||
/// Should be called at application startup.
|
||||
/// Fast path: the schema sentinel matches the current table registry → no table calls.
|
||||
/// Slow path (sentinel missing, stale, or unreadable): CreateTableIfNotExists for every
|
||||
/// table, then the sentinel is (re)written. Idempotent and safe under scale-out — two
|
||||
/// instances that both see a stale sentinel simply both run the idempotent full pass.
|
||||
/// Returns true when the full pass ran (callers use this to gate one-time work such as
|
||||
/// the SessionsIndex backfill, which can only be needed on fresh storage).
|
||||
/// </summary>
|
||||
public async Task InitializeTablesAsync()
|
||||
public async Task<bool> InitializeTablesAsync()
|
||||
{
|
||||
if (_tablesInitialized)
|
||||
{
|
||||
_logger.LogDebug("Tables already initialized, skipping");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
lock (_initLock)
|
||||
{
|
||||
if (_tablesInitialized) return;
|
||||
if (_tablesInitialized) return false;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Initializing Azure Table Storage tables...");
|
||||
var initStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
||||
var expectedHash = ComputeTableSchemaHash(Constants.TableNames.All);
|
||||
|
||||
if (await SchemaSentinelMatchesAsync(expectedHash))
|
||||
{
|
||||
_logger.LogInformation("Table schema sentinel matches ({HashPrefix}) — skipping table creation pass",
|
||||
expectedHash.Substring(0, 12));
|
||||
lock (_initLock) { _tablesInitialized = true; }
|
||||
LastInitialization = new InitializationSnapshot(initStopwatch.Elapsed.TotalMilliseconds, FullPassRan: false);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Table schema sentinel missing or stale — initializing Azure Table Storage tables...");
|
||||
var failCount = await EnsureAllTablesAsync();
|
||||
|
||||
if (failCount == 0)
|
||||
{
|
||||
await WriteSchemaSentinelAsync(expectedHash);
|
||||
}
|
||||
|
||||
lock (_initLock)
|
||||
{
|
||||
_tablesInitialized = failCount == 0;
|
||||
}
|
||||
LastInitialization = new InitializationSnapshot(initStopwatch.Elapsed.TotalMilliseconds, FullPassRan: true);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unconditional full pass: CreateTableIfNotExists for every table in the registry.
|
||||
/// Used by the startup slow path and by daily maintenance (recreates manually deleted tables).
|
||||
/// Returns the number of tables that failed to initialize.
|
||||
/// </summary>
|
||||
public async Task<int> EnsureAllTablesAsync()
|
||||
{
|
||||
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
|
||||
var successCount = 0;
|
||||
var failCount = 0;
|
||||
@@ -126,9 +193,74 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// CPE mapping seed is imported via Admin UI "Re-Seed Mappings" button
|
||||
// (pulls from GitHub, not embedded resource). No auto-import at startup.
|
||||
|
||||
lock (_initLock)
|
||||
return failCount;
|
||||
}
|
||||
|
||||
private async Task<bool> SchemaSentinelMatchesAsync(string expectedHash)
|
||||
{
|
||||
try
|
||||
{
|
||||
_tablesInitialized = failCount == 0;
|
||||
var table = _tableServiceClient.GetTableClient(Constants.TableNames.AdminConfiguration);
|
||||
var response = await table.GetEntityIfExistsAsync<TableEntity>(
|
||||
SchemaSentinelPartitionKey, SchemaSentinelRowKey, select: new[] { SchemaSentinelHashProperty });
|
||||
if (!response.HasValue) return false;
|
||||
return string.Equals(response.Value!.GetString(SchemaSentinelHashProperty), expectedHash, StringComparison.Ordinal);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Table missing (fresh storage) or transient error — fall back to the full pass.
|
||||
_logger.LogInformation(ex, "Table schema sentinel not readable — running full table initialization");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteSchemaSentinelAsync(string hash)
|
||||
{
|
||||
try
|
||||
{
|
||||
var table = _tableServiceClient.GetTableClient(Constants.TableNames.AdminConfiguration);
|
||||
var entity = new TableEntity(SchemaSentinelPartitionKey, SchemaSentinelRowKey)
|
||||
{
|
||||
[SchemaSentinelHashProperty] = hash,
|
||||
["TableCount"] = Constants.TableNames.All.Length,
|
||||
["UpdatedUtc"] = DateTime.UtcNow
|
||||
};
|
||||
await table.UpsertEntityAsync(entity, TableUpdateMode.Replace);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Non-fatal: the next start simply runs the full pass again.
|
||||
_logger.LogWarning(ex, "Failed to write table schema sentinel");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scale-out guard for the one-time SessionsIndex backfill: a conditional insert that
|
||||
/// only one instance can win. Returns true for the winner. A lost race (409) or any
|
||||
/// error yields false — the manual maintenance backfill remains the safety net.
|
||||
/// </summary>
|
||||
public async Task<bool> TryClaimSessionIndexBackfillAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var table = _tableServiceClient.GetTableClient(Constants.TableNames.AdminConfiguration);
|
||||
var entity = new TableEntity(SchemaSentinelPartitionKey, SessionIndexBackfillClaimRowKey)
|
||||
{
|
||||
["ClaimedUtc"] = DateTime.UtcNow,
|
||||
["Instance"] = Environment.MachineName
|
||||
};
|
||||
await table.AddEntityAsync(entity);
|
||||
return true;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
_logger.LogInformation("SessionsIndex backfill already claimed by another instance");
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to claim SessionsIndex backfill");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -136,6 +136,9 @@ describe('role catalog snapshot — privilege-leak guard', () => {
|
||||
// routing to the GlobalReadOrAdmin variant — same placement as get_app_install_metrics.
|
||||
'get_time_attribution',
|
||||
'get_usage_metrics',
|
||||
// Platform scope (GA + Global Reader): GlobalReadOrAdmin endpoint, operator-only
|
||||
// classifier diagnostics.
|
||||
'get_verdict_calibration',
|
||||
'get_vulnerability_summary',
|
||||
'list_blocked_devices',
|
||||
'list_session_annotations',
|
||||
@@ -197,6 +200,7 @@ describe('role catalog snapshot — privilege-leak guard', () => {
|
||||
'get_platform_metrics',
|
||||
'get_tenant_config',
|
||||
'get_tenant_config_schema',
|
||||
'get_verdict_calibration',
|
||||
'list_blocked_devices',
|
||||
'list_session_annotations',
|
||||
'list_session_reports',
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { shapeVerdictCalibration, type VerdictCalibrationPathRow } from '../verdict-calibration-shape.js';
|
||||
|
||||
function row(verdictPath: string, count: number, sharePct: number, extra: Partial<VerdictCalibrationPathRow> = {}): VerdictCalibrationPathRow {
|
||||
return {
|
||||
verdictPath,
|
||||
status: 'Succeeded',
|
||||
count,
|
||||
sharePct,
|
||||
overriddenByAdmin: 0,
|
||||
overriddenByLateCompletion: 0,
|
||||
overriddenOther: 0,
|
||||
window7: { count, sessions: 1000, sharePct },
|
||||
baseline28: { count, sessions: 2000, sharePct },
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function payload(paths: VerdictCalibrationPathRow[]) {
|
||||
return {
|
||||
success: true,
|
||||
totals: { sessions: 1000 },
|
||||
trend: { windowSessions: 1000, baselineSessions: 2000 },
|
||||
paths,
|
||||
alerts: [],
|
||||
};
|
||||
}
|
||||
|
||||
describe('shapeVerdictCalibration', () => {
|
||||
it('drops the per-row denominators and makes withheld values explicit null', () => {
|
||||
const out = shapeVerdictCalibration(payload([row('agent:complete', 500, 50)])) as any;
|
||||
expect(out.paths[0].window7).toEqual({ count: 500, sharePct: 50 });
|
||||
expect(out.paths[0].baseline28).toEqual({ count: 500, sharePct: 50 });
|
||||
expect(out.paths[0]).toHaveProperty('reEnrollRatePct', null);
|
||||
expect(out.paths[0]).toHaveProperty('lift', null);
|
||||
expect(out.omitted).toBeUndefined();
|
||||
expect(out.trend).toEqual({ windowSessions: 1000, baselineSessions: 2000 });
|
||||
});
|
||||
|
||||
it('keeps real numeric values (0 is not null)', () => {
|
||||
const out = shapeVerdictCalibration(payload([row('legacy:r3', 4, 0.1, { reEnrollRatePct: 0, lift: 0 })])) as any;
|
||||
expect(out.paths[0].reEnrollRatePct).toBe(0);
|
||||
expect(out.paths[0].lift).toBe(0);
|
||||
});
|
||||
|
||||
it('minSharePct trims the long tail and reports what was omitted', () => {
|
||||
const out = shapeVerdictCalibration(payload([
|
||||
row('agent:complete', 900, 90),
|
||||
row('legacy:r4', 10, 1),
|
||||
row('manual:succeeded', 1, 0.1),
|
||||
]), { minSharePct: 1 }) as any;
|
||||
expect(out.paths.map((p: any) => p.verdictPath)).toEqual(['agent:complete', 'legacy:r4']);
|
||||
expect(out.omitted).toEqual({ paths: 1, sessions: 1, reason: expect.stringContaining('share < 1%') });
|
||||
});
|
||||
|
||||
it('top keeps the N largest rows by backend order', () => {
|
||||
const out = shapeVerdictCalibration(payload([
|
||||
row('a', 50, 50), row('b', 30, 30), row('c', 20, 20),
|
||||
]), { top: 2 }) as any;
|
||||
expect(out.paths.map((p: any) => p.verdictPath)).toEqual(['a', 'b']);
|
||||
expect(out.omitted.paths).toBe(1);
|
||||
expect(out.omitted.reason).toContain('rank > 2');
|
||||
});
|
||||
|
||||
it('never trims a row that carries overrides — that is the signal', () => {
|
||||
const out = shapeVerdictCalibration(payload([
|
||||
row('agent:complete', 999, 99.9),
|
||||
row('sweep:r5_incomplete', 1, 0.1, { overriddenByLateCompletion: 3 }),
|
||||
]), { minSharePct: 5, top: 1 }) as any;
|
||||
expect(out.paths.map((p: any) => p.verdictPath)).toEqual(['agent:complete', 'sweep:r5_incomplete']);
|
||||
expect(out.omitted).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not mutate the input and passes non-matrix payloads through', () => {
|
||||
const input = payload([row('x', 1, 100)]);
|
||||
const snapshot = JSON.stringify(input);
|
||||
shapeVerdictCalibration(input, { top: 0 });
|
||||
expect(JSON.stringify(input)).toBe(snapshot);
|
||||
const err = { success: false, message: 'nope' };
|
||||
expect(shapeVerdictCalibration(err)).toBe(err);
|
||||
expect(shapeVerdictCalibration(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -476,6 +476,17 @@ const server = app.listen(PORT, '0.0.0.0', () => {
|
||||
// The number a user actually waits for on a cold start, minus the platform's
|
||||
// own activation (pod scheduling, container start), which is not visible here.
|
||||
console.error(`[boot ${bootMark()}] Autopilot-Monitor MCP Server listening on port ${PORT} — ready to serve.`);
|
||||
// Same number as a structured line so it is a trend, not prose: queryable in
|
||||
// ContainerAppConsoleLogs_CL next to the tool_call lines (see telemetry.ts),
|
||||
// e.g. `| where Log_s startswith '{"type":"boot"'`. Always on — one line per start.
|
||||
console.error(JSON.stringify({
|
||||
type: 'boot',
|
||||
readyMs: Math.round(process.uptime() * 1000),
|
||||
version: SERVER_VERSION,
|
||||
commit: BUILD_COMMIT,
|
||||
docsCommit: DOCS_COMMIT,
|
||||
timestamp: new Date().toISOString(),
|
||||
}));
|
||||
});
|
||||
|
||||
// --- Graceful shutdown ---
|
||||
|
||||
@@ -5,6 +5,7 @@ import { withToolTelemetry } from '../telemetry.js';
|
||||
import { getResourceContent, assertKnownEventType } from '../resource-catalog.js';
|
||||
import { READ_ONLY, READ_ONLY_OPEN, MUTATING, MAX_RESULT_SIZE_CHARS, toolResultText, SessionIdSchema, TenantGuidSchema, tenantIdDescription } from './shared.js';
|
||||
import { toolError } from './error-handler.js';
|
||||
import { shapeVerdictCalibration } from '../verdict-calibration-shape.js';
|
||||
|
||||
/**
|
||||
* Non-sensitive tenant fields safe to surface to the model. Keep-list (not
|
||||
@@ -838,6 +839,53 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo
|
||||
})
|
||||
);
|
||||
|
||||
// Tool: get_verdict_calibration — platform scope (GA + Global Reader). Operator diagnostics for
|
||||
// the session-verdict classifier (docs/backend/verdict-calibration.md) — never a customer surface.
|
||||
if (ga) server.registerTool(
|
||||
'get_verdict_calibration',
|
||||
{
|
||||
title: 'Verdict Calibration (classifier thermometer)',
|
||||
description:
|
||||
'Answer "which code path produced our session verdicts, and does any of them look wrong?". Per verdict ' +
|
||||
'path (VerdictPaths vocabulary — agent:complete, agent:failed, sweep:r5_incomplete, maxlife:r6, ' +
|
||||
'late:r4, retro:r6, rule:<id>, manual:failed, register:superseded, legacy:* for pre-instrumentation ' +
|
||||
'rows derived read-side): session count and share in the window, derivedCount (rows attributed by ' +
|
||||
'derivation — weaker evidence), the 7-day re-enrollment proxy (eligible7d = terminal sessions old ' +
|
||||
'enough to judge, reEnrolled7d = the same device registered another terminal session within 7 days; ' +
|
||||
'reEnrollRatePct is null below 20 eligible), the correction stream attributed to the PRIOR path ' +
|
||||
'(overriddenByAdmin / overriddenByLateCompletion / overriddenOther — a path that is frequently ' +
|
||||
'overridden was wrong), and a 7d-vs-28d share trend (window7, baseline28, lift; lift null without ' +
|
||||
'baseline; window/baseline denominators are the single `trend.windowSessions` / `trend.baselineSessions`). ' +
|
||||
'reEnrollRatePct and lift are always present — null means withheld, not missing. Read it as calibration: ' +
|
||||
'a Succeeded path whose re-enrollment rate sits at Failed level reconciles too boldly; an Incomplete ' +
|
||||
'path whose rate matches the Succeeded background is too cautious; a rising sweep:*/maxlife:* share ' +
|
||||
'is an agent-liveness signal, not a classifier one. Omit tenantId for the cross-tenant aggregate, or ' +
|
||||
'pass tenantId to scope to one tenant. minSharePct / top trim the long tail of one-session paths ' +
|
||||
'(rows carrying overrides are never trimmed); what was dropped is reported in `omitted`, totals and ' +
|
||||
'trend always cover the full window. Rows come from the 2-hourly maintenance sweep (computedAt); ' +
|
||||
'empty before its first run.',
|
||||
inputSchema: {
|
||||
tenantId: z.string().optional().describe('Optional — scope to one tenant. Omit for the cross-tenant aggregate.'),
|
||||
days: z.coerce.number().int().min(1).max(180).optional()
|
||||
.describe('Window in days (default 30, max 180 — aggregate retention). The 7d/28d trend is always today-anchored.'),
|
||||
minSharePct: z.coerce.number().min(0).max(100).optional()
|
||||
.describe('Drop paths whose share of the window is below this percentage (e.g. 1). Paths with overrides are always kept.'),
|
||||
top: z.coerce.number().int().min(1).max(100).optional()
|
||||
.describe('Keep only the N largest paths by session count. Paths with overrides are always kept.'),
|
||||
},
|
||||
annotations: READ_ONLY,
|
||||
},
|
||||
async (args) => withToolTelemetry('get_verdict_calibration', args, async () => {
|
||||
try {
|
||||
const { tenantId, days, minSharePct, top } = args;
|
||||
const data = await apiFetch(`/api/global/metrics/verdict-calibration${buildQuery({ tenantId, days })}`);
|
||||
return toolResultText(shapeVerdictCalibration(data, { minSharePct, top }), MAX_RESULT_SIZE_CHARS.adminStream);
|
||||
} catch (error: unknown) {
|
||||
return toolError('get_verdict_calibration', args, error);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Tool 18: list_session_reports — Global Admin only; not registered for normal users.
|
||||
if (ga) server.registerTool(
|
||||
'list_session_reports',
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* LLM-facing shaping of the verdict-calibration matrix (GET /api/global/metrics/verdict-calibration).
|
||||
*
|
||||
* The backend wire shape is shared with the admin metrics page and stays untouched; this layer only
|
||||
* trims what a model does not need and makes absent values explicit:
|
||||
* - `window7.sessions` / `baseline28.sessions` repeat `trend.windowSessions` / `trend.baselineSessions`
|
||||
* on every row — dropped (the denominators live once in `trend`).
|
||||
* - `reEnrollRatePct` / `lift` are omitted by the backend's WhenWritingNull serializer when null;
|
||||
* a missing key is easy to misread as "not applicable", so both are always present (null = withheld).
|
||||
* - `minSharePct` / `top` cut the long tail of one-session paths. Rows carrying overrides are never
|
||||
* cut (an override on a rare path is the signal the tool exists for). What was dropped is reported
|
||||
* in `omitted` so a filtered response never reads as complete coverage.
|
||||
*/
|
||||
|
||||
interface TrendCell {
|
||||
count: number;
|
||||
sessions?: number;
|
||||
sharePct: number;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationPathRow {
|
||||
verdictPath: string;
|
||||
status: string;
|
||||
count: number;
|
||||
sharePct: number;
|
||||
overriddenByAdmin: number;
|
||||
overriddenByLateCompletion: number;
|
||||
overriddenOther: number;
|
||||
reEnrollRatePct?: number | null;
|
||||
lift?: number | null;
|
||||
window7: TrendCell;
|
||||
baseline28: TrendCell;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationShapeOptions {
|
||||
/** Drop rows whose share (in the window) is below this percentage. */
|
||||
minSharePct?: number;
|
||||
/** Keep only the first N rows (the backend orders by count descending). */
|
||||
top?: number;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationOmitted {
|
||||
paths: number;
|
||||
sessions: number;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
function hasOverrides(row: VerdictCalibrationPathRow): boolean {
|
||||
return (row.overriddenByAdmin ?? 0) + (row.overriddenByLateCompletion ?? 0) + (row.overriddenOther ?? 0) > 0;
|
||||
}
|
||||
|
||||
function trimCell(cell: TrendCell | undefined): TrendCell {
|
||||
if (!cell) return { count: 0, sharePct: 0 };
|
||||
const { sessions: _sessions, ...rest } = cell;
|
||||
return rest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: returns a new object; the input is not mutated. Non-object or unsuccessful payloads
|
||||
* (error envelopes) pass through unchanged so the caller's error handling still sees them.
|
||||
*/
|
||||
export function shapeVerdictCalibration(data: unknown, options: VerdictCalibrationShapeOptions = {}): unknown {
|
||||
if (!data || typeof data !== 'object' || !Array.isArray((data as { paths?: unknown }).paths)) return data;
|
||||
const payload = data as { paths: VerdictCalibrationPathRow[] } & Record<string, unknown>;
|
||||
|
||||
const { minSharePct, top } = options;
|
||||
const kept: VerdictCalibrationPathRow[] = [];
|
||||
let omittedPaths = 0;
|
||||
let omittedSessions = 0;
|
||||
|
||||
payload.paths.forEach((row, index) => {
|
||||
const belowShare = minSharePct != null && row.sharePct < minSharePct;
|
||||
const beyondTop = top != null && index >= top;
|
||||
if ((belowShare || beyondTop) && !hasOverrides(row)) {
|
||||
omittedPaths++;
|
||||
omittedSessions += row.count ?? 0;
|
||||
return;
|
||||
}
|
||||
kept.push({
|
||||
...row,
|
||||
reEnrollRatePct: row.reEnrollRatePct ?? null,
|
||||
lift: row.lift ?? null,
|
||||
window7: trimCell(row.window7),
|
||||
baseline28: trimCell(row.baseline28),
|
||||
});
|
||||
});
|
||||
|
||||
const shaped: Record<string, unknown> = { ...payload, paths: kept };
|
||||
if (omittedPaths > 0) {
|
||||
const criteria = [
|
||||
minSharePct != null ? `share < ${minSharePct}%` : null,
|
||||
top != null ? `rank > ${top}` : null,
|
||||
].filter(Boolean).join(' or ');
|
||||
shaped.omitted = {
|
||||
paths: omittedPaths,
|
||||
sessions: omittedSessions,
|
||||
reason: `${omittedPaths} path(s) with ${criteria} dropped (rows carrying overrides are always kept); totals and trend still cover the full window.`,
|
||||
} satisfies VerdictCalibrationOmitted;
|
||||
}
|
||||
return shaped;
|
||||
}
|
||||
@@ -988,6 +988,11 @@ namespace AutopilotMonitor.Shared
|
||||
// like UsageMetrics.
|
||||
public const string DeviceHistories = "DeviceHistories";
|
||||
public const string DeviceJourneyAggregates = "DeviceJourneyAggregates";
|
||||
// VerdictCalibrationAggregates: PK=TenantId (or "global"), RK="{yyyy-MM-dd}" — daily
|
||||
// per-verdict-path buckets (count, derived, re-enrolled-within-7d, overridden-by-admin/
|
||||
// late-completion) the operator calibration matrix + drift radar read
|
||||
// (docs/backend/verdict-calibration.md). Additive across days; 180d retention; regenerable.
|
||||
public const string VerdictCalibrationAggregates = "VerdictCalibrationAggregates";
|
||||
|
||||
// Configuration tables
|
||||
public const string TenantConfiguration = "TenantConfiguration";
|
||||
@@ -1198,6 +1203,7 @@ namespace AutopilotMonitor.Shared
|
||||
TimeAttributionAggregates,
|
||||
DeviceHistories,
|
||||
DeviceJourneyAggregates,
|
||||
VerdictCalibrationAggregates,
|
||||
TenantConfiguration,
|
||||
AdminConfiguration,
|
||||
GlobalAdmins,
|
||||
|
||||
+14
@@ -55,6 +55,20 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// <summary>Active regression episodes of one tenant (RowKey-prefix scan; empty on failure — fail-soft reads).</summary>
|
||||
Task<List<RuleRegressionAlert>> GetRuleRegressionsAsync(string tenantId);
|
||||
|
||||
// --- Verdict-calibration drift (RowKey "verdictcalibration|{kind}|{path}|{status}") ---
|
||||
|
||||
/// <summary>Atomically opens a verdict-calibration episode; true = this pass owns it (fires the ops event once). Fail-closed.</summary>
|
||||
Task<bool> TryRegisterVerdictCalibrationAlertAsync(string tenantId, VerdictCalibrationAlert alert);
|
||||
|
||||
/// <summary>Refreshes an active episode's numbers; FirstNotifiedAt carried unchanged. Fail-soft.</summary>
|
||||
Task RefreshVerdictCalibrationAlertAsync(string tenantId, VerdictCalibrationAlert alert);
|
||||
|
||||
/// <summary>Closes an episode (re-armed). 404-tolerant.</summary>
|
||||
Task DeleteVerdictCalibrationAlertAsync(string tenantId, string kind, string verdictPath, string status);
|
||||
|
||||
/// <summary>Active verdict-calibration episodes of one tenant partition ("global" allowed); empty on failure.</summary>
|
||||
Task<List<VerdictCalibrationAlert>> GetVerdictCalibrationAlertsAsync(string tenantId);
|
||||
|
||||
// --- App-version duration regressions (RowKey "appversionregression|{app}|{version}") ---
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -80,6 +80,13 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// Fields outside the projection come back as defaults — callers must not read them.
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetGeoWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null);
|
||||
/// <summary>
|
||||
/// Column-projected cross-tenant variant of <see cref="GetSessionsByDateRangeAsync"/> that
|
||||
/// the maintenance tick drains ONCE and shares across its rolling sweeps (time attribution,
|
||||
/// device journeys, verdict calibration, calibration radar discovery). Fields outside the
|
||||
/// projection come back as defaults — sweeps must not read them.
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetMaintenanceWindowSessionsAsync(DateTime startDate, DateTime endDate);
|
||||
Task<List<SessionSummary>> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime);
|
||||
Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff);
|
||||
|
||||
@@ -92,6 +99,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults);
|
||||
|
||||
/// <summary>
|
||||
/// Self-deploying-profile sessions parked in a non-success, non-failure state
|
||||
/// (Incomplete / AwaitingUser / Stalled) — candidates for the one-time retro-reconcile
|
||||
/// through <c>EnrollmentTimeoutClassifier.IsSelfDeployingProvisioned</c> (kiosk tenant
|
||||
/// audit 2026-08-23). Server-side filter, capped at <paramref name="maxResults"/>.
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetSelfDeployingSilentSessionsAsync(string tenantId, int maxResults);
|
||||
|
||||
/// <summary>
|
||||
/// Narrow projection of every session row in the tenant partition (id, status, timing,
|
||||
/// serial). Used by the Pending-orphan resolution to match superseding sessions of the
|
||||
|
||||
@@ -128,6 +128,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// <summary>Deletes one daily FTR row — the sweep's stale-bucket reconcile.</summary>
|
||||
Task DeleteDeviceJourneyAggregateAsync(string tenantId, string dateKey);
|
||||
Task<int> DeleteDeviceJourneyAggregatesOlderThanAsync(DateTime cutoffDate);
|
||||
|
||||
// --- Verdict calibration (docs/backend/verdict-calibration.md) ---
|
||||
Task<bool> SaveVerdictCalibrationAggregateAsync(VerdictCalibrationDailyAggregate aggregate);
|
||||
/// <summary>Daily verdict-path rows of one tenant partition ("global" allowed), inclusive date range; counts are additive across days.</summary>
|
||||
Task<List<VerdictCalibrationDailyAggregate>> GetVerdictCalibrationAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate);
|
||||
/// <summary>Deletes one daily row — the sweep's stale-bucket reconcile.</summary>
|
||||
Task DeleteVerdictCalibrationAggregateAsync(string tenantId, string dateKey);
|
||||
Task<int> DeleteVerdictCalibrationAggregatesOlderThanAsync(DateTime cutoffDate);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -83,8 +83,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
string? tenantIdFilter, int days, IReadOnlyCollection<string>? allowedTenantIds = null);
|
||||
|
||||
// --- Session Updates ---
|
||||
/// <summary>
|
||||
/// Single status-write seam. <paramref name="verdictPath"/> is REQUIRED: every writer names
|
||||
/// the code path that produced the status (<see cref="VerdictPaths"/>), stamped for all
|
||||
/// statuses so the verdict calibration aggregate can count each path. Overriding a prior
|
||||
/// verdict preserves it in PriorStatus/PriorVerdictPath.
|
||||
/// </summary>
|
||||
Task<bool> UpdateSessionStatusAsync(
|
||||
string tenantId, string sessionId, SessionStatus status,
|
||||
string tenantId, string sessionId, SessionStatus status, string verdictPath,
|
||||
EnrollmentPhase? currentPhase = null, string? failureReason = null,
|
||||
DateTime? completedAt = null, DateTime? earliestEventTimestamp = null,
|
||||
DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null,
|
||||
@@ -123,7 +129,7 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
Task UpdateSessionDiagnosticsBlobAsync(
|
||||
string tenantId, string sessionId, string blobName, string? destination = null);
|
||||
Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned,
|
||||
SessionStatus? status = null, bool? isUserDriven = null);
|
||||
SessionStatus? status = null, bool? isUserDriven = null, string? verdictPath = null);
|
||||
Task UpdateSessionGeoAsync(string tenantId, string sessionId,
|
||||
string? country, string? region, string? city, string? loc);
|
||||
Task UpdateSessionImeAgentVersionAsync(string tenantId, string sessionId, string version);
|
||||
|
||||
@@ -8,6 +8,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// </summary>
|
||||
public interface IStorageInitializer
|
||||
{
|
||||
/// <summary>Startup path: cheap when the storage schema is already known to be current.</summary>
|
||||
Task InitializeAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Unconditional full pass that (re)creates every registered table. Used by daily
|
||||
/// maintenance to repair tables that were deleted out-of-band after the startup
|
||||
/// fast path stopped checking them.
|
||||
/// </summary>
|
||||
Task EnsureAllAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>Kinds of verdict-calibration drift the radar detects (docs/backend/verdict-calibration.md).</summary>
|
||||
public static class VerdictCalibrationAlertKinds
|
||||
{
|
||||
/// <summary>One verdict path's share of all sessions rose ≥2× over its 28-day baseline (Wilson-separated).</summary>
|
||||
public const string ShareRegression = "share_regression";
|
||||
|
||||
/// <summary>The combined sweep:* + maxlife:* share (agent went silent, backend had to decide) rose ≥2× — an agent-liveness signal, not a classifier one.</summary>
|
||||
public const string SilenceShareRegression = "silence_share_regression";
|
||||
|
||||
/// <summary>The pure fallthrough rule (r6) decides ≥20 % of classifier verdicts — the evidence the rules need is missing.</summary>
|
||||
public const string EvidenceGap = "evidence_gap";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One ACTIVE verdict-calibration alert episode (operator-only). Persisted as the
|
||||
/// <c>verdictcalibration|{kind}|{path}|{status}</c> keyspace of the notification tracker —
|
||||
/// the row IS the dedup (one ops event per episode) and the <c>alerts[]</c> payload of the
|
||||
/// calibration endpoint. Deleted when the signal re-arms (share back under 1.5× baseline,
|
||||
/// path stops occurring, evidence-gap share back under 15 %) or by the tracker's 30-day
|
||||
/// retention sweep. Numbers refresh on every radar pass; <see cref="FirstNotifiedAt"/> never moves.
|
||||
/// Dimension concentration is CORRELATION only — every consumer says so.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationAlert
|
||||
{
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>One of <see cref="VerdictCalibrationAlertKinds"/>.</summary>
|
||||
public string Kind { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Verdict path, or the group label for group kinds ("sweep+maxlife", "r6").</summary>
|
||||
public string VerdictPath { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Status the path produced, or "*" for group kinds.</summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
public int WindowHitCount { get; set; }
|
||||
public int WindowSessionCount { get; set; }
|
||||
public int BaselineHitCount { get; set; }
|
||||
public int BaselineSessionCount { get; set; }
|
||||
public double WindowRatePct { get; set; }
|
||||
public double BaselineRatePct { get; set; }
|
||||
|
||||
/// <summary>Window rate ÷ baseline rate. Null when the baseline rate is 0 or the kind is absolute (evidence gap).</summary>
|
||||
public double? Lift { get; set; }
|
||||
|
||||
public string WindowStartDate { get; set; } = string.Empty;
|
||||
public string WindowEndDate { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Dimension concentration captured when the alert FIRST fired; null = no clear concentration.</summary>
|
||||
public RuleRegressionDimension? Dimension { get; set; }
|
||||
|
||||
public DateTime FirstNotifiedAt { get; set; }
|
||||
public DateTime LastEvaluatedAt { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// One (verdict path × status) bucket of a daily verdict-calibration row. <c>Count</c> is the
|
||||
/// sessions whose CURRENT status was produced by this path; the <c>Overridden*</c> counters
|
||||
/// are sessions that once carried this path and were then overridden (their
|
||||
/// <c>PriorVerdictPath</c> names this bucket) — the correction stream. Re-enrollment is the
|
||||
/// delayed proxy signal: of the sessions old enough to judge (<see cref="Eligible7d"/>), how
|
||||
/// many saw the same device register another terminal session within 7 days.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationBucket
|
||||
{
|
||||
/// <summary><see cref="VerdictPaths"/> value, or the read-side derivation for pre-instrumentation rows.</summary>
|
||||
public string VerdictPath { get; set; } = string.Empty;
|
||||
|
||||
/// <summary><see cref="SessionStatus"/> name the path produced.</summary>
|
||||
public string Status { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Sessions currently on this path/status.</summary>
|
||||
public int Count { get; set; }
|
||||
|
||||
/// <summary>Of <see cref="Count"/>: rows without a stamped VerdictPath, attributed by <c>VerdictPathDerivation</c> (weaker evidence).</summary>
|
||||
public int DerivedCount { get; set; }
|
||||
|
||||
/// <summary>Of <see cref="Count"/>: sessions whose terminal moment lies ≥ 7 days before the compute time — the re-enrollment denominator.</summary>
|
||||
public int Eligible7d { get; set; }
|
||||
|
||||
/// <summary>Of <see cref="Eligible7d"/>: the device registered another terminal session within 7 days of this one's end.</summary>
|
||||
public int ReEnrolled7d { get; set; }
|
||||
|
||||
/// <summary>Sessions that carried this path and were then flipped by an administrator (MarkSessionSucceeded/Failed).</summary>
|
||||
public int OverriddenByAdmin { get; set; }
|
||||
|
||||
/// <summary>Sessions that carried this path and were then upgraded to Succeeded by a late agent completion report.</summary>
|
||||
public int OverriddenByLateCompletion { get; set; }
|
||||
|
||||
/// <summary>Sessions that carried this path and were overridden by any other writer (retro-reclassification, grace expiry, supersede).</summary>
|
||||
public int OverriddenOther { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Daily verdict-calibration rollup for one (tenant, date) — the operator's thermometer for the
|
||||
/// rule classifier (docs/backend/verdict-calibration.md). A session buckets on its StartedAt
|
||||
/// date (same convention as every other daily aggregate; the rolling sweep re-buckets
|
||||
/// late-terminating sessions idempotently). Every status is counted — non-terminal rows are
|
||||
/// part of the picture (a growing Stalled share is a signal) — and
|
||||
/// <see cref="TerminalSessionCount"/> carries the share denominator for terminal paths.
|
||||
/// Counts are additive across days; a window is the sum of daily rows. "global" mirror rows
|
||||
/// sum all tenants. Regenerable from Sessions + DeviceHistories at any time.
|
||||
/// </summary>
|
||||
public class VerdictCalibrationDailyAggregate
|
||||
{
|
||||
/// <summary>Bucketing/derivation algorithm version — a definition change never silently mixes semantics.</summary>
|
||||
public const int CurrentVersion = 1;
|
||||
|
||||
/// <summary>Calendar day (UTC, "yyyy-MM-dd") the sessions STARTED.</summary>
|
||||
public string Date { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Tenant GUID, or "global" for the cross-tenant row.</summary>
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
public int Version { get; set; } = CurrentVersion;
|
||||
|
||||
/// <summary>All sessions started on this date (any status).</summary>
|
||||
public int SessionCount { get; set; }
|
||||
|
||||
/// <summary>Sessions whose status is Succeeded / Failed / Incomplete — the share denominator for terminal paths.</summary>
|
||||
public int TerminalSessionCount { get; set; }
|
||||
|
||||
/// <summary>Ordered by VerdictPath, then Status (ordinal) for deterministic output.</summary>
|
||||
public List<VerdictCalibrationBucket> Buckets { get; set; } = new List<VerdictCalibrationBucket>();
|
||||
|
||||
public DateTime ComputedAt { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -220,14 +220,37 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public string FailureReason { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Origin of a Failed status. Values:
|
||||
/// Origin of a Failed status — persisted ONLY on a Failed write (a non-Failed verdict
|
||||
/// discards it; see <see cref="VerdictPath"/> for the status-independent attribution). Values:
|
||||
/// - "" / null: agent-reported (default; terminal enrollment_failed event)
|
||||
/// - "rule:<RuleId>": session failed because an analyze rule with MarkSessionAsFailed fired
|
||||
/// - "manual": operator flipped the session via the portal
|
||||
/// - "max_lifetime_watchdog": the agent's max-lifetime shutdown verdict was Failed
|
||||
/// Consumers use this to render rule-based failures distinctly (badge + link to rule).
|
||||
/// </summary>
|
||||
public string FailureSource { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Machine-readable origin of the CURRENT <see cref="Status"/> — which code path wrote it
|
||||
/// (<see cref="VerdictPaths"/> vocabulary, e.g. <c>agent:complete</c>, <c>sweep:r5_incomplete</c>,
|
||||
/// <c>manual:failed</c>). Stamped on every status write regardless of status, so the verdict
|
||||
/// calibration aggregate can count each path. Null on rows written before instrumentation
|
||||
/// (read-side derivation covers those).
|
||||
/// </summary>
|
||||
public string? VerdictPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The verdict this session carried BEFORE the current status overrode it — set only when a
|
||||
/// write replaces a prior Succeeded/Failed/Incomplete/AwaitingUser (admin mark, late agent
|
||||
/// completion upgrading a sweep verdict, retro-reclassification). Together with
|
||||
/// <see cref="PriorVerdictPath"/> this is the correction stream: "path X was overridden".
|
||||
/// Null when the status never overrode a verdict.
|
||||
/// </summary>
|
||||
public string? PriorStatus { get; set; }
|
||||
|
||||
/// <summary>The <see cref="VerdictPath"/> of the overridden prior verdict; see <see cref="PriorStatus"/>.</summary>
|
||||
public string? PriorVerdictPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Non-empty only when the BACKEND (not the agent) declared this session Succeeded:
|
||||
/// either the maintenance timeout sweep reconciled it (e.g. "user completed setup —
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Machine-readable vocabulary for <see cref="SessionSummary.VerdictPath"/> — WHICH code path
|
||||
/// produced the session's current <see cref="SessionStatus"/>. Format is
|
||||
/// <c>origin:detail</c>, lower-case, stable: values are a persisted contract that the verdict
|
||||
/// calibration aggregate (docs/backend/verdict-calibration.md) counts over days and tenants, so
|
||||
/// entries are append-only — never rename or reuse one. Unlike <c>FailureSource</c> (written
|
||||
/// only on Failed) the path is stamped for every status write, so a sweep "Incomplete", a late
|
||||
/// agent upgrade and an admin override are all countable.
|
||||
/// </summary>
|
||||
public static class VerdictPaths
|
||||
{
|
||||
// ── Agent-declared (ingest of the agent's own terminal/phase events) ──────────────────
|
||||
public const string AgentComplete = "agent:complete";
|
||||
public const string AgentCompleteSoft = "agent:complete_soft";
|
||||
public const string AgentFailed = "agent:failed";
|
||||
public const string AgentEspFailureFallback = "agent:esp_failure_fallback";
|
||||
public const string AgentGatherComplete = "agent:gather_complete";
|
||||
public const string AgentWhiteGlovePending = "agent:whiteglove_pending";
|
||||
public const string AgentWhiteGloveResumed = "agent:whiteglove_resumed";
|
||||
public const string AgentStallProbe = "agent:stall_probe";
|
||||
public const string AgentStallHeal = "agent:stall_heal";
|
||||
|
||||
// ── Backend-declared, non-classifier ─────────────────────────────────────────────────
|
||||
public const string IngestWhiteGloveAwaiting = "ingest:wg_awaiting";
|
||||
public const string SweepStalled = "sweep:stalled";
|
||||
public const string SweepWhiteGloveAwaiting = "sweep:wg_awaiting";
|
||||
public const string SweepSelfDeployingReconcile = "sweep:sd_reconcile";
|
||||
public const string RetroSelfDeployingReconcile = "retro:sd_reconcile";
|
||||
public const string RetroSuperseded = "retro:superseded";
|
||||
public const string RegisterSuperseded = "register:superseded";
|
||||
/// <summary>Status InProgress set by a fresh session registration.</summary>
|
||||
public const string RegisterNew = "register:new";
|
||||
/// <summary>WhiteGlove Part-2 re-registration flipped Pending to InProgress.</summary>
|
||||
public const string RegisterWhiteGloveResume = "register:whiteglove_resume";
|
||||
public const string ManualFailed = "manual:failed";
|
||||
public const string ManualSucceeded = "manual:succeeded";
|
||||
|
||||
/// <summary>Read-side derivation could not attribute a pre-instrumentation row. Never written.</summary>
|
||||
public const string LegacyUnknown = "legacy:unknown";
|
||||
|
||||
// ── Origins whose detail is a classifier rule id (see <see cref="ClassifierRules"/>) ──
|
||||
public const string OriginMaxLifetime = "maxlife";
|
||||
public const string OriginLateReconcile = "late";
|
||||
public const string OriginSweep = "sweep";
|
||||
public const string OriginRetro = "retro";
|
||||
public const string OriginRule = "rule";
|
||||
|
||||
/// <summary><c>{origin}:{detail}</c> — e.g. <c>sweep:r5_incomplete</c>, <c>rule:ANALYZE-ESP-001</c>.</summary>
|
||||
public static string Compose(string origin, string detail)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(origin)) throw new ArgumentException("origin required", nameof(origin));
|
||||
if (string.IsNullOrWhiteSpace(detail)) throw new ArgumentException("detail required", nameof(detail));
|
||||
return origin + ":" + detail;
|
||||
}
|
||||
|
||||
/// <summary>The <c>origin</c> half of a path (<c>"sweep"</c> for <c>"sweep:r6"</c>); the whole string when there is no colon.</summary>
|
||||
public static string Origin(string path)
|
||||
{
|
||||
if (string.IsNullOrEmpty(path)) return string.Empty;
|
||||
var i = path.IndexOf(':');
|
||||
return i < 0 ? path : path.Substring(0, i);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True for a verdict produced by a silence/timeout classifier RULE (<c>sweep:r5_incomplete</c>,
|
||||
/// <c>maxlife:r1</c>, <c>late:r4</c>, <c>retro:r6</c>) — the rows whose calibration drives rule
|
||||
/// tuning. Non-rule verdicts from the same origins (<c>sweep:stalled</c>, <c>retro:superseded</c>)
|
||||
/// are excluded.
|
||||
/// </summary>
|
||||
public static bool IsClassifierPath(string path)
|
||||
{
|
||||
if (string.IsNullOrEmpty(path)) return false;
|
||||
var origin = Origin(path);
|
||||
if (origin != OriginSweep && origin != OriginMaxLifetime && origin != OriginLateReconcile && origin != OriginRetro)
|
||||
return false;
|
||||
var detail = path.Length > origin.Length + 1 ? path.Substring(origin.Length + 1) : string.Empty;
|
||||
return detail.Length >= 2 && detail[0] == 'r' && char.IsDigit(detail[1]);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rule ids of <c>EnrollmentTimeoutClassifier.ClassifyTimedOutSession</c> — the <c>detail</c> half
|
||||
/// of a classifier verdict path. One id per <c>return</c> so the calibration matrix attributes a
|
||||
/// verdict to the exact rule, including the grace-dependent forks (1b, 5).
|
||||
/// </summary>
|
||||
public static class ClassifierRules
|
||||
{
|
||||
public const string R1ExplicitFailure = "r1";
|
||||
public const string R1bWhiteGloveAwaiting = "r1b_awaiting";
|
||||
public const string R1bWhiteGloveSucceeded = "r1b_succeeded";
|
||||
public const string R1cSelfDeploying = "r1c";
|
||||
public const string R2AccountSetupComplete = "r2";
|
||||
public const string R3EmergencyBreak = "r3";
|
||||
public const string R4DesktopHello = "r4";
|
||||
public const string R5DeviceSetupAwaiting = "r5_awaiting";
|
||||
public const string R5DeviceSetupIncomplete = "r5_incomplete";
|
||||
public const string R6Fallthrough = "r6";
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,11 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
"AppHomingFlippedWithEntraRoles",
|
||||
],
|
||||
Maintenance: [
|
||||
// Verdict-calibration drift radar (operator-only classifier diagnostics): a verdict path's
|
||||
// session share doubled over its 28d baseline, the silence share doubled, or the r6
|
||||
// fallthrough decides ≥20% of classifier verdicts. Warning-tier, once per episode
|
||||
// (tracker-deduped). Dual-register per memory feedback_ops_event_types_dual_register.
|
||||
"VerdictCalibrationDrift",
|
||||
"MaintenanceCompleted",
|
||||
"MaintenanceFailed",
|
||||
// Soft watchdog (Warning): a maintenance run completed but exceeded the 10min threshold and is
|
||||
|
||||
@@ -5,11 +5,13 @@ import { type MetricsSectionId } from "../metricsNavSections";
|
||||
import { SectionAgentMetrics } from "../sections/SectionAgentMetrics";
|
||||
import { SectionPlatformUsage } from "../sections/SectionPlatformUsage";
|
||||
import { SectionMcpUsage } from "../sections/SectionMcpUsage";
|
||||
import { SectionVerdictCalibration } from "../sections/SectionVerdictCalibration";
|
||||
|
||||
const SECTION_COMPONENTS: Record<MetricsSectionId, React.ComponentType> = {
|
||||
"platform-metrics": SectionAgentMetrics,
|
||||
"usage": SectionPlatformUsage,
|
||||
"mcp-usage": SectionMcpUsage,
|
||||
"verdict-calibration": SectionVerdictCalibration,
|
||||
};
|
||||
|
||||
export function SectionClient({ section }: { section: string }) {
|
||||
|
||||
@@ -2,6 +2,7 @@ export const METRICS_NAV_SECTIONS = [
|
||||
{ id: "platform-metrics", label: "Platform Metrics", description: "Agent performance, delivery latency, crash rates, and platform health" },
|
||||
{ id: "usage", label: "Platform Usage", description: "Platform usage statistics across all tenants" },
|
||||
{ id: "mcp-usage", label: "MCP Usage", description: "MCP API usage metrics across all users" },
|
||||
{ id: "verdict-calibration", label: "Verdict Calibration", description: "Which code path produced each session verdict, overrides and re-enrollment — classifier diagnostics" },
|
||||
] as const;
|
||||
|
||||
export type MetricsSectionId = (typeof METRICS_NAV_SECTIONS)[number]["id"];
|
||||
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useAuth } from "../../../../contexts/AuthContext";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { api } from "@/lib/api";
|
||||
import { useAggregatedAdminScope } from "@/hooks";
|
||||
import { TenantScopeSelector } from "@/components/TenantScopeSelector";
|
||||
import { SessionStatusBadge } from "@/components/SessionStatusBadge";
|
||||
import { SegmentedControl, TIME_RANGE_OPTIONS } from "@/components/SegmentedControl";
|
||||
import {
|
||||
ALERT_KIND_LABELS,
|
||||
VERDICT_PATH_ORIGIN_LABELS,
|
||||
alertsByPath,
|
||||
describeAlert,
|
||||
groupPathsByOrigin,
|
||||
trendGlyph,
|
||||
type VerdictCalibrationAlert,
|
||||
type VerdictCalibrationPath,
|
||||
type VerdictCalibrationResponse,
|
||||
} from "./verdictCalibrationLogic";
|
||||
|
||||
type DateRange = "7d" | "30d" | "90d";
|
||||
|
||||
const RANGE_DAYS: Record<DateRange, number> = { "7d": 7, "30d": 30, "90d": 90 };
|
||||
|
||||
export function SectionVerdictCalibration() {
|
||||
const { getAccessToken } = useAuth();
|
||||
const scope = useAggregatedAdminScope({ defaultAggregated: true });
|
||||
const { selectedTenantId, scopeInitialized } = scope;
|
||||
const [data, setData] = useState<VerdictCalibrationResponse | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [dateRange, setDateRange] = useState<DateRange>("30d");
|
||||
|
||||
const fetchData = useCallback(
|
||||
async (range: DateRange, tenantId: string) => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await authenticatedFetch(
|
||||
api.metrics.globalVerdictCalibration(RANGE_DAYS[range], tenantId || undefined),
|
||||
getAccessToken
|
||||
);
|
||||
if (res.status === 404) {
|
||||
// Deploy skew: backend without the route yet — reads as "no rows", never as an error.
|
||||
setData(null);
|
||||
return;
|
||||
}
|
||||
if (!res.ok) throw new Error(`Verdict calibration: ${res.status}`);
|
||||
setData((await res.json()) as VerdictCalibrationResponse);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError("Session expired. Please refresh the page.");
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : "Failed to load verdict calibration");
|
||||
}
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[getAccessToken]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!scopeInitialized) return;
|
||||
const run = async () => {
|
||||
await fetchData(dateRange, selectedTenantId);
|
||||
};
|
||||
void run();
|
||||
}, [fetchData, dateRange, selectedTenantId, scopeInitialized]);
|
||||
|
||||
const groups = data ? groupPathsByOrigin(data.paths) : [];
|
||||
const pathAlerts = alertsByPath(data?.alerts ?? []);
|
||||
|
||||
return (
|
||||
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
|
||||
<div className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-gray-900">Verdict Calibration</h2>
|
||||
<p className="text-sm text-gray-500">
|
||||
Which code path produced each session verdict, how often it was overridden, and whether the device came back.
|
||||
Platform-internal classifier diagnostics — correlations only, never causal claims.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<TenantScopeSelector scope={scope} allowAggregated />
|
||||
<SegmentedControl options={TIME_RANGE_OPTIONS} value={dateRange} onChange={(v) => setDateRange(v as DateRange)} />
|
||||
<button
|
||||
onClick={() => fetchData(dateRange, selectedTenantId)}
|
||||
disabled={loading}
|
||||
className="px-3 py-1.5 text-sm bg-white border border-gray-300 rounded-md hover:bg-gray-50 disabled:opacity-50"
|
||||
>
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="bg-red-50 border border-red-200 text-red-700 text-sm rounded-md px-4 py-3">{error}</div>
|
||||
)}
|
||||
|
||||
{data && (
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
|
||||
<Stat label="Sessions in window" value={data.totals.sessions.toLocaleString()} hint={`${data.totals.days} day rows · ${data.windowStart} – ${data.windowEnd}`} />
|
||||
<Stat label="Terminal" value={data.totals.terminal.toLocaleString()} hint="Succeeded / Failed / Incomplete" />
|
||||
<Stat label="Derived attribution" value={data.totals.derived.toLocaleString()} hint="rows without a stamped path (pre-instrumentation)" />
|
||||
<Stat
|
||||
label="Trend basis"
|
||||
value={`${data.trend.windowSessions.toLocaleString()} / ${data.trend.baselineSessions.toLocaleString()}`}
|
||||
hint={`last ${data.trend.windowDays}d vs prior ${data.trend.baselineDays}d sessions`}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{data && data.alerts.length > 0 && (
|
||||
<div className="bg-white shadow rounded-lg p-6">
|
||||
<h3 className="text-sm font-semibold text-gray-900">Active drift episodes</h3>
|
||||
<p className="text-xs text-gray-500 mb-3">
|
||||
Fired once per episode as a VerdictCalibrationDrift ops event; cleared when the share re-arms. Dimensions are
|
||||
correlated, not necessarily causal.
|
||||
</p>
|
||||
<ul className="divide-y divide-gray-100">
|
||||
{data.alerts.map((a) => (
|
||||
<AlertRow key={`${a.kind}|${a.verdictPath}|${a.status}`} alert={a} />
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="bg-white shadow rounded-lg p-6">
|
||||
{loading && !data ? (
|
||||
<p className="text-sm text-gray-500">Loading…</p>
|
||||
) : !data || data.paths.length === 0 ? (
|
||||
<p className="text-sm text-gray-500">
|
||||
No calibration rows yet. The maintenance sweep writes them every two hours; the first rows appear after its
|
||||
next pass.
|
||||
</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200 text-sm">
|
||||
<thead>
|
||||
<tr className="text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
<th className="px-3 py-2">Verdict path</th>
|
||||
<th className="px-3 py-2">Status</th>
|
||||
<th className="px-3 py-2 text-right">Sessions</th>
|
||||
<th className="px-3 py-2 text-right">Share</th>
|
||||
<th className="px-3 py-2 text-right" title="Share in the last 7 days vs the prior 28 days (lift = window share ÷ baseline share)">
|
||||
7d trend
|
||||
</th>
|
||||
<th className="px-3 py-2 text-right" title="Of terminal sessions at least 7 days old: the same device registered another terminal session within 7 days. Blank below 20 eligible sessions.">
|
||||
Re-enrolled ≤7d
|
||||
</th>
|
||||
<th className="px-3 py-2 text-right" title="Sessions that carried this path and were then overridden: by an administrator / by a late agent completion / by another writer">
|
||||
Overridden (admin / late / other)
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{groups.map((group) => (
|
||||
<GroupRows key={group.origin} origin={group.origin} rows={group.rows} alerts={pathAlerts} />
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
{data?.computedAt && (
|
||||
<p className="mt-3 text-xs text-gray-400">
|
||||
Rows computed {new Date(data.computedAt).toLocaleString()} · algorithm version{" "}
|
||||
{data.versions.join(", ") || "—"}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Stat({ label, value, hint }: { label: string; value: string; hint: string }) {
|
||||
return (
|
||||
<div className="bg-white shadow rounded-lg p-4">
|
||||
<p className="text-xs font-medium text-gray-500 uppercase tracking-wider">{label}</p>
|
||||
<p className="mt-1 text-2xl font-semibold text-gray-900">{value}</p>
|
||||
<p className="text-xs text-gray-400">{hint}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function AlertRow({ alert }: { alert: VerdictCalibrationAlert }) {
|
||||
return (
|
||||
<li className="py-2 text-sm">
|
||||
<span className="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-red-100 text-red-800 mr-2">
|
||||
↑ {ALERT_KIND_LABELS[alert.kind] ?? alert.kind}
|
||||
</span>
|
||||
<span className="font-mono text-xs text-gray-800">{alert.verdictPath}</span>
|
||||
{alert.status !== "*" && <span className="text-gray-500"> / {alert.status}</span>}
|
||||
<span className="text-gray-600"> — {describeAlert(alert)}</span>
|
||||
<span className="text-xs text-gray-400"> · since {new Date(alert.firstNotifiedAt).toLocaleDateString()}</span>
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function GroupRows({ origin, rows, alerts }: { origin: string; rows: VerdictCalibrationPath[]; alerts: Map<string, VerdictCalibrationAlert> }) {
|
||||
return (
|
||||
<>
|
||||
<tr className="bg-gray-50">
|
||||
<td colSpan={7} className="px-3 py-1.5 text-xs font-semibold text-gray-600">
|
||||
{VERDICT_PATH_ORIGIN_LABELS[origin] ?? origin}
|
||||
</td>
|
||||
</tr>
|
||||
{rows.map((row) => (
|
||||
<PathRow key={`${row.verdictPath}|${row.status}`} row={row} alert={alerts.get(`${row.verdictPath}|${row.status}`)} />
|
||||
))}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function PathRow({ row, alert }: { row: VerdictCalibrationPath; alert?: VerdictCalibrationAlert }) {
|
||||
const trend = trendGlyph(row);
|
||||
const overridden = row.overriddenByAdmin + row.overriddenByLateCompletion + row.overriddenOther;
|
||||
return (
|
||||
<tr>
|
||||
<td className="px-3 py-2 font-mono text-xs text-gray-800 whitespace-nowrap">
|
||||
{row.verdictPath}
|
||||
{alert && (
|
||||
<span
|
||||
className="ml-2 inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-red-100 text-red-800"
|
||||
title={describeAlert(alert)}
|
||||
>
|
||||
↑ Drift
|
||||
</span>
|
||||
)}
|
||||
{row.derivedCount > 0 && (
|
||||
<span
|
||||
className="ml-2 inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-gray-100 text-gray-600 border border-gray-200"
|
||||
title={`${row.derivedCount} of ${row.count} attributed read-side (no stamped path)`}
|
||||
>
|
||||
derived {row.derivedCount}
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2">
|
||||
<SessionStatusBadge status={row.status} />
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right tabular-nums">{row.count.toLocaleString()}</td>
|
||||
<td className="px-3 py-2 text-right tabular-nums">{row.sharePct.toFixed(1)}%</td>
|
||||
<td
|
||||
className={`px-3 py-2 text-right tabular-nums ${trend.className}`}
|
||||
title={`last 7d: ${row.window7.count} of ${row.window7.sessions} (${row.window7.sharePct.toFixed(1)}%) · prior 28d: ${row.baseline28.count} of ${row.baseline28.sessions} (${row.baseline28.sharePct.toFixed(1)}%)`}
|
||||
>
|
||||
{trend.text}
|
||||
</td>
|
||||
<td
|
||||
className="px-3 py-2 text-right tabular-nums"
|
||||
title={`${row.reEnrolled7d} of ${row.eligible7d} eligible`}
|
||||
>
|
||||
{row.reEnrollRatePct == null ? <span className="text-gray-400">— (n={row.eligible7d})</span> : `${row.reEnrollRatePct.toFixed(1)}%`}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right tabular-nums" title={overridden === 0 ? undefined : `${overridden} overridden in total`}>
|
||||
{overridden === 0 ? (
|
||||
<span className="text-gray-400">—</span>
|
||||
) : (
|
||||
`${row.overriddenByAdmin} / ${row.overriddenByLateCompletion} / ${row.overriddenOther}`
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
VERDICT_PATH_ORIGIN_LABELS,
|
||||
groupPathsByOrigin,
|
||||
pathOrigin,
|
||||
trendGlyph,
|
||||
type VerdictCalibrationPath,
|
||||
} from "../verdictCalibrationLogic";
|
||||
|
||||
function row(verdictPath: string, overrides: Partial<VerdictCalibrationPath> = {}): VerdictCalibrationPath {
|
||||
return {
|
||||
verdictPath,
|
||||
status: "Succeeded",
|
||||
count: 10,
|
||||
sharePct: 10,
|
||||
derivedCount: 0,
|
||||
eligible7d: 10,
|
||||
reEnrolled7d: 1,
|
||||
reEnrollRatePct: null,
|
||||
overriddenByAdmin: 0,
|
||||
overriddenByLateCompletion: 0,
|
||||
overriddenOther: 0,
|
||||
window7: { count: 5, sessions: 50, sharePct: 10 },
|
||||
baseline28: { count: 20, sessions: 200, sharePct: 10 },
|
||||
lift: 1,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("pathOrigin", () => {
|
||||
it("splits on the first colon and tolerates rule ids with colons-free detail", () => {
|
||||
expect(pathOrigin("sweep:r5_incomplete")).toBe("sweep");
|
||||
expect(pathOrigin("rule:ANALYZE-ESP-001")).toBe("rule");
|
||||
expect(pathOrigin("legacy:unknown")).toBe("legacy");
|
||||
expect(pathOrigin("weird")).toBe("weird");
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupPathsByOrigin", () => {
|
||||
it("orders groups by the documented origin order, unknown origins last, rows untouched", () => {
|
||||
const groups = groupPathsByOrigin([
|
||||
row("zzz:x"),
|
||||
row("manual:failed"),
|
||||
row("sweep:r6"),
|
||||
row("sweep:r5_incomplete"),
|
||||
row("agent:complete"),
|
||||
]);
|
||||
expect(groups.map((g) => g.origin)).toEqual(["agent", "sweep", "manual", "zzz"]);
|
||||
expect(groups[1].rows.map((r) => r.verdictPath)).toEqual(["sweep:r6", "sweep:r5_incomplete"]);
|
||||
});
|
||||
|
||||
it("has a label for every origin in the order list", () => {
|
||||
for (const origin of ["agent", "ingest", "sweep", "maxlife", "late", "retro", "register", "rule", "manual", "legacy"]) {
|
||||
expect(VERDICT_PATH_ORIGIN_LABELS[origin]).toBeTruthy();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("trendGlyph", () => {
|
||||
it("flags a doubled share with an up arrow and a halved share with a down arrow", () => {
|
||||
expect(trendGlyph(row("sweep:r6", { lift: 2.4 })).text).toBe("↑ 2.4×");
|
||||
expect(trendGlyph(row("sweep:r6", { lift: 0.4 })).text).toBe("↓ 0.4×");
|
||||
expect(trendGlyph(row("sweep:r6", { lift: 1.3 })).text).toBe("≈ 1.3×");
|
||||
});
|
||||
|
||||
it("withholds the arrow below five window hits — one session can double a tiny share", () => {
|
||||
const g = trendGlyph(row("sweep:r6", { lift: 3, window7: { count: 3, sessions: 50, sharePct: 6 } }));
|
||||
expect(g.text).toBe("≈ 3.0×");
|
||||
expect(g.className).toContain("text-gray-400");
|
||||
});
|
||||
|
||||
it("never invents a lift without a baseline", () => {
|
||||
expect(trendGlyph(row("sweep:r6", { lift: null, baseline28: { count: 0, sessions: 200, sharePct: 0 } })).text).toBe("new");
|
||||
expect(trendGlyph(row("sweep:r6", { lift: null, window7: { count: 0, sessions: 0, sharePct: 0 }, baseline28: { count: 0, sessions: 0, sharePct: 0 } })).text).toBe("—");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Pure helpers for the Verdict Calibration matrix (docs/backend/verdict-calibration.md).
|
||||
* Wire shape mirrors GetVerdictCalibrationFunction / VerdictCalibrationResponse (camelCase).
|
||||
*/
|
||||
|
||||
export interface TrendWindow {
|
||||
count: number;
|
||||
sessions: number;
|
||||
sharePct: number;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationPath {
|
||||
verdictPath: string;
|
||||
status: string;
|
||||
count: number;
|
||||
sharePct: number;
|
||||
derivedCount: number;
|
||||
eligible7d: number;
|
||||
reEnrolled7d: number;
|
||||
/** null below 20 eligible sessions — never a rate on a handful */
|
||||
reEnrollRatePct: number | null;
|
||||
overriddenByAdmin: number;
|
||||
overriddenByLateCompletion: number;
|
||||
overriddenOther: number;
|
||||
window7: TrendWindow;
|
||||
baseline28: TrendWindow;
|
||||
/** window share ÷ baseline share; null without a baseline */
|
||||
lift: number | null;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationDimension {
|
||||
dimension: string;
|
||||
value: string;
|
||||
hitCount: number;
|
||||
hitSharePct: number;
|
||||
allSharePct: number;
|
||||
lift: number;
|
||||
}
|
||||
|
||||
/** One ACTIVE drift episode (mirrors VerdictCalibrationAlert). */
|
||||
export interface VerdictCalibrationAlert {
|
||||
tenantId: string;
|
||||
kind: "share_regression" | "silence_share_regression" | "evidence_gap" | string;
|
||||
verdictPath: string;
|
||||
status: string;
|
||||
windowHitCount: number;
|
||||
windowSessionCount: number;
|
||||
baselineHitCount: number;
|
||||
baselineSessionCount: number;
|
||||
windowRatePct: number;
|
||||
baselineRatePct: number;
|
||||
lift: number | null;
|
||||
windowStartDate: string;
|
||||
windowEndDate: string;
|
||||
dimension: VerdictCalibrationDimension | null;
|
||||
firstNotifiedAt: string;
|
||||
lastEvaluatedAt: string;
|
||||
}
|
||||
|
||||
export const ALERT_KIND_LABELS: Record<string, string> = {
|
||||
share_regression: "Share regression",
|
||||
silence_share_regression: "Silence share regression (agent liveness)",
|
||||
evidence_gap: "Evidence gap (r6 fallthrough)",
|
||||
};
|
||||
|
||||
/** One-line alert summary; the dimension clause keeps the rule-radar wording contract. */
|
||||
export function describeAlert(a: VerdictCalibrationAlert): string {
|
||||
const base =
|
||||
`${a.windowHitCount} of ${a.windowSessionCount} (${a.windowRatePct.toFixed(1)}%) over 7d vs ` +
|
||||
`${a.baselineRatePct.toFixed(1)}% baseline (${a.baselineHitCount} of ${a.baselineSessionCount} over 28d)` +
|
||||
(a.lift != null ? ` — lift ${a.lift.toFixed(1)}×` : "");
|
||||
if (!a.dimension) return base;
|
||||
return `${base}. ${a.dimension.value} carries ${a.dimension.hitSharePct.toFixed(0)}% of hits vs ${a.dimension.allSharePct.toFixed(0)}% of all sessions — correlated, not necessarily causal.`;
|
||||
}
|
||||
|
||||
/** Index active per-path alerts by "path|status" for the matrix pill. */
|
||||
export function alertsByPath(alerts: VerdictCalibrationAlert[]): Map<string, VerdictCalibrationAlert> {
|
||||
const map = new Map<string, VerdictCalibrationAlert>();
|
||||
for (const a of alerts) {
|
||||
if (a.kind === "share_regression") map.set(`${a.verdictPath}|${a.status}`, a);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
export interface VerdictCalibrationResponse {
|
||||
success: boolean;
|
||||
tenantId: string;
|
||||
windowDays: number;
|
||||
windowStart: string;
|
||||
windowEnd: string;
|
||||
computedAt: string | null;
|
||||
versions: number[];
|
||||
totals: { sessions: number; terminal: number; derived: number; days: number };
|
||||
trend: { windowDays: number; baselineDays: number; windowSessions: number; baselineSessions: number };
|
||||
paths: VerdictCalibrationPath[];
|
||||
alerts: VerdictCalibrationAlert[];
|
||||
}
|
||||
|
||||
/** The `origin` half of a verdict path (`sweep` for `sweep:r6`); the whole string without a colon. */
|
||||
export function pathOrigin(path: string): string {
|
||||
const i = path.indexOf(":");
|
||||
return i < 0 ? path : path.slice(0, i);
|
||||
}
|
||||
|
||||
/** Display order + labels for the origin groups; unknown origins sort last, alphabetically. */
|
||||
export const VERDICT_PATH_ORIGIN_LABELS: Record<string, string> = {
|
||||
agent: "Agent-declared",
|
||||
ingest: "Ingest mapping",
|
||||
sweep: "Maintenance sweep (silence classifier)",
|
||||
maxlife: "Agent max-lifetime shutdown (silence classifier)",
|
||||
late: "Late telemetry reconcile (silence classifier)",
|
||||
retro: "Retro reclassification",
|
||||
register: "Session registration",
|
||||
rule: "Analyze rule (MarkSessionAsFailed)",
|
||||
manual: "Administrator",
|
||||
legacy: "Pre-instrumentation (derived, origin unknown)",
|
||||
};
|
||||
|
||||
const ORIGIN_ORDER = Object.keys(VERDICT_PATH_ORIGIN_LABELS);
|
||||
|
||||
export interface OriginGroup {
|
||||
origin: string;
|
||||
rows: VerdictCalibrationPath[];
|
||||
}
|
||||
|
||||
/** Groups rows by origin in display order; rows inside a group keep the server order (count desc). */
|
||||
export function groupPathsByOrigin(paths: VerdictCalibrationPath[]): OriginGroup[] {
|
||||
const byOrigin = new Map<string, VerdictCalibrationPath[]>();
|
||||
for (const p of paths) {
|
||||
const origin = pathOrigin(p.verdictPath);
|
||||
const list = byOrigin.get(origin);
|
||||
if (list) list.push(p);
|
||||
else byOrigin.set(origin, [p]);
|
||||
}
|
||||
return Array.from(byOrigin.entries())
|
||||
.sort(([a], [b]) => {
|
||||
const ia = ORIGIN_ORDER.indexOf(a);
|
||||
const ib = ORIGIN_ORDER.indexOf(b);
|
||||
if (ia !== -1 || ib !== -1) return (ia === -1 ? ORIGIN_ORDER.length : ia) - (ib === -1 ? ORIGIN_ORDER.length : ib);
|
||||
return a.localeCompare(b);
|
||||
})
|
||||
.map(([origin, rows]) => ({ origin, rows }));
|
||||
}
|
||||
|
||||
/**
|
||||
* 7d-vs-28d trend cell. Lift ≥ 2 (share at least doubled) and ≤ 0.5 (halved) get the arrow +
|
||||
* color; anything in between is a neutral "≈". No baseline → "new" (a new signal has no finite
|
||||
* lift — never invented). Below 5 window hits the arrow is withheld: one session can double a
|
||||
* tiny share.
|
||||
*/
|
||||
export function trendGlyph(row: VerdictCalibrationPath): { text: string; className: string } {
|
||||
if (row.lift == null) {
|
||||
return row.window7.count > 0 && row.baseline28.sessions > 0
|
||||
? { text: "new", className: "text-gray-500" }
|
||||
: { text: "—", className: "text-gray-400" };
|
||||
}
|
||||
const lift = row.lift;
|
||||
const label = `${lift.toFixed(1)}×`;
|
||||
if (row.window7.count < 5) return { text: `≈ ${label}`, className: "text-gray-400" };
|
||||
if (lift >= 2) return { text: `↑ ${label}`, className: "text-red-600 font-medium" };
|
||||
if (lift <= 0.5) return { text: `↓ ${label}`, className: "text-green-700" };
|
||||
return { text: `≈ ${label}`, className: "text-gray-500" };
|
||||
}
|
||||
@@ -275,55 +275,57 @@ export default function InstallsTab({ scope, timeRange }: InstallsTabProps) {
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-gray-50">
|
||||
<tr className="text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
<th className="px-4 py-3 cursor-pointer select-none" onClick={() => toggleSort("appName")}>App {sortIndicator("appName")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("totalInstalls")}>Installs {sortIndicator("totalInstalls")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("succeeded")}>Succeeded {sortIndicator("succeeded")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("failed")}>Failed {sortIndicator("failed")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("failureRate")}>Failure Rate {sortIndicator("failureRate")}</th>
|
||||
<th
|
||||
className="px-4 py-3 cursor-pointer select-none text-right"
|
||||
onClick={() => toggleSort("avgDurationSeconds")}
|
||||
title="Average duration of the final install attempt. IME evaluation passes and time queued behind other apps are not counted."
|
||||
>
|
||||
Avg Install Time {sortIndicator("avgDurationSeconds")}
|
||||
</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("trend")}>Trend {sortIndicator("trend")}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{pageRows.map((row) => (
|
||||
<tr key={row.appName} onClick={() => openApp(row.appName)} className="hover:bg-gray-50 cursor-pointer text-sm">
|
||||
<td className="px-4 py-3 text-gray-900">
|
||||
<span className="font-medium">{row.appName}</span>
|
||||
{appTypeBadge(row.appType)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right text-gray-700">{row.totalInstalls}</td>
|
||||
<td className="px-4 py-3 text-right text-emerald-700">
|
||||
{row.succeeded}
|
||||
{row.skipped > 0 && (
|
||||
<span
|
||||
className="ml-1 text-xs text-gray-400 dark:text-gray-500"
|
||||
title={`${row.skipped} skipped (not applicable) — excluded from failure rate and durations`}
|
||||
>
|
||||
+{row.skipped}s
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right text-red-700">{row.failed}</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<span className={row.failureRate >= 20 ? "text-red-700 font-semibold" : row.failureRate >= 5 ? "text-amber-700" : "text-gray-700"}>
|
||||
{row.failureRate.toFixed(1)}%
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right text-gray-700">{formatDuration(row.avgDurationSeconds, "—")}</td>
|
||||
<td className="px-4 py-3 text-right">{trendBadge(row)}</td>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-gray-50">
|
||||
<tr className="text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
<th className="px-4 py-3 cursor-pointer select-none" onClick={() => toggleSort("appName")}>App {sortIndicator("appName")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("totalInstalls")}>Installs {sortIndicator("totalInstalls")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("succeeded")}>Succeeded {sortIndicator("succeeded")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("failed")}>Failed {sortIndicator("failed")}</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("failureRate")}>Failure Rate {sortIndicator("failureRate")}</th>
|
||||
<th
|
||||
className="px-4 py-3 cursor-pointer select-none text-right"
|
||||
onClick={() => toggleSort("avgDurationSeconds")}
|
||||
title="Average duration of the final install attempt. IME evaluation passes and time queued behind other apps are not counted."
|
||||
>
|
||||
Avg Install Time {sortIndicator("avgDurationSeconds")}
|
||||
</th>
|
||||
<th className="px-4 py-3 cursor-pointer select-none text-right" onClick={() => toggleSort("trend")}>Trend {sortIndicator("trend")}</th>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{pageRows.map((row) => (
|
||||
<tr key={row.appName} onClick={() => openApp(row.appName)} className="hover:bg-gray-50 cursor-pointer text-sm">
|
||||
<td className="px-4 py-3 text-gray-900">
|
||||
<span className="font-medium">{row.appName}</span>
|
||||
{appTypeBadge(row.appType)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right text-gray-700">{row.totalInstalls}</td>
|
||||
<td className="px-4 py-3 text-right text-emerald-700">
|
||||
{row.succeeded}
|
||||
{row.skipped > 0 && (
|
||||
<span
|
||||
className="ml-1 text-xs text-gray-400 dark:text-gray-500"
|
||||
title={`${row.skipped} skipped (not applicable) — excluded from failure rate and durations`}
|
||||
>
|
||||
+{row.skipped}s
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right text-red-700">{row.failed}</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
<span className={row.failureRate >= 20 ? "text-red-700 font-semibold" : row.failureRate >= 5 ? "text-amber-700" : "text-gray-700"}>
|
||||
{row.failureRate.toFixed(1)}%
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-right text-gray-700">{formatDuration(row.avgDurationSeconds, "—")}</td>
|
||||
<td className="px-4 py-3 text-right">{trendBadge(row)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{pageCount > 1 && (
|
||||
<div className="flex items-center justify-between px-4 py-3 text-sm text-gray-600 border-t border-gray-200">
|
||||
<span>{filteredAndSorted.length} apps · page {page + 1} of {pageCount}</span>
|
||||
|
||||
@@ -151,36 +151,38 @@ export default function InventoryTab({ scope }: { scope: SoftwareTabScope }) {
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-gray-50">
|
||||
<tr className="text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
<th className="px-4 py-3">Software</th>
|
||||
<th className="px-4 py-3">Version</th>
|
||||
<th className="px-4 py-3">Publisher</th>
|
||||
<th className="px-4 py-3 text-right">Sessions</th>
|
||||
<th className="px-4 py-3 text-right">Last seen</th>
|
||||
<th className="px-4 py-3 text-right">CPE</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{pageRows.map((row, idx) => (
|
||||
<tr key={`${row.normalizedVendor}|${row.normalizedName}|${row.normalizedVersion}|${idx}`} className="text-sm">
|
||||
<td className="px-4 py-3 text-gray-900 font-medium">{row.displayName || row.normalizedName}</td>
|
||||
<td className="px-4 py-3 text-gray-700">{row.normalizedVersion || "—"}</td>
|
||||
<td className="px-4 py-3 text-gray-700">{row.publisher || "—"}</td>
|
||||
<td className="px-4 py-3 text-right text-gray-700">{row.sessionCount}</td>
|
||||
<td className="px-4 py-3 text-right text-gray-500">{formatDate(row.lastSeenAt)}</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
{row.cpeUri ? (
|
||||
<span className="inline-block px-1.5 py-0.5 rounded text-xs bg-emerald-100 text-emerald-800">mapped</span>
|
||||
) : (
|
||||
<span className="inline-block px-1.5 py-0.5 rounded text-xs bg-amber-100 text-amber-800">unmatched</span>
|
||||
)}
|
||||
</td>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y divide-gray-200">
|
||||
<thead className="bg-gray-50">
|
||||
<tr className="text-left text-xs font-medium text-gray-500 uppercase tracking-wider">
|
||||
<th className="px-4 py-3">Software</th>
|
||||
<th className="px-4 py-3">Version</th>
|
||||
<th className="px-4 py-3">Publisher</th>
|
||||
<th className="px-4 py-3 text-right">Sessions</th>
|
||||
<th className="px-4 py-3 text-right">Last seen</th>
|
||||
<th className="px-4 py-3 text-right">CPE</th>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{pageRows.map((row, idx) => (
|
||||
<tr key={`${row.normalizedVendor}|${row.normalizedName}|${row.normalizedVersion}|${idx}`} className="text-sm">
|
||||
<td className="px-4 py-3 text-gray-900 font-medium">{row.displayName || row.normalizedName}</td>
|
||||
<td className="px-4 py-3 text-gray-700">{row.normalizedVersion || "—"}</td>
|
||||
<td className="px-4 py-3 text-gray-700">{row.publisher || "—"}</td>
|
||||
<td className="px-4 py-3 text-right text-gray-700">{row.sessionCount}</td>
|
||||
<td className="px-4 py-3 text-right text-gray-500">{formatDate(row.lastSeenAt)}</td>
|
||||
<td className="px-4 py-3 text-right">
|
||||
{row.cpeUri ? (
|
||||
<span className="inline-block px-1.5 py-0.5 rounded text-xs bg-emerald-100 text-emerald-800">mapped</span>
|
||||
) : (
|
||||
<span className="inline-block px-1.5 py-0.5 rounded text-xs bg-amber-100 text-amber-800">unmatched</span>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{pageCount > 1 && (
|
||||
<div className="flex items-center justify-between px-4 py-3 text-sm text-gray-600 border-t border-gray-200">
|
||||
<span>{filtered.length} titles · page {page + 1} of {pageCount}</span>
|
||||
|
||||
@@ -459,12 +459,12 @@ function AppDetailContent() {
|
||||
>
|
||||
← Back to all apps
|
||||
</button>
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-normal text-gray-900 inline-flex items-center">
|
||||
{appName}
|
||||
<div className="flex flex-wrap items-center justify-between gap-y-3">
|
||||
<div className="min-w-0">
|
||||
<h1 className="text-2xl font-normal text-gray-900 flex flex-wrap items-center gap-x-3 gap-y-1">
|
||||
<span className="break-words">{appName}</span>
|
||||
{analytics?.appType && (
|
||||
<span className="ml-3 px-2 py-0.5 text-xs rounded bg-blue-100 text-blue-800">
|
||||
<span className="px-2 py-0.5 text-xs rounded bg-blue-100 text-blue-800">
|
||||
{analytics.appType}
|
||||
</span>
|
||||
)}
|
||||
@@ -475,7 +475,7 @@ function AppDetailContent() {
|
||||
: ""}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="flex flex-wrap items-center gap-2 sm:gap-3">
|
||||
<TenantScopeSelector scope={scope} allowAggregated />
|
||||
{([7, 30, 90] as const).map((d) => (
|
||||
<button
|
||||
@@ -679,69 +679,71 @@ function AppDetailContent() {
|
||||
{/* Top failure codes table */}
|
||||
<Card title="Top Failure Codes">
|
||||
{analytics.topFailureCodes.length > 0 ? (
|
||||
<table className="min-w-full text-sm">
|
||||
<thead className="text-left text-xs text-gray-500 uppercase tracking-wider border-b">
|
||||
<tr>
|
||||
<th className="px-3 py-2">Code</th>
|
||||
<th className="px-3 py-2">Description</th>
|
||||
<th className="px-3 py-2">Exit Code</th>
|
||||
<th className="px-3 py-2 text-right">Count</th>
|
||||
<th className="px-3 py-2">Sample Message</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{analytics.topFailureCodes.map((row) => {
|
||||
const entry = getErrorCodeEntry(row.code);
|
||||
return (
|
||||
<tr key={row.code}>
|
||||
<td className="px-3 py-2 font-mono text-xs text-gray-900">
|
||||
{formatErrorCode(row.code)}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-gray-700">
|
||||
{entry ? (
|
||||
<>
|
||||
{entry.description}
|
||||
<span
|
||||
className={`ml-2 px-1.5 py-0.5 rounded text-xs ${
|
||||
entry.confidence === "high"
|
||||
? "bg-emerald-100 text-emerald-800"
|
||||
: entry.confidence === "medium"
|
||||
? "bg-amber-100 text-amber-800"
|
||||
: "bg-gray-100 text-gray-700"
|
||||
}`}
|
||||
title={`Source: ${entry.source}`}
|
||||
>
|
||||
{entry.confidence}
|
||||
</span>
|
||||
</>
|
||||
) : (
|
||||
<span className="text-gray-400">Unknown code</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2 font-mono text-xs">
|
||||
{row.exitCode != null ? (
|
||||
(() => {
|
||||
const ec = getErrorCodeEntry(row.exitCode);
|
||||
return (
|
||||
<span title={ec?.description ?? ""}>
|
||||
{row.exitCode}
|
||||
{ec ? ` (${ec.description.slice(0, 24)}${ec.description.length > 24 ? "…" : ""})` : ""}
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full text-sm">
|
||||
<thead className="text-left text-xs text-gray-500 uppercase tracking-wider border-b">
|
||||
<tr>
|
||||
<th className="px-3 py-2">Code</th>
|
||||
<th className="px-3 py-2">Description</th>
|
||||
<th className="px-3 py-2">Exit Code</th>
|
||||
<th className="px-3 py-2 text-right">Count</th>
|
||||
<th className="px-3 py-2">Sample Message</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{analytics.topFailureCodes.map((row) => {
|
||||
const entry = getErrorCodeEntry(row.code);
|
||||
return (
|
||||
<tr key={row.code}>
|
||||
<td className="px-3 py-2 font-mono text-xs text-gray-900">
|
||||
{formatErrorCode(row.code)}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-gray-700">
|
||||
{entry ? (
|
||||
<>
|
||||
{entry.description}
|
||||
<span
|
||||
className={`ml-2 px-1.5 py-0.5 rounded text-xs ${
|
||||
entry.confidence === "high"
|
||||
? "bg-emerald-100 text-emerald-800"
|
||||
: entry.confidence === "medium"
|
||||
? "bg-amber-100 text-amber-800"
|
||||
: "bg-gray-100 text-gray-700"
|
||||
}`}
|
||||
title={`Source: ${entry.source}`}
|
||||
>
|
||||
{entry.confidence}
|
||||
</span>
|
||||
);
|
||||
})()
|
||||
) : (
|
||||
<span className="text-gray-400">—</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right text-gray-900">{row.count}</td>
|
||||
<td className="px-3 py-2 text-gray-500 text-xs truncate max-w-md" title={row.sampleMessage}>
|
||||
{row.sampleMessage || "—"}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</>
|
||||
) : (
|
||||
<span className="text-gray-400">Unknown code</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2 font-mono text-xs">
|
||||
{row.exitCode != null ? (
|
||||
(() => {
|
||||
const ec = getErrorCodeEntry(row.exitCode);
|
||||
return (
|
||||
<span title={ec?.description ?? ""}>
|
||||
{row.exitCode}
|
||||
{ec ? ` (${ec.description.slice(0, 24)}${ec.description.length > 24 ? "…" : ""})` : ""}
|
||||
</span>
|
||||
);
|
||||
})()
|
||||
) : (
|
||||
<span className="text-gray-400">—</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right text-gray-900">{row.count}</td>
|
||||
<td className="px-3 py-2 text-gray-500 text-xs truncate max-w-md" title={row.sampleMessage}>
|
||||
{row.sampleMessage || "—"}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
) : (
|
||||
<EmptyState text="No failures recorded" />
|
||||
)}
|
||||
@@ -750,44 +752,46 @@ function AppDetailContent() {
|
||||
{/* Device model breakdown */}
|
||||
<Card title="Device Model Correlation">
|
||||
{analytics.deviceModelBreakdown.length > 0 ? (
|
||||
<table className="min-w-full text-sm">
|
||||
<thead className="text-left text-xs text-gray-500 uppercase tracking-wider border-b">
|
||||
<tr>
|
||||
<th className="px-3 py-2">Manufacturer</th>
|
||||
<th className="px-3 py-2">Model</th>
|
||||
<th className="px-3 py-2 text-right">Installs</th>
|
||||
<th className="px-3 py-2 text-right">Failed</th>
|
||||
<th className="px-3 py-2 text-right">Failure Rate</th>
|
||||
<th className="px-3 py-2 text-right">Lift vs baseline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{analytics.deviceModelBreakdown.map((row, i) => (
|
||||
<tr key={`${row.manufacturer}-${row.model}-${i}`}>
|
||||
<td className="px-3 py-2 text-gray-700">{row.manufacturer}</td>
|
||||
<td className="px-3 py-2 text-gray-900">{row.model}</td>
|
||||
<td className="px-3 py-2 text-right text-gray-700">{row.installs}</td>
|
||||
<td className="px-3 py-2 text-right text-red-700">{row.failed}</td>
|
||||
<td className="px-3 py-2 text-right text-gray-900">
|
||||
{row.failureRate.toFixed(1)}%
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right">
|
||||
<span
|
||||
className={`px-2 py-0.5 rounded text-xs font-medium ${
|
||||
row.liftVsBaseline >= 2
|
||||
? "bg-red-100 text-red-800"
|
||||
: row.liftVsBaseline >= 1.2
|
||||
? "bg-amber-100 text-amber-800"
|
||||
: "bg-emerald-100 text-emerald-800"
|
||||
}`}
|
||||
>
|
||||
{row.liftVsBaseline.toFixed(2)}×
|
||||
</span>
|
||||
</td>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full text-sm">
|
||||
<thead className="text-left text-xs text-gray-500 uppercase tracking-wider border-b">
|
||||
<tr>
|
||||
<th className="px-3 py-2">Manufacturer</th>
|
||||
<th className="px-3 py-2">Model</th>
|
||||
<th className="px-3 py-2 text-right">Installs</th>
|
||||
<th className="px-3 py-2 text-right">Failed</th>
|
||||
<th className="px-3 py-2 text-right">Failure Rate</th>
|
||||
<th className="px-3 py-2 text-right">Lift vs baseline</th>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{analytics.deviceModelBreakdown.map((row, i) => (
|
||||
<tr key={`${row.manufacturer}-${row.model}-${i}`}>
|
||||
<td className="px-3 py-2 text-gray-700">{row.manufacturer}</td>
|
||||
<td className="px-3 py-2 text-gray-900">{row.model}</td>
|
||||
<td className="px-3 py-2 text-right text-gray-700">{row.installs}</td>
|
||||
<td className="px-3 py-2 text-right text-red-700">{row.failed}</td>
|
||||
<td className="px-3 py-2 text-right text-gray-900">
|
||||
{row.failureRate.toFixed(1)}%
|
||||
</td>
|
||||
<td className="px-3 py-2 text-right">
|
||||
<span
|
||||
className={`px-2 py-0.5 rounded text-xs font-medium ${
|
||||
row.liftVsBaseline >= 2
|
||||
? "bg-red-100 text-red-800"
|
||||
: row.liftVsBaseline >= 1.2
|
||||
? "bg-amber-100 text-amber-800"
|
||||
: "bg-emerald-100 text-emerald-800"
|
||||
}`}
|
||||
>
|
||||
{row.liftVsBaseline.toFixed(2)}×
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
) : (
|
||||
<EmptyState text="Not enough installs per device model to compute correlation" />
|
||||
)}
|
||||
@@ -795,7 +799,7 @@ function AppDetailContent() {
|
||||
|
||||
{/* Affected sessions panel */}
|
||||
<Card title="Affected Sessions">
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<div className="flex flex-wrap items-center gap-2 mb-3">
|
||||
{(["failed", "all", "succeeded"] as const).map((s) => (
|
||||
<button
|
||||
key={s}
|
||||
@@ -868,117 +872,119 @@ function AppDetailContent() {
|
||||
<div className="text-center text-gray-500 p-4">Loading sessions…</div>
|
||||
) : sessions && sessions.items.length > 0 ? (
|
||||
<>
|
||||
<table className="min-w-full text-sm">
|
||||
<thead className="text-left text-xs text-gray-500 uppercase tracking-wider border-b">
|
||||
<tr>
|
||||
{activeSessionColumns.map((col) => (
|
||||
<th
|
||||
key={col.key}
|
||||
className={`px-3 py-2 ${col.numeric ? "text-right" : ""}`}
|
||||
>
|
||||
{col.label}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{sessions.items.map((row) => {
|
||||
const entry = row.failureCode ? getErrorCodeEntry(row.failureCode) : null;
|
||||
return (
|
||||
<tr
|
||||
key={`${row.tenantId}-${row.sessionId}`}
|
||||
onClick={() => router.push(sessionUrl(row.sessionId))}
|
||||
className="hover:bg-gray-50 cursor-pointer"
|
||||
>
|
||||
{activeSessionColumns.map((col) => {
|
||||
switch (col.key) {
|
||||
case "device":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-gray-900">
|
||||
{row.deviceName || "—"}
|
||||
</td>
|
||||
);
|
||||
case "tenant": {
|
||||
const friendly = tenantLookup.get(row.tenantId);
|
||||
return (
|
||||
<td
|
||||
key={col.key}
|
||||
className="px-3 py-2 text-gray-700 text-xs"
|
||||
title={row.tenantId}
|
||||
>
|
||||
{friendly || `${row.tenantId.substring(0, 8)}…`}
|
||||
</td>
|
||||
);
|
||||
}
|
||||
case "model":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-gray-700 text-xs">
|
||||
{row.manufacturer} {row.model}
|
||||
</td>
|
||||
);
|
||||
case "version":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-gray-700 font-mono text-xs">
|
||||
{row.appVersion || "—"}
|
||||
</td>
|
||||
);
|
||||
case "status":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2">
|
||||
<span
|
||||
className={`px-2 py-0.5 rounded text-xs ${
|
||||
row.status === "Failed"
|
||||
? "bg-red-100 text-red-800"
|
||||
: row.status === "Succeeded"
|
||||
? "bg-emerald-100 text-emerald-800"
|
||||
: "bg-gray-100 text-gray-700"
|
||||
}`}
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full text-sm">
|
||||
<thead className="text-left text-xs text-gray-500 uppercase tracking-wider border-b">
|
||||
<tr>
|
||||
{activeSessionColumns.map((col) => (
|
||||
<th
|
||||
key={col.key}
|
||||
className={`px-3 py-2 ${col.numeric ? "text-right" : ""}`}
|
||||
>
|
||||
{col.label}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-100">
|
||||
{sessions.items.map((row) => {
|
||||
const entry = row.failureCode ? getErrorCodeEntry(row.failureCode) : null;
|
||||
return (
|
||||
<tr
|
||||
key={`${row.tenantId}-${row.sessionId}`}
|
||||
onClick={() => router.push(sessionUrl(row.sessionId))}
|
||||
className="hover:bg-gray-50 cursor-pointer"
|
||||
>
|
||||
{activeSessionColumns.map((col) => {
|
||||
switch (col.key) {
|
||||
case "device":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-gray-900">
|
||||
{row.deviceName || "—"}
|
||||
</td>
|
||||
);
|
||||
case "tenant": {
|
||||
const friendly = tenantLookup.get(row.tenantId);
|
||||
return (
|
||||
<td
|
||||
key={col.key}
|
||||
className="px-3 py-2 text-gray-700 text-xs"
|
||||
title={row.tenantId}
|
||||
>
|
||||
{row.status}
|
||||
</span>
|
||||
</td>
|
||||
);
|
||||
case "attempts":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-right text-gray-700">
|
||||
{row.attemptNumber || "—"}
|
||||
</td>
|
||||
);
|
||||
case "failureCode":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 font-mono text-xs">
|
||||
{row.failureCode ? (
|
||||
<span title={entry?.description ?? ""}>
|
||||
{formatErrorCode(row.failureCode)}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">—</span>
|
||||
)}
|
||||
</td>
|
||||
);
|
||||
case "duration":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-right text-gray-700">
|
||||
{formatDuration(row.durationSeconds)}
|
||||
{(row.installPassCount ?? 0) > 1 && (
|
||||
{friendly || `${row.tenantId.substring(0, 8)}…`}
|
||||
</td>
|
||||
);
|
||||
}
|
||||
case "model":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-gray-700 text-xs">
|
||||
{row.manufacturer} {row.model}
|
||||
</td>
|
||||
);
|
||||
case "version":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-gray-700 font-mono text-xs">
|
||||
{row.appVersion || "—"}
|
||||
</td>
|
||||
);
|
||||
case "status":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2">
|
||||
<span
|
||||
className="ml-1 text-xs text-gray-400 dark:text-gray-500"
|
||||
title={`Processed in ${row.installPassCount} IME passes (evaluation + install) — install time counts only the final attempt`}
|
||||
className={`px-2 py-0.5 rounded text-xs ${
|
||||
row.status === "Failed"
|
||||
? "bg-red-100 text-red-800"
|
||||
: row.status === "Succeeded"
|
||||
? "bg-emerald-100 text-emerald-800"
|
||||
: "bg-gray-100 text-gray-700"
|
||||
}`}
|
||||
>
|
||||
×{row.installPassCount}
|
||||
{row.status}
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
})}
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
);
|
||||
case "attempts":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-right text-gray-700">
|
||||
{row.attemptNumber || "—"}
|
||||
</td>
|
||||
);
|
||||
case "failureCode":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 font-mono text-xs">
|
||||
{row.failureCode ? (
|
||||
<span title={entry?.description ?? ""}>
|
||||
{formatErrorCode(row.failureCode)}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-gray-400">—</span>
|
||||
)}
|
||||
</td>
|
||||
);
|
||||
case "duration":
|
||||
return (
|
||||
<td key={col.key} className="px-3 py-2 text-right text-gray-700">
|
||||
{formatDuration(row.durationSeconds)}
|
||||
{(row.installPassCount ?? 0) > 1 && (
|
||||
<span
|
||||
className="ml-1 text-xs text-gray-400 dark:text-gray-500"
|
||||
title={`Processed in ${row.installPassCount} IME passes (evaluation + install) — install time counts only the final attempt`}
|
||||
>
|
||||
×{row.installPassCount}
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
})}
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{/* Pagination */}
|
||||
{sessions.total > SESSIONS_PAGE_SIZE && (
|
||||
<div className="flex items-center justify-between mt-3 text-xs text-gray-600">
|
||||
|
||||
+1
-1
@@ -648,7 +648,7 @@ export default function VulnerabilityReportSection({
|
||||
{/* Data Freshness */}
|
||||
<div className="mt-3 text-xs text-gray-400">
|
||||
{dataFreshness && <>Data Freshness: {dataFreshness} · </>}
|
||||
<span className="italic">Experimental — best-effort CVE matching</span>
|
||||
<span className="italic">Best-effort CVE matching</span>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -190,15 +190,7 @@ export default function AgentAnalyzersSection({
|
||||
|
||||
{/* ── Software Inventory & Vulnerability Analyzer ── */}
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<h3 className="text-sm font-semibold text-gray-700 uppercase tracking-wider">Software Inventory & Vulnerability Analyzer</h3>
|
||||
<span className="inline-flex items-center px-2 py-0.5 rounded-full text-xs font-semibold bg-emerald-100 text-emerald-700 border border-emerald-200">
|
||||
Experimental
|
||||
</span>
|
||||
<span className="inline-flex items-center px-2 py-0.5 rounded-full text-xs font-semibold bg-amber-100 text-amber-700 border border-amber-200">
|
||||
Pre-Release
|
||||
</span>
|
||||
</div>
|
||||
<h3 className="text-sm font-semibold text-gray-700 uppercase tracking-wider mb-3">Software Inventory & Vulnerability Analyzer</h3>
|
||||
<p className="text-sm text-gray-500 mb-4">
|
||||
Collects installed software from the Windows registry at enrollment start and completion.
|
||||
Produces a normalized inventory snapshot, detects software installed during enrollment (delta detection),
|
||||
|
||||
@@ -494,18 +494,18 @@ function OverallStatusBanner({ metrics }: { metrics: SlaMetricsResponse }) {
|
||||
const metCount = checks.filter(c => c.met).length;
|
||||
|
||||
return (
|
||||
<div className={`rounded-lg shadow p-4 mb-8 flex items-center justify-between ${
|
||||
<div className={`rounded-lg shadow p-4 mb-8 flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between ${
|
||||
allMet
|
||||
? "bg-green-50 dark:bg-green-900/20 border border-green-200 dark:border-green-800"
|
||||
: "bg-red-50 dark:bg-red-900/20 border border-red-200 dark:border-red-800"
|
||||
}`}>
|
||||
<div className="flex items-center space-x-3">
|
||||
<div className="flex items-center gap-3">
|
||||
{allMet ? (
|
||||
<svg className="h-8 w-8 text-green-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<svg className="h-8 w-8 shrink-0 text-green-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z" />
|
||||
</svg>
|
||||
) : (
|
||||
<svg className="h-8 w-8 text-red-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<svg className="h-8 w-8 shrink-0 text-red-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-2.5L13.732 4c-.77-.833-1.964-.833-2.732 0L4.082 16.5c-.77.833.192 2.5 1.732 2.5z" />
|
||||
</svg>
|
||||
)}
|
||||
@@ -518,19 +518,19 @@ function OverallStatusBanner({ metrics }: { metrics: SlaMetricsResponse }) {
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center space-x-4">
|
||||
<div className="flex flex-wrap items-center gap-x-4 gap-y-2 sm:justify-end">
|
||||
{checks.map((c) => (
|
||||
<div key={c.label} className="flex items-center space-x-1.5">
|
||||
<div key={c.label} className="flex items-center gap-1.5">
|
||||
{c.met ? (
|
||||
<svg className="h-4 w-4 text-green-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<svg className="h-4 w-4 shrink-0 text-green-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2.5} d="M5 13l4 4L19 7" />
|
||||
</svg>
|
||||
) : (
|
||||
<svg className="h-4 w-4 text-red-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<svg className="h-4 w-4 shrink-0 text-red-500" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2.5} d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
)}
|
||||
<span className={`text-sm ${c.met ? "text-green-700 dark:text-green-400" : "text-red-700 dark:text-red-400"}`}>
|
||||
<span className={`text-sm whitespace-nowrap ${c.met ? "text-green-700 dark:text-green-400" : "text-red-700 dark:text-red-400"}`}>
|
||||
{c.label}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -90,10 +90,7 @@ export function BuiltInSectionsList({ sections, loading = false, realmJoinWatche
|
||||
pills={
|
||||
<>
|
||||
{display.patternSummary && (
|
||||
// Secondary detail: hidden on narrow screens so the path keeps its width.
|
||||
<ContextPill title={display.patternTitle} className="hidden sm:inline-flex">
|
||||
{display.patternSummary}
|
||||
</ContextPill>
|
||||
<ContextPill title={display.patternTitle}>{display.patternSummary}</ContextPill>
|
||||
)}
|
||||
{condition && (
|
||||
<ContextPill title={condition.title}>
|
||||
|
||||
@@ -45,9 +45,11 @@ interface DiagnosticsPathRowProps {
|
||||
}
|
||||
|
||||
/**
|
||||
* One diagnostics path on ONE line: path (truncated, full value in the tooltip), description,
|
||||
* guard verdict, context pills, subfolder control, remove. Long lists stay scannable and the
|
||||
* add-row above them stays within reach.
|
||||
* One diagnostics path per row. From `sm` up everything sits on ONE line — path (truncated,
|
||||
* full value in the tooltip), description, guard verdict, context pills, subfolder control,
|
||||
* remove — so long lists stay scannable and the add-row above them stays within reach.
|
||||
* Below `sm` the path owns the first line(s) and wraps in full (a phone cannot hover a
|
||||
* tooltip, and a path cut to "C:\…" is useless); badges and controls flow onto the next line.
|
||||
*/
|
||||
export function DiagnosticsPathRow({
|
||||
path,
|
||||
@@ -62,8 +64,11 @@ export function DiagnosticsPathRow({
|
||||
pathClassName = "text-gray-700 dark:text-gray-200",
|
||||
}: DiagnosticsPathRowProps) {
|
||||
return (
|
||||
<div className={`flex min-w-0 items-center gap-2 rounded-md border px-2.5 py-1 ${className}`}>
|
||||
<span className={`min-w-0 flex-1 truncate font-mono text-xs ${pathClassName}`} title={title ?? path}>
|
||||
<div className={`flex min-w-0 flex-wrap items-center gap-x-2 gap-y-1 rounded-md border px-2.5 py-1 sm:flex-nowrap ${className}`}>
|
||||
<span
|
||||
className={`min-w-0 basis-full break-all font-mono text-xs sm:flex-1 sm:basis-auto sm:truncate sm:break-normal ${pathClassName}`}
|
||||
title={title ?? path}
|
||||
>
|
||||
{path}
|
||||
</span>
|
||||
{description && (
|
||||
|
||||
@@ -292,6 +292,10 @@ export const api = {
|
||||
`${API_BASE_URL}/api/metrics/device-journeys${qs({ days: String(days) })}`,
|
||||
globalDeviceJourneys: (days: number, tenantId?: string) =>
|
||||
`${API_BASE_URL}/api/global/metrics/device-journeys${qs({ days: String(days), tenantId })}`,
|
||||
// Verdict calibration matrix (operator-only; GlobalReadOrAdmin). Omit tenantId for the
|
||||
// cross-tenant aggregate. Daily rows are additive, so the window follows days honestly.
|
||||
globalVerdictCalibration: (days: number, tenantId?: string) =>
|
||||
`${API_BASE_URL}/api/global/metrics/verdict-calibration${qs({ days: String(days), tenantId })}`,
|
||||
geographic: (tenantId: string, days: number, groupBy: string) =>
|
||||
`${API_BASE_URL}/api/metrics/geographic${qs({ tenantId, days: String(days), groupBy })}`,
|
||||
globalGeographic: (days: number, groupBy: string, tenantId?: string) =>
|
||||
|
||||
@@ -261,6 +261,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
|
||||
{ id: "ga-usage", label: "Platform Usage", href: route("/admin/metrics/usage") },
|
||||
{ id: "ga-active-users", label: "Active Users", href: "/admin/presence" },
|
||||
{ id: "ga-mcp-usage", label: "MCP Usage", href: route("/admin/metrics/mcp-usage") },
|
||||
{ id: "ga-verdict-calibration", label: "Verdict Calibration", href: route("/admin/metrics/verdict-calibration") },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -210,6 +210,18 @@ export const SHARED_MANIFEST = {
|
||||
"name": "failureSource",
|
||||
"optional": false
|
||||
},
|
||||
{
|
||||
"name": "verdictPath",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "priorStatus",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "priorVerdictPath",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "reconcileReason",
|
||||
"optional": false
|
||||
|
||||
@@ -206,6 +206,18 @@
|
||||
"name": "failureSource",
|
||||
"optional": false
|
||||
},
|
||||
{
|
||||
"name": "verdictPath",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "priorStatus",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "priorVerdictPath",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "reconcileReason",
|
||||
"optional": false
|
||||
|
||||
Reference in New Issue
Block a user