diff --git a/docs/backend/verdict-calibration.md b/docs/backend/verdict-calibration.md index 02b96779..9355c822 100644 --- a/docs/backend/verdict-calibration.md +++ b/docs/backend/verdict-calibration.md @@ -100,7 +100,11 @@ and is wiped with the tenant on offboarding. 20 eligible**, the three override counters, a today-anchored 7d-vs-28d share trend (`window7`, `baseline28`, `lift` null without baseline) — plus totals, the trend basis and the active `alerts[]`. Empty but well-formed before the first sweep. -* MCP `get_verdict_calibration` (platform scope: GA + Global Reader). +* MCP `get_verdict_calibration` (platform scope: GA + Global Reader). Same endpoint, LLM-shaped in + `verdict-calibration-shape.ts`: per-row `window7.sessions` / `baseline28.sessions` are dropped (the + denominators live once in `trend`), `reEnrollRatePct` / `lift` are always present (`null` = withheld), + and `minSharePct` / `top` trim the one-session tail — rows carrying overrides are never trimmed and + everything dropped is reported in `omitted`. The wire shape the admin page consumes is untouched. * Portal: Global Admin → Metrics → Verdict Calibration (`/admin/metrics/verdict-calibration`): matrix grouped by origin, status pill, `derived n` gray pill, trend glyph (arrow only from 5 window hits and lift ≥ 2 / ≤ 0.5), re-enrollment rate or `— (n=…)`, overrides @@ -154,6 +158,7 @@ Kill switch `VerdictCalibrationRadarDisabled`. * `src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetVerdictCalibrationFunction.cs` * `src/Backend/AutopilotMonitor.Functions/Helpers/VerdictCalibrationRadar.cs`, `Services/MaintenanceService.VerdictCalibrationRadar.cs` * `src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts` — `get_verdict_calibration` +* `src/McpServer/autopilot-monitor-mcp/src/verdict-calibration-shape.ts` — LLM-facing response shaping (`minSharePct` / `top`, explicit nulls) * `src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionVerdictCalibration.tsx` * Tests: `VerdictPathTests`, `VerdictCalibrationTests`, `VerdictCalibrationRadarTests` * Related: [Silent-Session Classification](silent-session-classification.md), [Device Journeys](device-journeys.md), [Rule Regression Radar](rule-regression-radar.md) diff --git a/docs/log.md b/docs/log.md index 2542443f..9607368c 100644 --- a/docs/log.md +++ b/docs/log.md @@ -2,6 +2,7 @@ ## 2026-08-23 +* **Update**: `backend/verdict-calibration.md` — MCP `get_verdict_calibration` response shaping: per-row trend denominators dropped, explicit nulls, `minSharePct` / `top` with an `omitted` report. * **Update**: `backend/verdict-calibration.md` — share-regression radar restricted to backend-decided paths; agent-declared/registration paths (customer workflow mix) and derived legacy paths no longer alert, stale episodes re-arm themselves. * **Update**: `backend/time-attribution.md`, `backend/device-journeys.md`, `backend/verdict-calibration.md` — the rolling maintenance sweeps (time attribution, device journeys, verdict calibration, calibration-radar discovery) now share ONE projected 35-day cross-tenant Sessions scan per tick (`LoadSweepWindowSessionsAsync` + `MaintenanceSweepSessionProjection`, pinned by `MaintenanceSweepProjectionEquivalenceTests`) instead of four full-table drains; each sweep slices its own window by StartedAt. * **New**: `backend/verdict-calibration.md` — the classifier thermometer: every status write now stamps `VerdictPath` (required parameter on `UpdateSessionStatusAsync`; `EnrollmentTimeoutClassifier` returns its rule id), overriding a verdict preserves `PriorStatus`/`PriorVerdictPath`, `VerdictPathDerivation` attributes pre-instrumentation rows read-side (`legacy:*` when ambiguous). New daily aggregate `VerdictCalibrationAggregates` (per-path count/derived/eligible7d/reEnrolled7d/overridden-by-admin|late|other, sweep after the device-journey pass, 180d retention), `GET global/metrics/verdict-calibration`, MCP `get_verdict_calibration`, admin metrics section, and the `VerdictCalibrationRadar` (share / silence-share / r6 evidence-gap episodes in the tracker keyspace `verdictcalibration|`, `VerdictCalibrationDrift` ops event, no tenant bell). diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/verdict-calibration-shape.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/verdict-calibration-shape.test.ts new file mode 100644 index 00000000..dc917e5c --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/verdict-calibration-shape.test.ts @@ -0,0 +1,83 @@ +import { describe, it, expect } from 'vitest'; +import { shapeVerdictCalibration, type VerdictCalibrationPathRow } from '../verdict-calibration-shape.js'; + +function row(verdictPath: string, count: number, sharePct: number, extra: Partial = {}): VerdictCalibrationPathRow { + return { + verdictPath, + status: 'Succeeded', + count, + sharePct, + overriddenByAdmin: 0, + overriddenByLateCompletion: 0, + overriddenOther: 0, + window7: { count, sessions: 1000, sharePct }, + baseline28: { count, sessions: 2000, sharePct }, + ...extra, + }; +} + +function payload(paths: VerdictCalibrationPathRow[]) { + return { + success: true, + totals: { sessions: 1000 }, + trend: { windowSessions: 1000, baselineSessions: 2000 }, + paths, + alerts: [], + }; +} + +describe('shapeVerdictCalibration', () => { + it('drops the per-row denominators and makes withheld values explicit null', () => { + const out = shapeVerdictCalibration(payload([row('agent:complete', 500, 50)])) as any; + expect(out.paths[0].window7).toEqual({ count: 500, sharePct: 50 }); + expect(out.paths[0].baseline28).toEqual({ count: 500, sharePct: 50 }); + expect(out.paths[0]).toHaveProperty('reEnrollRatePct', null); + expect(out.paths[0]).toHaveProperty('lift', null); + expect(out.omitted).toBeUndefined(); + expect(out.trend).toEqual({ windowSessions: 1000, baselineSessions: 2000 }); + }); + + it('keeps real numeric values (0 is not null)', () => { + const out = shapeVerdictCalibration(payload([row('legacy:r3', 4, 0.1, { reEnrollRatePct: 0, lift: 0 })])) as any; + expect(out.paths[0].reEnrollRatePct).toBe(0); + expect(out.paths[0].lift).toBe(0); + }); + + it('minSharePct trims the long tail and reports what was omitted', () => { + const out = shapeVerdictCalibration(payload([ + row('agent:complete', 900, 90), + row('legacy:r4', 10, 1), + row('manual:succeeded', 1, 0.1), + ]), { minSharePct: 1 }) as any; + expect(out.paths.map((p: any) => p.verdictPath)).toEqual(['agent:complete', 'legacy:r4']); + expect(out.omitted).toEqual({ paths: 1, sessions: 1, reason: expect.stringContaining('share < 1%') }); + }); + + it('top keeps the N largest rows by backend order', () => { + const out = shapeVerdictCalibration(payload([ + row('a', 50, 50), row('b', 30, 30), row('c', 20, 20), + ]), { top: 2 }) as any; + expect(out.paths.map((p: any) => p.verdictPath)).toEqual(['a', 'b']); + expect(out.omitted.paths).toBe(1); + expect(out.omitted.reason).toContain('rank > 2'); + }); + + it('never trims a row that carries overrides — that is the signal', () => { + const out = shapeVerdictCalibration(payload([ + row('agent:complete', 999, 99.9), + row('sweep:r5_incomplete', 1, 0.1, { overriddenByLateCompletion: 3 }), + ]), { minSharePct: 5, top: 1 }) as any; + expect(out.paths.map((p: any) => p.verdictPath)).toEqual(['agent:complete', 'sweep:r5_incomplete']); + expect(out.omitted).toBeUndefined(); + }); + + it('does not mutate the input and passes non-matrix payloads through', () => { + const input = payload([row('x', 1, 100)]); + const snapshot = JSON.stringify(input); + shapeVerdictCalibration(input, { top: 0 }); + expect(JSON.stringify(input)).toBe(snapshot); + const err = { success: false, message: 'nope' }; + expect(shapeVerdictCalibration(err)).toBe(err); + expect(shapeVerdictCalibration(null)).toBeNull(); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts index 40e7e175..9378d641 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts @@ -5,6 +5,7 @@ import { withToolTelemetry } from '../telemetry.js'; import { getResourceContent, assertKnownEventType } from '../resource-catalog.js'; import { READ_ONLY, READ_ONLY_OPEN, MUTATING, MAX_RESULT_SIZE_CHARS, toolResultText, SessionIdSchema, TenantGuidSchema, tenantIdDescription } from './shared.js'; import { toolError } from './error-handler.js'; +import { shapeVerdictCalibration } from '../verdict-calibration-shape.js'; /** * Non-sensitive tenant fields safe to surface to the model. Keep-list (not @@ -854,23 +855,31 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo 'reEnrollRatePct is null below 20 eligible), the correction stream attributed to the PRIOR path ' + '(overriddenByAdmin / overriddenByLateCompletion / overriddenOther — a path that is frequently ' + 'overridden was wrong), and a 7d-vs-28d share trend (window7, baseline28, lift; lift null without ' + - 'baseline). Read it as calibration: a Succeeded path whose re-enrollment rate sits at Failed level ' + - 'reconciles too boldly; an Incomplete path whose rate matches the Succeeded background is too ' + - 'cautious; a rising sweep:*/maxlife:* share is an agent-liveness signal, not a classifier one. ' + - 'Omit tenantId for the cross-tenant aggregate, or pass tenantId to scope to one tenant. Rows come ' + - 'from the 2-hourly maintenance sweep (computedAt); empty before its first run.', + 'baseline; window/baseline denominators are the single `trend.windowSessions` / `trend.baselineSessions`). ' + + 'reEnrollRatePct and lift are always present — null means withheld, not missing. Read it as calibration: ' + + 'a Succeeded path whose re-enrollment rate sits at Failed level reconciles too boldly; an Incomplete ' + + 'path whose rate matches the Succeeded background is too cautious; a rising sweep:*/maxlife:* share ' + + 'is an agent-liveness signal, not a classifier one. Omit tenantId for the cross-tenant aggregate, or ' + + 'pass tenantId to scope to one tenant. minSharePct / top trim the long tail of one-session paths ' + + '(rows carrying overrides are never trimmed); what was dropped is reported in `omitted`, totals and ' + + 'trend always cover the full window. Rows come from the 2-hourly maintenance sweep (computedAt); ' + + 'empty before its first run.', inputSchema: { tenantId: z.string().optional().describe('Optional — scope to one tenant. Omit for the cross-tenant aggregate.'), days: z.coerce.number().int().min(1).max(180).optional() .describe('Window in days (default 30, max 180 — aggregate retention). The 7d/28d trend is always today-anchored.'), + minSharePct: z.coerce.number().min(0).max(100).optional() + .describe('Drop paths whose share of the window is below this percentage (e.g. 1). Paths with overrides are always kept.'), + top: z.coerce.number().int().min(1).max(100).optional() + .describe('Keep only the N largest paths by session count. Paths with overrides are always kept.'), }, annotations: READ_ONLY, }, async (args) => withToolTelemetry('get_verdict_calibration', args, async () => { try { - const { tenantId, days } = args; + const { tenantId, days, minSharePct, top } = args; const data = await apiFetch(`/api/global/metrics/verdict-calibration${buildQuery({ tenantId, days })}`); - return toolResultText(data, MAX_RESULT_SIZE_CHARS.adminStream); + return toolResultText(shapeVerdictCalibration(data, { minSharePct, top }), MAX_RESULT_SIZE_CHARS.adminStream); } catch (error: unknown) { return toolError('get_verdict_calibration', args, error); } diff --git a/src/McpServer/autopilot-monitor-mcp/src/verdict-calibration-shape.ts b/src/McpServer/autopilot-monitor-mcp/src/verdict-calibration-shape.ts new file mode 100644 index 00000000..c2bd32a4 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/verdict-calibration-shape.ts @@ -0,0 +1,102 @@ +/** + * LLM-facing shaping of the verdict-calibration matrix (GET /api/global/metrics/verdict-calibration). + * + * The backend wire shape is shared with the admin metrics page and stays untouched; this layer only + * trims what a model does not need and makes absent values explicit: + * - `window7.sessions` / `baseline28.sessions` repeat `trend.windowSessions` / `trend.baselineSessions` + * on every row — dropped (the denominators live once in `trend`). + * - `reEnrollRatePct` / `lift` are omitted by the backend's WhenWritingNull serializer when null; + * a missing key is easy to misread as "not applicable", so both are always present (null = withheld). + * - `minSharePct` / `top` cut the long tail of one-session paths. Rows carrying overrides are never + * cut (an override on a rare path is the signal the tool exists for). What was dropped is reported + * in `omitted` so a filtered response never reads as complete coverage. + */ + +interface TrendCell { + count: number; + sessions?: number; + sharePct: number; +} + +export interface VerdictCalibrationPathRow { + verdictPath: string; + status: string; + count: number; + sharePct: number; + overriddenByAdmin: number; + overriddenByLateCompletion: number; + overriddenOther: number; + reEnrollRatePct?: number | null; + lift?: number | null; + window7: TrendCell; + baseline28: TrendCell; + [key: string]: unknown; +} + +export interface VerdictCalibrationShapeOptions { + /** Drop rows whose share (in the window) is below this percentage. */ + minSharePct?: number; + /** Keep only the first N rows (the backend orders by count descending). */ + top?: number; +} + +export interface VerdictCalibrationOmitted { + paths: number; + sessions: number; + reason: string; +} + +function hasOverrides(row: VerdictCalibrationPathRow): boolean { + return (row.overriddenByAdmin ?? 0) + (row.overriddenByLateCompletion ?? 0) + (row.overriddenOther ?? 0) > 0; +} + +function trimCell(cell: TrendCell | undefined): TrendCell { + if (!cell) return { count: 0, sharePct: 0 }; + const { sessions: _sessions, ...rest } = cell; + return rest; +} + +/** + * Pure: returns a new object; the input is not mutated. Non-object or unsuccessful payloads + * (error envelopes) pass through unchanged so the caller's error handling still sees them. + */ +export function shapeVerdictCalibration(data: unknown, options: VerdictCalibrationShapeOptions = {}): unknown { + if (!data || typeof data !== 'object' || !Array.isArray((data as { paths?: unknown }).paths)) return data; + const payload = data as { paths: VerdictCalibrationPathRow[] } & Record; + + const { minSharePct, top } = options; + const kept: VerdictCalibrationPathRow[] = []; + let omittedPaths = 0; + let omittedSessions = 0; + + payload.paths.forEach((row, index) => { + const belowShare = minSharePct != null && row.sharePct < minSharePct; + const beyondTop = top != null && index >= top; + if ((belowShare || beyondTop) && !hasOverrides(row)) { + omittedPaths++; + omittedSessions += row.count ?? 0; + return; + } + kept.push({ + ...row, + reEnrollRatePct: row.reEnrollRatePct ?? null, + lift: row.lift ?? null, + window7: trimCell(row.window7), + baseline28: trimCell(row.baseline28), + }); + }); + + const shaped: Record = { ...payload, paths: kept }; + if (omittedPaths > 0) { + const criteria = [ + minSharePct != null ? `share < ${minSharePct}%` : null, + top != null ? `rank > ${top}` : null, + ].filter(Boolean).join(' or '); + shaped.omitted = { + paths: omittedPaths, + sessions: omittedSessions, + reason: `${omittedPaths} path(s) with ${criteria} dropped (rows carrying overrides are always kept); totals and trend still cover the full window.`, + } satisfies VerdictCalibrationOmitted; + } + return shaped; +}