mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
feat(mcp-search): walk failure types first on problem queries; bigger budgets
Live validation showed "app install timeout" still returned only benign
app_install_started, even at depth=deep. Root cause: the per-type fan-out is
sequential and budget-bounded, so a high-volume benign type (app_install_started
— every app logs "Installing") is walked first and exhausts the wall-clock
budget before the rarer app_install_failed is fetched. The real signal lives in
the failure event's MESSAGE ("[Win32App] Installation is timeout"), not its type
name, so the discriminating word never lifts the failure type's walk order.
Proof: "installation timeout failed" (where "failed" matches the type name, so
app_install_failed ranks #1 and is walked first) surfaces those exact events @1.0.
- prioritizeFailureTypes: when queryHasProblemIntent, stable-partition candidate
event types so failure-signal names (fail/error/timeout/denied/crash/…) are
walked first — they survive the per-type cap AND are fetched before benign
high-volume types drain the budget. (Pattern excludes stall/stuck: "stall" is a
substring of "in[stall]" — same mid-word trap, caught by a unit test.)
- Bigger budgets per the "wait a bit longer for good results, don't abort early"
goal: fast 3->5 pages / 12s->20s, deep 20->30 pages / 40s->50s (kept under the
MCP client's ~60s tool-call timeout).
Tests: prioritizeFailureTypes unit tests (reorder + no-op); full MCP suite 145
passed. Requires rebuild+redeploy of the MCP Container App.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ import {
|
||||
extractEventTypeCandidates,
|
||||
selectEventTypeCandidates,
|
||||
diversifyBySession,
|
||||
prioritizeFailureTypes,
|
||||
} from '../tools/search.js';
|
||||
|
||||
type Page = { events?: Array<Record<string, unknown>>; nextLink?: string };
|
||||
@@ -352,6 +353,24 @@ describe('scoreEvent — synonym-aware matching', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('prioritizeFailureTypes', () => {
|
||||
it('moves failure-signal types ahead of benign ones, preserving relative order', () => {
|
||||
// The "app install timeout" trap: app_install_started (benign, high-volume) is walked
|
||||
// first and starves app_install_failed of the budget. Reorder → failure first.
|
||||
const out = prioritizeFailureTypes([
|
||||
'app_install_started', 'app_install_failed', 'app_install_completed', 'error_detected',
|
||||
]);
|
||||
expect(out).toEqual([
|
||||
'app_install_failed', 'error_detected', 'app_install_started', 'app_install_completed',
|
||||
]);
|
||||
});
|
||||
|
||||
it('is a no-op when no type is a failure signal', () => {
|
||||
const input = ['tpm_status', 'download_progress', 'desktop_arrived'];
|
||||
expect(prioritizeFailureTypes(input)).toEqual(input);
|
||||
});
|
||||
});
|
||||
|
||||
describe('diversifyBySession', () => {
|
||||
const ev = (sid: string, score: number) => ({ event: { _sessionId: sid }, score });
|
||||
|
||||
|
||||
@@ -37,6 +37,32 @@ export function extractEventTypeCandidates(keywords: string[]): string[] {
|
||||
.map((c) => c.et);
|
||||
}
|
||||
|
||||
/**
|
||||
* Event types whose NAME signals a failure/anomaly. Used to reorder the candidate walk when
|
||||
* the query has problem intent: the per-type fan-out is sequential and budget-bounded, so a
|
||||
* high-volume benign type (e.g. app_install_started — every app logs "Installing") walked
|
||||
* first can consume the wall-clock budget before a rarer failure type (app_install_failed)
|
||||
* is ever fetched. Walking failures first guarantees they are scanned within budget — the
|
||||
* key signal often lives in a failure event's message, not its type name (e.g. "install
|
||||
* timeout" is an app_install_failed event whose message reads "Installation is timeout").
|
||||
*/
|
||||
// NB: only tokens that appear in real failure TYPE NAMES, and none that are a substring of a
|
||||
// benign type name — e.g. "stall"/"stuck" are excluded because "stall" sits inside "in[stall]"
|
||||
// (those are message-level concepts, not type names).
|
||||
const FAILURE_TYPE_PATTERN = /fail|error|timeout|denied|crash|fault|abort|block|reject|unsupported/;
|
||||
|
||||
/**
|
||||
* Stable-partition candidate event types so failure-signal types come first, preserving the
|
||||
* relative order within each group. Applied before the per-type cap when the query signals a
|
||||
* problem, so failure types both survive the cap and are walked before benign high-volume types.
|
||||
*/
|
||||
export function prioritizeFailureTypes(types: string[]): string[] {
|
||||
const failures: string[] = [];
|
||||
const rest: string[] = [];
|
||||
for (const t of types) (FAILURE_TYPE_PATTERN.test(t) ? failures : rest).push(t);
|
||||
return [...failures, ...rest];
|
||||
}
|
||||
|
||||
// ── Cross-session paginated fan-out ───────────────────────────────────────
|
||||
//
|
||||
// The cross-session search walks the already-paginated /api/raw/events
|
||||
@@ -93,11 +119,20 @@ function clamp01(x: number): number {
|
||||
*/
|
||||
const LEGACY_FALLBACK_SESSION_CAP = 5;
|
||||
|
||||
/** Tier-1 (fast): a couple pages per type — still quick, but less easily truncated. */
|
||||
const SEMANTIC_BUDGET: FetchBudget = { maxPagesPerType: 3, wallClockMs: 12_000 };
|
||||
/**
|
||||
* "fast" depth: a few pages per type — still quick, but less easily truncated. The
|
||||
* wall-clock is generous enough that a high-volume benign type can't starve the walk
|
||||
* before the failure-priority reorder (selectEventTypeCandidates → prioritizeFailureTypes)
|
||||
* has surfaced the failure types.
|
||||
*/
|
||||
const SEMANTIC_BUDGET: FetchBudget = { maxPagesPerType: 5, wallClockMs: 20_000 };
|
||||
|
||||
/** Tier-3 (deep): wide multi-page recall, bounded by a generous wall-clock. */
|
||||
const DEEP_BUDGET: FetchBudget = { maxPagesPerType: 20, wallClockMs: 40_000 };
|
||||
/**
|
||||
* "deep" depth: wide multi-page recall, bounded by a generous wall-clock. Prefer waiting a
|
||||
* little longer for complete results over an early truncation — kept under the MCP client's
|
||||
* typical 60s tool-call timeout with headroom.
|
||||
*/
|
||||
const DEEP_BUDGET: FetchBudget = { maxPagesPerType: 30, wallClockMs: 50_000 };
|
||||
|
||||
/**
|
||||
* `depth` presets for the unified search_events tool. "fast" is the former TIER-1 (quick,
|
||||
@@ -275,11 +310,14 @@ async function fetchSessionEvents(
|
||||
if (queryKeywords && queryKeywords.length > 0) {
|
||||
const { candidates: ranked, semanticTypeScores } = await selectEventTypeCandidates(query, queryKeywords, eventTypeIndex);
|
||||
if (ranked.length > 0) {
|
||||
// On a problem query, walk failure-signal types first so they survive the cap AND are
|
||||
// fetched before high-volume benign types exhaust the budget (see prioritizeFailureTypes).
|
||||
const ordered = queryHasProblemIntent(queryKeywords) ? prioritizeFailureTypes(ranked) : ranked;
|
||||
// Cap the per-type fan-out to the most-relevant types. If we dropped any,
|
||||
// recall is partial → flag truncated rather than silently narrowing. (Legacy
|
||||
// is reserved for the genuine "no keyword maps to any event type" case below.)
|
||||
const candidates = ranked.slice(0, MAX_EVENT_TYPE_CANDIDATES);
|
||||
const candidatesDropped = ranked.length > candidates.length;
|
||||
const candidates = ordered.slice(0, MAX_EVENT_TYPE_CANDIDATES);
|
||||
const candidatesDropped = ordered.length > candidates.length;
|
||||
const basePath = pickGlobalOrTenantPath('/api/global/raw/events', '/api/raw/events');
|
||||
const { events, truncated, anySucceeded } = await fetchEventsViaIndex(candidates, basePath, tenantId, budget);
|
||||
if (anySucceeded) {
|
||||
|
||||
Reference in New Issue
Block a user