fix(mcp): search keyword merge, pre-baked embedding model, fuse gating (review H2/H3/M8)

H2: search_events caller keywords are now lowercased and MERGED with
auto-extraction via resolveQueryKeywords() — previously `??` replaced
extraction entirely and case-sensitive keywords like ["BitLocker"]
silently returned zero results.

H3: the ~23 MB MiniLM model is pre-baked at Docker build time (warmup
via the built provider keeps the model name single-sourced) into
HF_CACHE_DIR=/app/.hf-cache and copied into the final stage, so a
scale-to-zero cold start never depends on the HuggingFace CDN; boot
indexing falls back to the fuse backend with a loud log instead of
crash-looping on init failure.

M8: new SearchProvider.semanticCapable flag (vector=true, fuse=false);
selectEventTypeCandidates gates the semantic path on it so inverted
Fuse scores no longer flow through cosine-calibrated thresholds, and
the search_events response now reports the real searchBackend
(keyword-only vs hybrid) plus the provider name.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/McpServer/autopilot-monitor-mcp/Dockerfile

@@ -11,16 +11,27 @@ COPY src/McpServer/autopilot-monitor-mcp/tsconfig.json ./
 COPY src/McpServer/autopilot-monitor-mcp/src/ ./src/
 RUN npm run build
 
+# Pre-bake the ~23 MB embedding model at image build time so a scale-to-zero cold
+# start never depends on the HuggingFace CDN (ephemeral FS would re-download on
+# every cold start; an unreachable CDN would crash-loop boot). Warming up via the
+# built provider keeps the model name/dtype single-sourced in
+# vector-search-provider.ts, which honors HF_CACHE_DIR.
+ENV HF_CACHE_DIR=/app/.hf-cache
+RUN node --input-type=module -e 'const { embed } = await import("/app/dist/vector-search-provider.js"); await embed("model prefetch warmup");' \
+  && test -d /app/.hf-cache
+
 # Stage 2: Production
 FROM node:22-slim
 WORKDIR /app
 
 ENV NODE_ENV=production
+ENV HF_CACHE_DIR=/app/.hf-cache
 
 COPY --from=build /app/package.json /app/package-lock.json ./
 RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force
 
 COPY --from=build /app/dist/ ./dist/
+COPY --from=build /app/.hf-cache/ ./.hf-cache/
 
 # Knowledge base rules (analyze, gather, ime-log-patterns)
 COPY rules/analyze/ ./rules/analyze/

src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-search.test.ts

@@ -20,6 +20,7 @@ import {
   orderBySelectivity,
   dedupeSynonymConcepts,
   entityRecallHint,
+  resolveQueryKeywords,
 } from '../tools/search.js';
 import { scanLexical } from '../search-provider.js';
 import type { SearchDocument } from '../search-provider.js';
@@ -327,6 +328,33 @@ describe('normalizeRawEvent (raw PascalCase → enriched camelCase)', () => {
   });
 });
 
+describe('resolveQueryKeywords (caller keywords are additive + lowercased)', () => {
+  it('lowercases caller keywords so they can match the lowercased corpus', () => {
+    // Regression: `keywords: ["BitLocker"]` used to flow through verbatim and match
+    // nothing — silent empty result.
+    expect(resolveQueryKeywords('encryption stuck', ['BitLocker'])).toEqual(['bitlocker', 'encryption', 'stuck']);
+  });
+
+  it('merges caller keywords with auto-extraction instead of replacing it', () => {
+    // Regression: `??` replaced extraction entirely, discarding query-derived recall.
+    const out = resolveQueryKeywords('bitlocker encryption failed', ['0x87D1041C']);
+    expect(out).toEqual(['0x87d1041c', 'bitlocker', 'encryption', 'failed']);
+  });
+
+  it('dedupes a caller keyword already extracted from the query', () => {
+    expect(resolveQueryKeywords('bitlocker failed', ['BitLocker'])).toEqual(['bitlocker', 'failed']);
+  });
+
+  it('drops empty / whitespace-only caller keywords', () => {
+    expect(resolveQueryKeywords('bitlocker failed', ['  ', '', ' TPM '])).toEqual(['tpm', 'bitlocker', 'failed']);
+  });
+
+  it('falls back to pure auto-extraction when no keywords are supplied', () => {
+    expect(resolveQueryKeywords('bitlocker failed')).toEqual(['bitlocker', 'failed']);
+    expect(resolveQueryKeywords('bitlocker failed', [])).toEqual(['bitlocker', 'failed']);
+  });
+});
+
 describe('queryHasProblemIntent', () => {
   it('detects failure-intent stems (incl. inflections)', () => {
     expect(queryHasProblemIntent(['error'])).toBe(true);
@@ -434,8 +462,9 @@ describe('extractEventTypeCandidates — ranking + broad-catalog coverage', () =
 
 describe('selectEventTypeCandidates (lexical + semantic blend)', () => {
   // Minimal fake vector provider returning canned semantic hits.
-  const provider = (eventTypes: string[], opts: { throws?: boolean; size?: number } = {}) => ({
+  const provider = (eventTypes: string[], opts: { throws?: boolean; size?: number; semanticCapable?: boolean } = {}) => ({
     name: 'fake',
+    semanticCapable: opts.semanticCapable ?? true,
     size: opts.size ?? eventTypes.length,
     index: async () => {},
     search: async () => {
@@ -480,6 +509,14 @@ describe('selectEventTypeCandidates (lexical + semantic blend)', () => {
     expect(out.candidates).toEqual(['bitlocker_status']);
     expect(out.semanticTypeScores.size).toBe(0);
   });
+
+  it('skips semantic selection for a non-semantic (fuse) backend — its scores are not cosines', async () => {
+    // Regression: gating on index PRESENCE alone let SEARCH_BACKEND=fuse feed inverted
+    // fuzzy-match scores through cosine-calibrated thresholds as fake cosines.
+    const out = await selectEventTypeCandidates('bitlocker', ['bitlocker'], provider(['tpm_status'], { semanticCapable: false }));
+    expect(out.candidates).toEqual(['bitlocker_status']);
+    expect(out.semanticTypeScores.size).toBe(0);
+  });
 });
 
 describe('scoreEvent — semantic floor (the core recall fix)', () => {

src/McpServer/autopilot-monitor-mcp/src/fuse-search-provider.ts

@@ -12,6 +12,9 @@ import { scanLexical } from './search-provider.js';
 
 export class FuseSearchProvider implements SearchProvider {
   readonly name = 'fuse';
+  // Inverted Fuse scores are NOT cosine similarities — consumers with
+  // cosine-calibrated thresholds must skip semantic treatment for this backend.
+  readonly semanticCapable = false;
   private documents: SearchDocument[] = [];
   private fuse: Fuse<SearchDocument> | null = null;
 

src/McpServer/autopilot-monitor-mcp/src/index.ts

@@ -9,6 +9,7 @@ import { registerResources } from './resources.js';
 import { registerPrompts } from './prompts.js';
 import { loadKnowledgeDocs } from './knowledge-base.js';
 import { createSearchProvider } from './search-factory.js';
+import type { SearchBackend, SearchProvider } from './search-provider.js';
 import { buildEventTypeSearchDocs } from './resource-catalog.js';
 import { createOAuthRouter } from './oauth.js';
 import { accessGuard } from './access-guard.js';
@@ -39,18 +40,38 @@ if (!process.env.MCP_PUBLIC_URL) {
 
 console.error('Loading knowledge base documents…');
 const docs = await loadKnowledgeDocs(RULES_DIR);
-
-console.error(`Initializing search provider (${docs.length} documents)…`);
-const knowledgeBase = await createSearchProvider();
-await knowledgeBase.index(docs);
-console.error(`Search provider ready: ${knowledgeBase.name} — ${knowledgeBase.size} documents indexed.`);
-
-// Separate tiny provider over the event-type catalog → semantic candidate selection
-// for event search ("app stuck downloading" → download_progress/do_telemetry). Shares
-// the embedder singleton with the knowledge base, so this adds ~no memory.
 const eventTypeDocs = buildEventTypeSearchDocs();
-const eventTypeIndex = await createSearchProvider();
-await eventTypeIndex.index(eventTypeDocs);
+
+async function buildSearchIndexes(backend?: SearchBackend): Promise<{
+  knowledgeBase: SearchProvider;
+  eventTypeIndex: SearchProvider;
+}> {
+  const kb = await createSearchProvider(backend);
+  await kb.index(docs);
+  // Separate tiny provider over the event-type catalog → semantic candidate selection
+  // for event search ("app stuck downloading" → download_progress/do_telemetry). Shares
+  // the embedder singleton with the knowledge base, so this adds ~no memory.
+  const et = await createSearchProvider(backend);
+  await et.index(eventTypeDocs);
+  return { knowledgeBase: kb, eventTypeIndex: et };
+}
+
+// Boot indexing must never crash-loop the container: the vector provider's first
+// index() call loads the embedding model, and although the Docker image pre-bakes
+// the model cache, a missing/corrupt cache falls through to a HuggingFace CDN
+// download — unreachable CDN or blocked egress would otherwise be a top-level
+// rejection killing every scale-to-zero cold start. Degrade to the keyword (fuse)
+// backend instead; search quality drops but the server stays up.
+console.error(`Initializing search provider (${docs.length} documents)…`);
+let knowledgeBase: SearchProvider;
+let eventTypeIndex: SearchProvider;
+try {
+  ({ knowledgeBase, eventTypeIndex } = await buildSearchIndexes());
+} catch (err) {
+  console.error('[startup] Search index initialization failed (embedding model unavailable?) — falling back to the keyword (fuse) backend:', err);
+  ({ knowledgeBase, eventTypeIndex } = await buildSearchIndexes('fuse'));
+}
+console.error(`Search provider ready: ${knowledgeBase.name} — ${knowledgeBase.size} documents indexed.`);
 console.error(`Event-type index ready: ${eventTypeIndex.name} — ${eventTypeIndex.size} types indexed.`);
 
 // Server-level guidance. The host surfaces this once per connection, so it is

src/McpServer/autopilot-monitor-mcp/src/search-provider.ts

@@ -33,6 +33,16 @@ export interface SearchProvider {
   /** Human-readable backend name (e.g. "vector/all-MiniLM-L6-v2", "fuse"). */
   readonly name: string;
 
+  /**
+   * True only when `search()` scores are genuine embedding cosine similarities.
+   * Consumers that apply cosine-calibrated thresholds or floors (e.g. the
+   * event-type candidate selection's SEMANTIC_TYPE_MIN_SCORE and the per-event
+   * semantic floor) MUST gate on this flag: a fuzzy-text backend's inverted
+   * Fuse scores live on a different scale, and flowing them through cosine
+   * thresholds produces arbitrary candidate selection and floor values.
+   */
+  readonly semanticCapable: boolean;
+
   /** Number of currently indexed documents. */
   readonly size: number;
 

src/McpServer/autopilot-monitor-mcp/src/tools/search.ts

@@ -348,7 +348,12 @@ export type EventTypeCandidates = { candidates: string[]; semanticTypeScores: Ma
  * semantic signals: keyword/prefix matches first (high precision), then the semantically
  * closest types from the vector-indexed catalog (recall for queries whose words don't
  * appear in any type name, e.g. "app stuck downloading" → download_progress). Degrades to
- * keyword-only when no vector index is available (fuse backend / not yet indexed / error).
+ * keyword-only when no semantic index is available (fuse backend / not yet indexed / error).
+ *
+ * The semantic path is gated on `semanticCapable`, not mere index presence: the
+ * SEMANTIC_TYPE_MIN_SCORE threshold and the downstream per-event floor are calibrated
+ * for embedding cosines — a fuse index would feed inverted fuzzy-match scores through
+ * them as fake cosines, selecting arbitrary candidates.
  *
  * The returned `semanticTypeScores` map carries each semantically-selected type's cosine
  * score downstream so scoreEvent() can floor (not discard) those events — without it the
@@ -361,7 +366,7 @@ export async function selectEventTypeCandidates(
 ): Promise<EventTypeCandidates> {
   const keywordCandidates = extractEventTypeCandidates(keywords);
   const semanticTypeScores = new Map<string, number>();
-  if (!eventTypeIndex || eventTypeIndex.size === 0) {
+  if (!eventTypeIndex || !eventTypeIndex.semanticCapable || eventTypeIndex.size === 0) {
     return { candidates: keywordCandidates, semanticTypeScores };
   }
 
@@ -506,6 +511,21 @@ function extractKeywords(query: string): string[] {
     .filter((w) => (w.length > 2 || DOMAIN_SHORT_KEYWORDS.has(w)) && (DOMAIN_SHORT_KEYWORDS.has(w) || !KEYWORD_STOP_WORDS.has(w)));
 }
 
+/**
+ * Resolve the final keyword set: caller-supplied `keywords` are ADDITIVE to
+ * auto-extraction, never a replacement — the schema promises "additional", and
+ * replacing silently discarded all query-derived recall. Caller keywords are
+ * also lowercased because every downstream matcher (candidate selection,
+ * scoreEvent, specificity) compares against lowercased text — an uppercase
+ * keyword like "BitLocker" matched nothing and produced a silent empty result.
+ */
+export function resolveQueryKeywords(query: string, callerKeywords?: string[]): string[] {
+  const supplied = (callerKeywords ?? [])
+    .map((k) => k.trim().toLowerCase())
+    .filter((k) => k.length > 0);
+  return [...new Set([...supplied, ...extractKeywords(query)])];
+}
+
 // ── Error-code fallback ─────────────────────────────────────────────────
 
 /**
@@ -953,7 +973,7 @@ export function registerSearchTools(
         minScore: z.coerce.number().min(0).max(1).optional()
           .describe('Minimum relevance score (0-1). Defaults: 0.1 (fast) / 0.05 (deep). Lower surfaces weaker matches.'),
         keywords: z.array(z.string()).optional()
-          .describe('Additional exact keywords for matching. Auto-extracted from query if omitted.'),
+          .describe('Additional exact keywords, merged (case-insensitively) with those auto-extracted from the query.'),
         guaranteedTopRanked: z.coerce.number().min(0).max(50).optional().default(3)
           .describe('How many top results (by pure relevance score) are locked to the head in rank order, exempt from cross-session diversification. Default 3 keeps the strongest hits on top while spreading the rest across sessions. Set =topK to trust the ranking and disable diversification entirely; set 0 for maximum session diversity. Ignored for single-session (sessionId) searches.'),
       },
@@ -967,7 +987,7 @@ export function registerSearchTools(
         const minScore = args.minScore ?? preset.defaultMinScore;
 
         // Extract keywords FIRST so we can use them for index-based pre-filtering
-        const queryKeywords = keywords ?? extractKeywords(query);
+        const queryKeywords = resolveQueryKeywords(query, keywords);
         if (queryKeywords.length === 0) {
           return toolResultText(
             { query, resultCount: 0, results: [], note: 'No searchable keywords extracted from query.' },
@@ -1012,7 +1032,11 @@ export function registerSearchTools(
 
         return toolResultText({
           query,
-          searchBackend: 'hybrid-keyword-semantic',
+          // Honest reporting: semantic type-selection contributed only when the
+          // per-type cosine map is non-empty. Single-session, legacy-fallback,
+          // fuse-backend and embedder-failure paths all rank purely by keyword.
+          searchBackend: semanticTypeScores.size > 0 ? 'hybrid-keyword-semantic' : 'keyword-only',
+          searchProvider: eventTypeIndex?.name,
           depth,
           searchMethod,
           keywordsUsed: queryKeywords,

src/McpServer/autopilot-monitor-mcp/src/vector-search-provider.ts

@@ -19,7 +19,14 @@ let embedder: FeatureExtractionPipeline | null = null;
 async function getEmbedder(): Promise<FeatureExtractionPipeline> {
   if (!embedder) {
     // Dynamic import + cast to avoid TS2590 from the overloaded pipeline() signature
-    const { pipeline } = await import('@huggingface/transformers');
+    const { pipeline, env } = await import('@huggingface/transformers');
+    // The library's default model cache lives inside node_modules/@huggingface/
+    // transformers/.cache — an npm-layout implementation detail. HF_CACHE_DIR pins
+    // it to a stable path so the Docker build can pre-bake the model and a
+    // scale-to-zero cold start never depends on the HuggingFace CDN.
+    if (process.env.HF_CACHE_DIR) {
+      env.cacheDir = process.env.HF_CACHE_DIR;
+    }
     embedder = (await (pipeline as Function)('feature-extraction', MODEL_NAME, {
       dtype: 'q8',
     })) as FeatureExtractionPipeline;
@@ -57,6 +64,7 @@ interface StoredDocument extends SearchDocument {
 
 export class VectorSearchProvider implements SearchProvider {
   readonly name = `vector/${MODEL_NAME}`;
+  readonly semanticCapable = true;
   private documents: StoredDocument[] = [];
 
   get size(): number {