diff --git a/src/McpServer/autopilot-monitor-mcp/package-lock.json b/src/McpServer/autopilot-monitor-mcp/package-lock.json index 2b9a8acc..e35f4526 100644 --- a/src/McpServer/autopilot-monitor-mcp/package-lock.json +++ b/src/McpServer/autopilot-monitor-mcp/package-lock.json @@ -1,15 +1,16 @@ { "name": "autopilot-monitor-mcp", - "version": "1.4.0", + "version": "1.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "autopilot-monitor-mcp", - "version": "1.4.0", + "version": "1.4.5", "dependencies": { "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.28.0", + "compression": "^1.8.1", "express": "^5.2.1", "fuse.js": "^7.1.0", "zod": "^3.22.0" @@ -18,6 +19,7 @@ "autopilot-monitor-mcp": "dist/index.js" }, "devDependencies": { + "@types/compression": "^1.8.1", "@types/express": "^5.0.6", "@types/node": "^22.0.0", "tsx": "^4.0.0", @@ -1447,6 +1449,17 @@ "assertion-error": "^2.0.1" } }, + "node_modules/@types/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-kCFuWS0ebDbmxs0AXYn6e2r2nrGAb5KwQhknjSPSPgJcGd8+HVSILlUyFhGqML2gk39HcG7D1ydW9/qpYkN00Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*", + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -1804,6 +1817,60 @@ "node": ">=18" } }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/content-disposition": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", @@ -3002,6 +3069,15 @@ "node": ">= 0.8" } }, + "node_modules/on-headers": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3304,6 +3380,26 @@ "node": ">= 18" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", diff --git a/src/McpServer/autopilot-monitor-mcp/package.json b/src/McpServer/autopilot-monitor-mcp/package.json index 46d8e224..c3a22f79 100644 --- a/src/McpServer/autopilot-monitor-mcp/package.json +++ b/src/McpServer/autopilot-monitor-mcp/package.json @@ -18,11 +18,13 @@ "dependencies": { "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.28.0", + "compression": "^1.8.1", "express": "^5.2.1", "fuse.js": "^7.1.0", "zod": "^3.22.0" }, "devDependencies": { + "@types/compression": "^1.8.1", "@types/express": "^5.0.6", "@types/node": "^22.0.0", "tsx": "^4.0.0", diff --git a/src/McpServer/autopilot-monitor-mcp/src/index.ts b/src/McpServer/autopilot-monitor-mcp/src/index.ts index 6c1b5c1d..bb3cc535 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/index.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/index.ts @@ -2,6 +2,7 @@ import { resolve, dirname } from 'node:path'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import express, { type ErrorRequestHandler } from 'express'; +import compression from 'compression'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { registerTools } from './tools.js'; @@ -188,6 +189,15 @@ const app = express(); // throttle in access-guard relies on. app.set('trust proxy', 1); +// gzip tool responses. JSON compresses extremely well (measured 5-30× on these +// payloads) and the CPU cost is sub-millisecond per response at the default +// level 6 (≈0.3 ms for 33 KB, ≈0.9 ms for 135 KB on a full vCPU). The default +// 1 KB threshold means small handshakes/errors are sent uncompressed, so the +// cost lands only on the large query results where the egress/transfer win is +// large. Requires the transport's enableJsonResponse (set below) — gzip over an +// SSE (text/event-stream) frame would risk breaking stream framing. +app.use(compression()); + // Tight body-size limit for /oauth/register, registered BEFORE the global // parser so the smaller limit wins (the global parser's body-already-parsed // short-circuit then skips re-parsing). RFC 7591 registration requests carry @@ -248,6 +258,14 @@ app.all('/mcp', async (req, res) => { const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined, // stateless: no session tracking + // Return a single buffered application/json response instead of an SSE + // (text/event-stream) frame. This server is stateless request/response and + // emits NO server→client notifications (see the rationale below), so SSE + // buys nothing — and a plain JSON body is what lets the compression + // middleware gzip large tool results (SSE would have to be left uncompressed + // to avoid breaking stream framing). Spec-compatible: the client already must + // Accept application/json for a POST, so no client regresses. + enableJsonResponse: true, }); // accessGuard ran runWithCaller({ platform role + delegated scope }) around next(), so the // caller's resolved scope is available here (and stays active through transport.handleRequest, where diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/search.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/search.ts index 33379080..e2e6ecef 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/search.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/search.ts @@ -30,8 +30,8 @@ const KNOWN_EVENT_TYPES = ALL_EVENT_TYPES; * catalog never starves the index walk into the weak legacy fallback. */ export function extractEventTypeCandidates(keywords: string[]): string[] { - return KNOWN_EVENT_TYPES - .map((et) => ({ et, hits: keywords.reduce((n, kw) => n + (prefixAwareMatch(et, kw) ? 1 : 0), 0) })) + return KNOWN_EVENT_TYPES_SPLIT + .map(({ et, words }) => ({ et, hits: keywords.reduce((n, kw) => n + (prefixAwareMatchPreSplit(et, words, kw) ? 1 : 0), 0) })) .filter((c) => c.hits > 0) .sort((a, b) => b.hits - a.hits) .map((c) => c.et); @@ -581,19 +581,43 @@ export function queryHasProblemIntent(keywords: string[]): boolean { const MIN_PREFIX_LEN = 4; -/** Check if keyword matches text via substring or shared prefix (min 4 chars). */ -function prefixAwareMatch(text: string, keyword: string): boolean { - if (text.includes(keyword)) return true; - // Split text into words and check shared prefix - const words = text.split(/[\s_\-.:,/]+/); +/** Split text into word tokens for prefix matching (same delimiters everywhere). */ +function splitWords(text: string): string[] { + return text.split(/[\s_\-.:,/]+/); +} + +/** True when any word shares a >= MIN_PREFIX_LEN prefix with the keyword. */ +function sharesWordPrefix(words: string[], keyword: string): boolean { + if (keyword.length < MIN_PREFIX_LEN) return false; for (const word of words) { - if (word.length < MIN_PREFIX_LEN || keyword.length < MIN_PREFIX_LEN) continue; + if (word.length < MIN_PREFIX_LEN) continue; const prefixLen = Math.min(word.length, keyword.length, MIN_PREFIX_LEN + 2); if (word.slice(0, prefixLen) === keyword.slice(0, prefixLen)) return true; } return false; } +/** Check if keyword matches text via substring or shared word-prefix (min 4 chars). */ +function prefixAwareMatch(text: string, keyword: string): boolean { + if (text.includes(keyword)) return true; + return sharesWordPrefix(splitWords(text), keyword); +} + +/** + * prefixAwareMatch over a corpus whose word splits are precomputed — used for the + * fixed ~130-type catalog, which is scanned in full twice per query (candidate + * selection + specificity). Reusing the precomputed words avoids re-splitting + * every type string on every keyword on every request. + */ +function prefixAwareMatchPreSplit(text: string, words: string[], keyword: string): boolean { + if (text.includes(keyword)) return true; + return sharesWordPrefix(words, keyword); +} + +/** The known event-type catalog with each type's word split precomputed once at module load. */ +const KNOWN_EVENT_TYPES_SPLIT: ReadonlyArray<{ et: string; words: string[] }> = + KNOWN_EVENT_TYPES.map((et) => ({ et, words: splitWords(et) })); + /** * A keyword that selects FEW event types is far more discriminating than one that selects many: * "bitlocker" maps to a single type (bitlocker_status), "failed" to ~7. We use this specificity to @@ -607,7 +631,7 @@ export function computeKeywordSpecificity(keywords: string[]): Map(); for (const kw of keywords) { if (m.has(kw)) continue; - m.set(kw, KNOWN_EVENT_TYPES.reduce((n, et) => n + (prefixAwareMatch(et, kw) ? 1 : 0), 0)); + m.set(kw, KNOWN_EVENT_TYPES_SPLIT.reduce((n, { et, words }) => n + (prefixAwareMatchPreSplit(et, words, kw) ? 1 : 0), 0)); } return m; } @@ -806,6 +830,7 @@ export function scoreEvent( boostFailures = false, semanticTypeScores?: Map, keywordSpecificity?: Map, + precomputedConcepts?: string[], ): ScoredEvent | null { const fields: Array<{ name: string; text: string; weight: number }> = [ { name: 'eventType', text: (e.eventType ?? '').toLowerCase(), weight: FIELD_WEIGHTS.eventType }, @@ -818,7 +843,10 @@ export function scoreEvent( // Collapse query keywords that share a synonym group to a single concept so two query words // expressing ONE idea (e.g. "stuck" + "timeout") don't double-count coverage/score against an // off-topic type (hello_wait_timeout) and outrank the actually-named entity (app_download_started). - const concepts = dedupeSynonymConcepts(queryKeywords); + // This is query-invariant, so the cross-session loop computes it once and passes it in to avoid + // recomputing the dedupe (and its Set/Map allocations) for every one of potentially thousands of + // fetched events; standalone callers (unit tests) omit it and get the same value computed here. + const concepts = precomputedConcepts ?? dedupeSynonymConcepts(queryKeywords); let totalScore = 0; const matched: string[] = []; @@ -1005,9 +1033,12 @@ export function registerSearchTools( await fetchSessionEvents(sessionId, tenantId, queryKeywords, preset.budget, query, eventTypeIndex, keywordSpecificity); const boostFailures = queryHasProblemIntent(queryKeywords); + // Synonym-collapse the query keywords ONCE — scoreEvent runs per fetched event (potentially + // thousands), and the concept set is identical across all of them. + const concepts = dedupeSynonymConcepts(queryKeywords); const scored: Array = []; for (let i = 0; i < events.length; i++) { - const result = scoreEvent(events[i], queryKeywords, boostFailures, semanticTypeScores, keywordSpecificity); + const result = scoreEvent(events[i], queryKeywords, boostFailures, semanticTypeScores, keywordSpecificity, concepts); if (result && result.score >= minScore) { scored.push({ ...result, index: i, event: events[i] }); } diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts index 91de8809..ee46650a 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts @@ -119,6 +119,9 @@ export function toolResultText( maxResultSizeChars: number, ): { content: Array<{ type: 'text'; text: string }>; _meta: Record } { return { + // Pretty-printed: the output is frequently eyeballed during interactive MCP use, and + // the indentation whitespace gzips away to almost nothing on the wire (compression + // middleware), so readability is kept essentially for free. content: [{ type: 'text' as const, text: JSON.stringify(data, null, 2) }], _meta: { 'anthropic/maxResultSizeChars': maxResultSizeChars }, }; diff --git a/src/McpServer/autopilot-monitor-mcp/src/vector-search-provider.ts b/src/McpServer/autopilot-monitor-mcp/src/vector-search-provider.ts index 0db562d3..5f511d13 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/vector-search-provider.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/vector-search-provider.ts @@ -54,17 +54,20 @@ export async function embed(text: string): Promise { // ── Helpers ────────────────────────────────────────────────── +/** + * Cosine similarity for L2-normalized vectors == their dot product. + * + * Every embedding here is unit-normalized at creation: embed() and index() both + * pass `normalize: true`, and the precomputed index is built via the same index() + * path. So ‖a‖ = ‖b‖ = 1 and the cosine denominator is 1 — computing the two norms + * + two sqrts + a divide per document is wasted work, and this runs over every + * indexed document on every query. A plain dot product is ~3× less arithmetic per + * document with identical results for normalized inputs. + */ function cosineSimilarity(a: number[], b: number[]): number { let dot = 0; - let normA = 0; - let normB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - normA += a[i] * a[i]; - normB += b[i] * b[i]; - } - const denom = Math.sqrt(normA) * Math.sqrt(normB); - return denom === 0 ? 0 : dot / denom; + for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; + return dot; } // ── Provider implementation ──────────────────────────────────