diff --git a/internal b/internal index 91cd3ad6..488d17b4 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit 91cd3ad6d7be7cb69ae46051e1f30049d19242a1 +Subproject commit 488d17b4ea1080dc565c6c72d3d7456c6368231c diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServiceMcpServerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServiceMcpServerTests.cs index 41577c7d..6d86df4e 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServiceMcpServerTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServiceMcpServerTests.cs @@ -77,17 +77,62 @@ public class HealthCheckServiceMcpServerTests } [Fact] - public async Task Check_TimeoutOrConnectionFailure_ReportsWarningForColdStart() + public async Task Check_Timeout_ReportsWarmingSoTheCallerCanRePoll() { var svc = BuildServiceWithFactory( new ThrowingHttpClientFactory(new TaskCanceledException("timeout"))); var check = await svc.CheckMcpServerAsync(); - // Failure to wake the scaled-to-zero container within the budget is a warning, - // not a hard outage (a re-check may still succeed once it finishes warming). + // A scaled-to-zero container that did not answer inside the budget is "warming", + // NOT a fault: the caller re-checks and the activation we triggered keeps running. + Assert.Equal("warming", check.Status); + // The default budget is pinned here because it is set nowhere else in the repo — + // the code default IS the live production value. + Assert.Contains("3s", check.Message); + } + + [Fact] + public async Task Check_Timeout_HonoursConfiguredBudget() + { + var svc = BuildServiceWithFactory( + new ThrowingHttpClientFactory(new TaskCanceledException("timeout")), + extraConfig: new Dictionary { ["McpServerHealthTimeoutSeconds"] = "7" }); + + var check = await svc.CheckMcpServerAsync(); + + Assert.Equal("warming", check.Status); + Assert.Contains("7s", check.Message); + } + + [Fact] + public async Task Check_ConnectionFailure_ReportsWarningNotWarmingAndKeepsTheHostOut() + { + var svc = BuildServiceWithFactory( + new ThrowingHttpClientFactory(new System.Net.Http.HttpRequestException("no such host mcp.example.test"))); + + var check = await svc.CheckMcpServerAsync(); + + // Reaching the network and failing to connect is a real warning — distinct from + // the expected cold-start path, so it must not be softened to "warming". Assert.Equal("warning", check.Status); - Assert.Contains("could not be reached", check.Message); + Assert.DoesNotContain("http", check.Message, System.StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("mcp.example.test", check.Message); + } + + [Theory] + [InlineData(12, false)] + [InlineData(750, false)] + [InlineData(751, true)] + [InlineData(2400, true)] + public void McpReachableMessage_marksOnlyAnswersAboveTheWarmThresholdAsColdStart(long elapsedMs, bool expectColdStart) + { + var message = HealthCheckService.McpReachableMessage(elapsedMs); + + // Both arms keep the word "reachable" — the healthy-path assertions rely on it. + Assert.Contains("reachable", message); + Assert.Contains($"{elapsedMs}ms", message); + Assert.Equal(expectColdStart, message.Contains("cold start")); } /// @@ -101,13 +146,21 @@ public class HealthCheckServiceMcpServerTests return BuildServiceWithFactory(new StubHttpClientFactory(handler)); } - private static HealthCheckService BuildServiceWithFactory(System.Net.Http.IHttpClientFactory factory) + private static HealthCheckService BuildServiceWithFactory( + System.Net.Http.IHttpClientFactory factory, + Dictionary? extraConfig = null) { + var settings = new Dictionary + { + ["McpServerUrl"] = "https://mcp.example.test", + }; + if (extraConfig != null) + { + foreach (var kv in extraConfig) settings[kv.Key] = kv.Value; + } + var config = new ConfigurationBuilder() - .AddInMemoryCollection(new Dictionary - { - ["McpServerUrl"] = "https://mcp.example.test", - }) + .AddInMemoryCollection(settings) .Build(); return new HealthCheckService( diff --git a/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs b/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs index 180ca3ce..869dca30 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs @@ -49,10 +49,12 @@ public class HealthCheckService }; // NOTE: CheckMcpServerAsync is deliberately NOT part of this batch. The MCP - // Container App scales to zero on idle, so a probe may need to wake it and can - // take many seconds. Bundling it here would block the entire dashboard on that - // one slow check. It is exposed via its own endpoint (GET health/mcp) that the - // frontend fetches independently and folds into the card grid incrementally. + // Container App scales to zero on idle; a cold probe answers "warming" after a + // short budget and the caller re-polls until the replica is up. That re-poll loop + // must stay off the blocking batch, and "warming" must never reach this aggregate + // — a scale-from-zero is expected behaviour, not a degraded platform. It is + // exposed via its own endpoint (GET health/mcp) that the frontend fetches + // independently and folds into the card grid incrementally. var checks = await Task.WhenAll( CheckStorageBackendAsync(includeEndpointUrls), CheckProcessingBackendAsync(includeEndpointUrls), @@ -244,13 +246,32 @@ public class HealthCheckService /// only included in the details when is set — /// callers must only enable it for Global Admins. /// - /// The Container App runs with minReplicas=0 and scales to zero on idle. Azure Container - /// Apps holds an inbound request while it activates a cold replica, so this probe doubles - /// as the wake signal — a generous timeout (McpServerHealthTimeoutSeconds, default - /// 30s) gives the cold start time to complete and report healthy. Only when the server - /// cannot be woken within that budget is it a warning; a reachable-but-erroring server - /// (non-2xx) is unhealthy. This is exposed via its own health/mcp endpoint, never - /// the blocking all-checks batch, so the dashboard never waits on the wake. + /// The Container App runs with minReplicas=0 and scales to zero on idle, and a cold start + /// is dominated by Container Apps activation, not by us: measured at 24.5s and 31.5s to + /// first byte, of which ~98% is KEDA scheduling plus the image mount and ~500ms is the + /// application (see internal/docs/mcp/docs-corpus.md, "Cold start"). No UI wait + /// covers that, so this probe deliberately does not try to sit it out — an earlier design + /// waited 30s as a "wake signal" and still timed out in 50 of 83 observed calls, i.e. it + /// bought a long wall in front of the same answer. + /// + /// + /// The probe therefore has two jobs: answer fast and honestly within + /// McpServerHealthTimeoutSeconds (default 3s — chosen to sit in the widest observed + /// latency gap, 2347ms..3930ms, so jitter cannot flip the verdict), and leave the wake + /// running. The inbound request has already reached the Container Apps activator by the + /// time we give up, so aborting our side does not undo the scale-from-zero it triggered; + /// the caller re-checks and a later probe finds the replica warm. + /// + /// + /// Status vocabulary: healthy = answered 2xx; warming = did not answer inside + /// the budget, a cold start is in progress and the caller should re-check (must NOT be + /// rated into any aggregate — it is expected, not a fault); warning = reachable but + /// the connection failed; unhealthy = answered non-2xx. Exposed via its own + /// health/mcp endpoint, never the blocking all-checks batch. + /// + /// + /// McpServerHealthTimeoutSeconds is set nowhere in the repo, infra or workflows, so + /// the default below IS the live production value — changing it changes production. /// /// internal async Task CheckMcpServerAsync(bool includeEndpointUrl = false) @@ -268,11 +289,9 @@ public class HealthCheckService { check.Details = new Dictionary { ["Server URL"] = baseUrl }; } - var timeoutSeconds = ParsePositiveInt("McpServerHealthTimeoutSeconds", 30); - - // A cold start that takes longer than this is treated as a successful wake rather - // than an instant-warm hit, so the message can say so. - const long WarmHitThresholdMs = 2500; + // 3s: warm answers were measured at 4..148ms over 14 days, so this is ~20x the warm + // maximum, while a cold start needs 20-30s and is unreachable for any UI wait. + var timeoutSeconds = ParsePositiveInt("McpServerHealthTimeoutSeconds", 3); try { @@ -293,24 +312,32 @@ public class HealthCheckService var version = await TryReadMcpVersionAsync(response); check.Status = "healthy"; - check.Message = sw.ElapsedMilliseconds > WarmHitThresholdMs - ? $"MCP server woke from idle and is reachable ({sw.ElapsedMilliseconds}ms)" - : $"MCP server reachable ({sw.ElapsedMilliseconds}ms)"; + check.Message = McpReachableMessage(sw.ElapsedMilliseconds); if (!string.IsNullOrWhiteSpace(version)) { check.Details ??= new Dictionary(); check.Details["Version"] = version; } } - catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException or HttpRequestException) + catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException) { - // Timeout or connection failure after the full wake budget. The probe held the - // request long enough for a cold replica to activate, so a failure here means the - // server could not be woken — surface as a warning (not a hard outage: the next - // re-check may still succeed once it finishes warming). + // Our own timeout: no external cancellation token reaches this path, so a + // cancellation here is always the budget above expiring. The scale-from-zero we + // triggered keeps running after we give up, so this is a re-check prompt, not a + // fault — and with minReplicas=0 it is the expected path for an idle server, + // which is why it logs at Information and must not colour any aggregate. + check.Status = "warming"; + check.Message = $"MCP server did not answer within {timeoutSeconds}s — it scales to zero when idle and is likely still starting (a cold start takes ~20-30s)."; + _logger.LogInformation(ex, "MCP server health probe found a cold server (no answer within {TimeoutSeconds}s); activation continues in the background", timeoutSeconds); + } + catch (HttpRequestException ex) + { + // Reached the network but could not connect — distinct from the cold-start path + // above and genuinely worth a warning. ex.Message can carry the host name, so it + // stays out of the user-visible text (same rule as the non-2xx branch). check.Status = "warning"; - check.Message = $"MCP server could not be reached within {timeoutSeconds}s — it may still be warming up. Re-check in a moment."; - _logger.LogWarning(ex, "MCP server health probe did not respond within {TimeoutSeconds}s (wake attempt failed)", timeoutSeconds); + check.Message = "MCP server unreachable (connection failed)"; + _logger.LogWarning(ex, "MCP server health probe could not connect"); } catch (Exception ex) { @@ -322,6 +349,24 @@ public class HealthCheckService return check; } + /// + /// An answer slower than this came from a replica that had just started, not from a warm + /// one, so the message can say so. 750ms is ~5x the warm maximum measured over 14 days + /// (4..148ms) and deliberately generous: those measurements come from a near-idle, + /// single-user system, and warm latency will rise under concurrency. It stays well below + /// the next observed cluster (2289ms), so the distinction remains sharp. + /// + private const long WarmHitThresholdMs = 750; + + /// + /// Wording for a reachable MCP server. Extracted so the threshold can be pinned by a test + /// without a real delay. Both arms keep the word "reachable". + /// + internal static string McpReachableMessage(long elapsedMs) => + elapsedMs > WarmHitThresholdMs + ? $"MCP server reachable after a cold start ({elapsedMs}ms)" + : $"MCP server reachable ({elapsedMs}ms)"; + /// /// Best-effort parse of the MCP /health JSON body for its version field. /// A missing or malformed body is non-fatal — the probe already proved reachability. diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index 86a7e4d4..ef5e424f 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -2336,6 +2336,7 @@ export interface HardwareRejectedResponse { export interface HealthCheck { name: string; description: string; + /** One of: "healthy" (working), "warning" (degraded but reachable), "unhealthy" (failing), "unknown" (not determined), "warming" (health/mcp only — the scale-to-zero MCP container did not answer inside the probe budget and is still starting; the caller should re-check, and it must NOT be rated as a fault in any aggregate or overall banner). */ status: string; message: string; details?: Record; diff --git a/src/Shared/AutopilotMonitor.Shared/Models/InfrastructureApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/InfrastructureApiModels.cs index 2f4c3f37..f2e03972 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/InfrastructureApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/InfrastructureApiModels.cs @@ -137,6 +137,13 @@ namespace AutopilotMonitor.Shared.Models { public string Name { get; set; } = string.Empty; public string Description { get; set; } = string.Empty; + /// + /// One of: "healthy" (working), "warning" (degraded but reachable), "unhealthy" + /// (failing), "unknown" (not determined), "warming" (health/mcp only — the + /// scale-to-zero MCP container did not answer inside the probe budget and is still + /// starting; the caller should re-check, and it must NOT be rated as a fault in any + /// aggregate or overall banner). + /// public string Status { get; set; } = "unknown"; public string Message { get; set; } = string.Empty; public Dictionary? Details { get; set; } diff --git a/src/Web/autopilot-monitor-web/app/health-check/page.tsx b/src/Web/autopilot-monitor-web/app/health-check/page.tsx index 8e6f525b..c5b729aa 100644 --- a/src/Web/autopilot-monitor-web/app/health-check/page.tsx +++ b/src/Web/autopilot-monitor-web/app/health-check/page.tsx @@ -8,7 +8,15 @@ import { useGlobalAdminUi } from '@/hooks/useGlobalAdminUi'; import { useState, useEffect, useCallback } from 'react'; import * as signalR from '@microsoft/signalr'; import { api } from '@/lib/api'; -import { isUrlDetail, visibleHealthChecks, visibleHealthDetails } from '@/lib/healthCheckView'; +import { + isUrlDetail, + visibleHealthChecks, + visibleHealthDetails, + resolveMcpCardState, + MCP_WARMING_POLL_MS, + MCP_PROBE_TIMEOUT_MS, +} from '@/lib/healthCheckView'; +import { useLatest } from '@/hooks/useLatest'; import type { DetailedHealthCheckResponse, HealthCheck, McpHealthCheckResponse } from '@/utils/wire-types.generated'; import { DocsLink } from "@/components/DocsLink"; import { DOCS_PATHS } from "@/lib/docsPaths"; @@ -81,6 +89,9 @@ export default function HealthCheckPage() { // dashboard on it. The card renders a "checking" state immediately and updates in place. const [mcpCheck, setMcpCheck] = useState(null); const [mcpLoading, setMcpLoading] = useState(false); + // Counts only the automatic warming re-probes; reset solely by the Re-check button, so + // a reset inside performMcpCheck can never make the poll loop immortal. + const [mcpAttempts, setMcpAttempts] = useState(0); // Portal build stamp: /version.json is a static file from this same origin (generated at // build time, Cache-Control: no-store) — plain fetch, no auth. Card stays hidden when the // file is absent (e.g. a dev server without the prebuild step). @@ -113,14 +124,16 @@ export default function HealthCheckPage() { // Separate, non-blocking probe for the MCP server. Runs concurrently with — and // independently of — performHealthCheck so a cold-starting MCP container never holds - // up the other cards. Resolves to healthy / warning / unhealthy; the card shows a - // "checking" state until then. + // up the other cards. The server answers "warming" for a scaled-to-zero container + // instead of holding the request; the effect below then re-probes until it is up. const performMcpCheck = useCallback(async () => { setMcpLoading(true); try { let data: McpHealthCheckResponse; try { - data = await fetchJson(api.health.mcp(), getAccessToken); + data = await fetchJson(api.health.mcp(), getAccessToken, { + signal: AbortSignal.timeout(MCP_PROBE_TIMEOUT_MS), + }); } catch (err) { if (!(err instanceof ApiError)) throw err; setMcpCheck({ @@ -135,18 +148,40 @@ export default function HealthCheckPage() { } if (data?.check) setMcpCheck(data.check); } catch (error) { - // Network/token errors: surface on the card itself, never as a blocking page error. + // Our own client-side budget expiring means the same thing the server's does: the + // container is still starting. Keep it on the warming track so the re-poll continues. + const clientTimedOut = error instanceof DOMException && error.name === 'TimeoutError'; setMcpCheck({ name: 'MCP Server', description: 'AI query interface availability', - status: 'warning', - message: error instanceof Error ? error.message : 'MCP status check could not complete', + status: clientTimedOut ? 'warming' : 'warning', + message: clientTimedOut + ? 'MCP status check timed out — the server is likely still starting.' + : error instanceof Error ? error.message : 'MCP status check could not complete', }); } finally { setMcpLoading(false); } }, [getAccessToken]); + const mcpCardState = resolveMcpCardState({ check: mcpCheck, loading: mcpLoading, attempts: mcpAttempts }); + const { display: mcpDisplay, ratedStatus: mcpRated, shouldPoll: mcpShouldPoll } = mcpCardState; + + // Re-probe while the MCP container is warming, so the card turns green on its own. + // useLatest keeps the interval from being torn down and rebuilt whenever the callback + // identity changes — that would reset the cadence and desync the attempt counter. + const mcpProbeRef = useLatest(performMcpCheck); + const mcpLoadingRef = useLatest(mcpLoading); + useEffect(() => { + if (!mcpShouldPoll) return; + const id = window.setInterval(() => { + if (mcpLoadingRef.current) return; // never stack probes + setMcpAttempts(a => a + 1); + void mcpProbeRef.current(); + }, MCP_WARMING_POLL_MS); + return () => window.clearInterval(id); + }, [mcpShouldPoll, mcpProbeRef, mcpLoadingRef]); + useEffect(() => { if (!showBuildDetails) return; let cancelled = false; @@ -203,16 +238,17 @@ export default function HealthCheckPage() { case 'healthy': return { bg: 'bg-green-50 dark:bg-green-900/20', border: 'border-green-200 dark:border-green-800', text: 'text-green-700 dark:text-green-400', accent: 'border-green-500', badge: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-400' }; case 'unhealthy': return { bg: 'bg-red-50 dark:bg-red-900/20', border: 'border-red-200 dark:border-red-800', text: 'text-red-700 dark:text-red-400', accent: 'border-red-500', badge: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-400' }; case 'warning': return { bg: 'bg-yellow-50 dark:bg-yellow-900/20', border: 'border-yellow-200 dark:border-yellow-800', text: 'text-yellow-700 dark:text-yellow-400', accent: 'border-yellow-500', badge: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/40 dark:text-yellow-400' }; - case 'checking': return { bg: 'bg-blue-50 dark:bg-blue-900/20', border: 'border-blue-200 dark:border-blue-800', text: 'text-blue-700 dark:text-blue-400', accent: 'border-blue-500', badge: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-400' }; + // "warming" shares the blue in-progress look: a scale-from-zero cold start is + // expected behaviour, and yellow would read as a fault. + case 'checking': + case 'warming': return { bg: 'bg-blue-50 dark:bg-blue-900/20', border: 'border-blue-200 dark:border-blue-800', text: 'text-blue-700 dark:text-blue-400', accent: 'border-blue-500', badge: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-400' }; default: return { bg: 'bg-gray-50 dark:bg-gray-800', border: 'border-gray-200 dark:border-gray-700', text: 'text-gray-700 dark:text-gray-300', accent: 'border-gray-500', badge: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300' }; } }; const connStatus = getConnectionStatus(); - // The MCP card only contributes to the banner/counts once it has resolved to a rated - // status. While it is still probing (null/loading) or not viewable (unknown), it stays - // neutral so a slow cold start never flips the overall banner to "warning". - const mcpRated = mcpCheck && mcpCheck.status !== 'unknown' ? mcpCheck.status : null; + // mcpRated comes from resolveMcpCardState: null while probing, warming or not viewable, + // so a cold start never flips the overall banner. See lib/healthCheckView.ts. const totalChecks = healthResult ? healthResult.checks.length + 1 + (mcpRated ? 1 : 0) : 0; const healthyChecks = healthResult ? healthResult.checks.filter(c => c.status === 'healthy').length @@ -236,7 +272,7 @@ export default function HealthCheckPage() {

System Health

- {/* Individual Check Cards */} - {healthResult && ( -
+ {/* Individual Check Cards. The section is NOT gated on healthResult: the MCP card + runs on its own track (and re-polls itself through a cold start), so a failing + detailed check must not hide it — that would leave the poll running invisibly. */} +

Backend Services

- {visibleHealthChecks(healthResult.checks, operatorView).map((check, index) => { + {(healthResult ? visibleHealthChecks(healthResult.checks, operatorView) : []).map((check, index) => { const colors = getStatusColor(check.status); return (
@@ -493,13 +530,11 @@ export default function HealthCheckPage() { ); })} - {/* MCP Server — fetched on its own track; renders a "checking" state while the - probe (possibly waking the scaled-to-zero container) is in flight, then - updates in place without blocking any of the cards above. */} + {/* MCP Server — fetched on its own track; a scaled-to-zero container reports + "warming" and the card re-probes itself until it is up, without blocking + any of the cards above. State machine: lib/healthCheckView.ts. */} {(() => { - const display = mcpLoading - ? { name: 'MCP Server', description: 'AI query interface availability', status: 'checking', message: 'Probing MCP server — waking it from idle if needed…', details: undefined as Record | undefined } - : (mcpCheck ?? { name: 'MCP Server', description: 'AI query interface availability', status: 'unknown', message: 'Not checked yet', details: undefined as Record | undefined }); + const display = mcpDisplay; const colors = getStatusColor(display.status); return (
@@ -507,7 +542,7 @@ export default function HealthCheckPage() {
- {display.status === 'checking' ? ( + {display.status === 'checking' || display.status === 'warming' ? ( @@ -531,7 +566,7 @@ export default function HealthCheckPage() {
- {display.status === 'checking' ? 'checking…' : display.status} + {display.status === 'checking' ? 'checking…' : display.status === 'warming' ? 'warming…' : display.status}
@@ -546,9 +581,9 @@ export default function HealthCheckPage() { ); })()}
-
- )} +
); } + diff --git a/src/Web/autopilot-monitor-web/lib/__tests__/healthCheckView.test.ts b/src/Web/autopilot-monitor-web/lib/__tests__/healthCheckView.test.ts index 39633817..9a870065 100644 --- a/src/Web/autopilot-monitor-web/lib/__tests__/healthCheckView.test.ts +++ b/src/Web/autopilot-monitor-web/lib/__tests__/healthCheckView.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest"; import type { HealthCheck } from "@/utils/wire-types.generated"; -import { isUrlDetail, visibleHealthChecks, visibleHealthDetails } from "../healthCheckView"; +import { + isUrlDetail, + visibleHealthChecks, + visibleHealthDetails, + resolveMcpCardState, + MCP_WARMING_MAX_ATTEMPTS, +} from "../healthCheckView"; const checks: HealthCheck[] = [ { @@ -90,3 +96,70 @@ describe("visibleHealthDetails", () => { expect(visibleHealthDetails({ "Server URL": "https://mcp.example.com" }, false)).toBeUndefined(); }); }); + +describe("resolveMcpCardState", () => { + const warming = (message = "MCP server did not answer within 3s — it scales to zero when idle."): HealthCheck => ({ + name: "MCP Server", + description: "AI query interface availability", + status: "warming", + message, + }); + + it("shows a checking card and rates nothing while the first probe is in flight", () => { + const s = resolveMcpCardState({ check: null, loading: true, attempts: 0 }); + expect(s.display.status).toBe("checking"); + expect(s.ratedStatus).toBeNull(); + expect(s.shouldPoll).toBe(false); + }); + + it("stays neutral and not-yet-checked before anything ran", () => { + const s = resolveMcpCardState({ check: null, loading: false, attempts: 0 }); + expect(s.display.status).toBe("unknown"); + expect(s.ratedStatus).toBeNull(); + expect(s.shouldPoll).toBe(false); + }); + + it("polls on a warming server and keeps it out of the banner", () => { + const s = resolveMcpCardState({ check: warming(), loading: false, attempts: 0 }); + expect(s.display.status).toBe("warming"); + expect(s.ratedStatus).toBeNull(); + expect(s.shouldPoll).toBe(true); + expect(s.display.message).toContain("attempt 1 of 8"); + }); + + it("keeps showing warming while the next probe runs, so the card cannot flicker", () => { + const s = resolveMcpCardState({ check: warming(), loading: true, attempts: 3 }); + expect(s.display.status).toBe("warming"); + expect(s.display.message).toContain("attempt 4 of 8"); + expect(s.shouldPoll).toBe(true); + }); + + it("turns a never-arriving container into a real warning once the re-polls are spent", () => { + const s = resolveMcpCardState({ check: warming(), loading: false, attempts: MCP_WARMING_MAX_ATTEMPTS }); + expect(s.display.status).toBe("warning"); + expect(s.ratedStatus).toBe("warning"); + expect(s.shouldPoll).toBe(false); + }); + + it("rates a healthy server and stops polling", () => { + const check: HealthCheck = { name: "MCP Server", description: "d", status: "healthy", message: "MCP server reachable (41ms)" }; + const s = resolveMcpCardState({ check, loading: false, attempts: 2 }); + expect(s.display).toBe(check); + expect(s.ratedStatus).toBe("healthy"); + expect(s.shouldPoll).toBe(false); + }); + + it("keeps a not-permitted (unknown) card out of the banner", () => { + const check: HealthCheck = { name: "MCP Server", description: "d", status: "unknown", message: "Not permitted" }; + const s = resolveMcpCardState({ check, loading: false, attempts: 0 }); + expect(s.ratedStatus).toBeNull(); + expect(s.shouldPoll).toBe(false); + }); + + it("rates an unhealthy server into the banner", () => { + const check: HealthCheck = { name: "MCP Server", description: "d", status: "unhealthy", message: "MCP server returned 503" }; + const s = resolveMcpCardState({ check, loading: false, attempts: 0 }); + expect(s.ratedStatus).toBe("unhealthy"); + expect(s.shouldPoll).toBe(false); + }); +}); diff --git a/src/Web/autopilot-monitor-web/lib/healthCheckView.ts b/src/Web/autopilot-monitor-web/lib/healthCheckView.ts index 1e899513..7d033236 100644 --- a/src/Web/autopilot-monitor-web/lib/healthCheckView.ts +++ b/src/Web/autopilot-monitor-web/lib/healthCheckView.ts @@ -28,6 +28,109 @@ export function visibleHealthChecks(checks: readonly HealthCheck[], operatorView .map((c) => ({ ...c, details: visibleHealthDetails(c.details, false) })); } +// ── MCP card: cold start, and how the card recovers from it ──────────────────── + +/** + * The MCP Container App runs minReplicas=0. A cold start is 20-30s of Container Apps + * activation, so the backend probe gives up after ~3s and answers status "warming" + * (see HealthCheckService.CheckMcpServerAsync) rather than holding the request. The card + * then re-polls on its own until the replica is up. + * + * 5s x 8 attempts ≈ 46s of coverage against a measured 13-25s activation (worst observed + * first byte 31.5s). Shorter intervals buy nothing — every attempt costs the backend a 3s + * held request — and more attempts would turn a genuinely broken container into a silent + * forever-spinner instead of an honest warning. + */ +export const MCP_WARMING_POLL_MS = 5_000; +export const MCP_WARMING_MAX_ATTEMPTS = 8; + +/** + * Client-side budget for one probe. Comfortably above the backend's ~3s answer, and + * deliberately NOT tighter: authenticatedFetch reuses the caller's signal for its 401 + * retry, so a nearly-spent budget would abort the retry on arrival. + */ +export const MCP_PROBE_TIMEOUT_MS = 10_000; + +const MCP_CARD_BASE = { + name: "MCP Server", + description: "AI query interface availability", +} as const; + +export interface McpCardState { + /** What the card renders. */ + display: HealthCheck; + /** What the overall banner and the healthy/total counts may use — null = stay neutral. */ + ratedStatus: string | null; + /** Whether the page should keep re-probing on the warming cadence. */ + shouldPoll: boolean; +} + +/** + * Single source of truth for the MCP card: display, banner rating and whether to keep + * polling. Pure so it can be tested without a DOM (the web suite runs on node). + * + * A cold start must never colour the overall banner — scaling from zero is expected + * behaviour, not a degraded platform — so "warming" rates as null exactly the way + * "unknown" already does. Only once the re-polls are exhausted does it become a real + * warning, because by then the container genuinely is not coming up. + */ +export function resolveMcpCardState(args: { + check: HealthCheck | null; + loading: boolean; + attempts: number; +}): McpCardState { + const { check, loading, attempts } = args; + + if (check?.status === "warming") { + const exhausted = attempts >= MCP_WARMING_MAX_ATTEMPTS; + if (exhausted) { + return { + display: { + ...MCP_CARD_BASE, + status: "warning", + message: `MCP server did not come up within ~${Math.round((MCP_WARMING_MAX_ATTEMPTS * MCP_WARMING_POLL_MS) / 1000)}s. Use Re-check, or the container may be failing to start.`, + }, + ratedStatus: "warning", + shouldPoll: false, + }; + } + // Deliberately keeps showing "warming" while the next probe is in flight: switching to + // "checking" every 5s would make the card flicker for the whole cold start. + return { + display: { + ...MCP_CARD_BASE, + status: "warming", + message: `${check.message} Re-checking automatically (attempt ${attempts + 1} of ${MCP_WARMING_MAX_ATTEMPTS})…`, + details: check.details, + }, + ratedStatus: null, + shouldPoll: true, + }; + } + + if (loading) { + return { + display: { ...MCP_CARD_BASE, status: "checking", message: "Probing MCP server…" }, + ratedStatus: null, + shouldPoll: false, + }; + } + + if (!check) { + return { + display: { ...MCP_CARD_BASE, status: "unknown", message: "Not checked yet" }, + ratedStatus: null, + shouldPoll: false, + }; + } + + return { + display: check, + ratedStatus: check.status === "unknown" ? null : check.status, + shouldPoll: false, + }; +} + /** Removes URL-valued rows unless the operator view is on; undefined when nothing is left. */ export function visibleHealthDetails( details: Record | undefined, diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.json b/src/Web/autopilot-monitor-web/utils/shared-manifests.json index abd4baab..d3796351 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.json @@ -10959,7 +10959,8 @@ "type": { "kind": "primitive", "name": "string" - } + }, + "doc": "One of: \"healthy\" (working), \"warning\" (degraded but reachable), \"unhealthy\" (failing), \"unknown\" (not determined), \"warming\" (health/mcp only — the scale-to-zero MCP container did not answer inside the probe budget and is still starting; the caller should re-check, and it must NOT be rated as a fault in any aggregate or overall banner)." }, { "name": "message", diff --git a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts index 8257bf3a..a64e48b3 100644 --- a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts @@ -2336,6 +2336,7 @@ export interface HardwareRejectedResponse { export interface HealthCheck { name: string; description: string; + /** One of: "healthy" (working), "warning" (degraded but reachable), "unhealthy" (failing), "unknown" (not determined), "warming" (health/mcp only — the scale-to-zero MCP container did not answer inside the probe budget and is still starting; the caller should re-check, and it must NOT be rated as a fault in any aggregate or overall banner). */ status: string; message: string; details?: Record;