backend+web: the MCP health probe answers a cold container in 3s with status "warming" instead of holding the request for 30s, and the System Health card re-probes until it is up (D-220)

Measured over 14 days, GET health/mcp answered in 4..148ms when warm and timed out at 30s
in 50 of 83 calls — the median call took 27,972ms. A cold start is 20-30s of Container Apps
activation (~98% KEDA plus image mount, ~500ms application), so no UI wait covers it and the
generous timeout only bought a long wall in front of the same answer. The browser's own
budget is the same 30s, so the backend's carefully worded warning never reached the user at
all: they saw a raw DOMException.

Backend: the budget drops to 3s, which sits in the widest observed latency gap
(2347ms..3930ms) so jitter cannot flip the verdict. Our own cancellation now reports the new
status "warming" at Information level (with minReplicas=0 a cold probe is the expected path,
and logging it as a warning poisons the ops signal); HttpRequestException is split out of
that catch and stays a real warning, still without the host in the message. The warm-hit
threshold drops to 750ms — ~5x the warm maximum, deliberately generous because every
measurement comes from a near-idle single-user system and warm latency will rise under
concurrency — and moves into McpReachableMessage so a test can pin the boundary. The XML doc
now describes the probe that exists rather than the wake-signal design the measurements
disproved.

"warming" is a new status value rather than an added DTO boolean because the backend deploys
before the web: an unknown value degrades to a grey card with a clean banner, whereas a
boolean would leave old web showing yellow and "Warnings detected" on every cold start.
HealthCheck.Status now documents the five-value vocabulary, which rides into both generated
TypeScript mirrors as JSDoc.

Web: resolveMcpCardState owns display, banner rating and the poll decision as a pure
function (the suite runs on node, so testable logic cannot live in JSX). A warming server
re-probes every 5s for up to 8 attempts and stays out of the overall banner exactly as
"unknown" already does — scaling from zero is expected behaviour, not a degraded platform —
and only becomes a real warning once the re-polls are spent. The card keeps showing
"warming" while the next probe is in flight so it cannot flicker, carries the blue
in-progress look instead of yellow, and was hoisted out of the healthResult guard: a failing
detailed check would otherwise hide a card that is still polling.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-09-07 18:25:56 +02:00
parent a1e2b76a9f
commit 066b6e6f04
10 changed files with 384 additions and 65 deletions
@@ -77,17 +77,62 @@ public class HealthCheckServiceMcpServerTests
}
[Fact]
public async Task Check_TimeoutOrConnectionFailure_ReportsWarningForColdStart()
public async Task Check_Timeout_ReportsWarmingSoTheCallerCanRePoll()
{
var svc = BuildServiceWithFactory(
new ThrowingHttpClientFactory(new TaskCanceledException("timeout")));
var check = await svc.CheckMcpServerAsync();
// Failure to wake the scaled-to-zero container within the budget is a warning,
// not a hard outage (a re-check may still succeed once it finishes warming).
// A scaled-to-zero container that did not answer inside the budget is "warming",
// NOT a fault: the caller re-checks and the activation we triggered keeps running.
Assert.Equal("warming", check.Status);
// The default budget is pinned here because it is set nowhere else in the repo —
// the code default IS the live production value.
Assert.Contains("3s", check.Message);
}
[Fact]
public async Task Check_Timeout_HonoursConfiguredBudget()
{
var svc = BuildServiceWithFactory(
new ThrowingHttpClientFactory(new TaskCanceledException("timeout")),
extraConfig: new Dictionary<string, string?> { ["McpServerHealthTimeoutSeconds"] = "7" });
var check = await svc.CheckMcpServerAsync();
Assert.Equal("warming", check.Status);
Assert.Contains("7s", check.Message);
}
[Fact]
public async Task Check_ConnectionFailure_ReportsWarningNotWarmingAndKeepsTheHostOut()
{
var svc = BuildServiceWithFactory(
new ThrowingHttpClientFactory(new System.Net.Http.HttpRequestException("no such host mcp.example.test")));
var check = await svc.CheckMcpServerAsync();
// Reaching the network and failing to connect is a real warning — distinct from
// the expected cold-start path, so it must not be softened to "warming".
Assert.Equal("warning", check.Status);
Assert.Contains("could not be reached", check.Message);
Assert.DoesNotContain("http", check.Message, System.StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain("mcp.example.test", check.Message);
}
[Theory]
[InlineData(12, false)]
[InlineData(750, false)]
[InlineData(751, true)]
[InlineData(2400, true)]
public void McpReachableMessage_marksOnlyAnswersAboveTheWarmThresholdAsColdStart(long elapsedMs, bool expectColdStart)
{
var message = HealthCheckService.McpReachableMessage(elapsedMs);
// Both arms keep the word "reachable" — the healthy-path assertions rely on it.
Assert.Contains("reachable", message);
Assert.Contains($"{elapsedMs}ms", message);
Assert.Equal(expectColdStart, message.Contains("cold start"));
}
/// <summary>
@@ -101,13 +146,21 @@ public class HealthCheckServiceMcpServerTests
return BuildServiceWithFactory(new StubHttpClientFactory(handler));
}
private static HealthCheckService BuildServiceWithFactory(System.Net.Http.IHttpClientFactory factory)
private static HealthCheckService BuildServiceWithFactory(
System.Net.Http.IHttpClientFactory factory,
Dictionary<string, string?>? extraConfig = null)
{
var settings = new Dictionary<string, string?>
{
["McpServerUrl"] = "https://mcp.example.test",
};
if (extraConfig != null)
{
foreach (var kv in extraConfig) settings[kv.Key] = kv.Value;
}
var config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["McpServerUrl"] = "https://mcp.example.test",
})
.AddInMemoryCollection(settings)
.Build();
return new HealthCheckService(
@@ -49,10 +49,12 @@ public class HealthCheckService
};
// NOTE: CheckMcpServerAsync is deliberately NOT part of this batch. The MCP
// Container App scales to zero on idle, so a probe may need to wake it and can
// take many seconds. Bundling it here would block the entire dashboard on that
// one slow check. It is exposed via its own endpoint (GET health/mcp) that the
// frontend fetches independently and folds into the card grid incrementally.
// Container App scales to zero on idle; a cold probe answers "warming" after a
// short budget and the caller re-polls until the replica is up. That re-poll loop
// must stay off the blocking batch, and "warming" must never reach this aggregate
// — a scale-from-zero is expected behaviour, not a degraded platform. It is
// exposed via its own endpoint (GET health/mcp) that the frontend fetches
// independently and folds into the card grid incrementally.
var checks = await Task.WhenAll(
CheckStorageBackendAsync(includeEndpointUrls),
CheckProcessingBackendAsync(includeEndpointUrls),
@@ -244,13 +246,32 @@ public class HealthCheckService
/// only included in the details when <paramref name="includeEndpointUrl"/> is set —
/// callers must only enable it for Global Admins.
/// <para>
/// The Container App runs with minReplicas=0 and scales to zero on idle. Azure Container
/// Apps holds an inbound request while it activates a cold replica, so this probe doubles
/// as the wake signal — a generous timeout (<c>McpServerHealthTimeoutSeconds</c>, default
/// 30s) gives the cold start time to complete and report healthy. Only when the server
/// cannot be woken within that budget is it a warning; a reachable-but-erroring server
/// (non-2xx) is unhealthy. This is exposed via its own <c>health/mcp</c> endpoint, never
/// the blocking all-checks batch, so the dashboard never waits on the wake.
/// The Container App runs with minReplicas=0 and scales to zero on idle, and a cold start
/// is dominated by Container Apps activation, not by us: measured at 24.5s and 31.5s to
/// first byte, of which ~98% is KEDA scheduling plus the image mount and ~500ms is the
/// application (see <c>internal/docs/mcp/docs-corpus.md</c>, "Cold start"). No UI wait
/// covers that, so this probe deliberately does not try to sit it out — an earlier design
/// waited 30s as a "wake signal" and still timed out in 50 of 83 observed calls, i.e. it
/// bought a long wall in front of the same answer.
/// </para>
/// <para>
/// The probe therefore has two jobs: answer fast and honestly within
/// <c>McpServerHealthTimeoutSeconds</c> (default 3s — chosen to sit in the widest observed
/// latency gap, 2347ms..3930ms, so jitter cannot flip the verdict), and leave the wake
/// running. The inbound request has already reached the Container Apps activator by the
/// time we give up, so aborting our side does not undo the scale-from-zero it triggered;
/// the caller re-checks and a later probe finds the replica warm.
/// </para>
/// <para>
/// Status vocabulary: <c>healthy</c> = answered 2xx; <c>warming</c> = did not answer inside
/// the budget, a cold start is in progress and the caller should re-check (must NOT be
/// rated into any aggregate — it is expected, not a fault); <c>warning</c> = reachable but
/// the connection failed; <c>unhealthy</c> = answered non-2xx. Exposed via its own
/// <c>health/mcp</c> endpoint, never the blocking all-checks batch.
/// </para>
/// <para>
/// <c>McpServerHealthTimeoutSeconds</c> is set nowhere in the repo, infra or workflows, so
/// the default below IS the live production value — changing it changes production.
/// </para>
/// </summary>
internal async Task<HealthCheck> CheckMcpServerAsync(bool includeEndpointUrl = false)
@@ -268,11 +289,9 @@ public class HealthCheckService
{
check.Details = new Dictionary<string, object> { ["Server URL"] = baseUrl };
}
var timeoutSeconds = ParsePositiveInt("McpServerHealthTimeoutSeconds", 30);
// A cold start that takes longer than this is treated as a successful wake rather
// than an instant-warm hit, so the message can say so.
const long WarmHitThresholdMs = 2500;
// 3s: warm answers were measured at 4..148ms over 14 days, so this is ~20x the warm
// maximum, while a cold start needs 20-30s and is unreachable for any UI wait.
var timeoutSeconds = ParsePositiveInt("McpServerHealthTimeoutSeconds", 3);
try
{
@@ -293,24 +312,32 @@ public class HealthCheckService
var version = await TryReadMcpVersionAsync(response);
check.Status = "healthy";
check.Message = sw.ElapsedMilliseconds > WarmHitThresholdMs
? $"MCP server woke from idle and is reachable ({sw.ElapsedMilliseconds}ms)"
: $"MCP server reachable ({sw.ElapsedMilliseconds}ms)";
check.Message = McpReachableMessage(sw.ElapsedMilliseconds);
if (!string.IsNullOrWhiteSpace(version))
{
check.Details ??= new Dictionary<string, object>();
check.Details["Version"] = version;
}
}
catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException or HttpRequestException)
catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
{
// Timeout or connection failure after the full wake budget. The probe held the
// request long enough for a cold replica to activate, so a failure here means the
// server could not be woken — surface as a warning (not a hard outage: the next
// re-check may still succeed once it finishes warming).
// Our own timeout: no external cancellation token reaches this path, so a
// cancellation here is always the budget above expiring. The scale-from-zero we
// triggered keeps running after we give up, so this is a re-check prompt, not a
// fault — and with minReplicas=0 it is the expected path for an idle server,
// which is why it logs at Information and must not colour any aggregate.
check.Status = "warming";
check.Message = $"MCP server did not answer within {timeoutSeconds}s — it scales to zero when idle and is likely still starting (a cold start takes ~20-30s).";
_logger.LogInformation(ex, "MCP server health probe found a cold server (no answer within {TimeoutSeconds}s); activation continues in the background", timeoutSeconds);
}
catch (HttpRequestException ex)
{
// Reached the network but could not connect — distinct from the cold-start path
// above and genuinely worth a warning. ex.Message can carry the host name, so it
// stays out of the user-visible text (same rule as the non-2xx branch).
check.Status = "warning";
check.Message = $"MCP server could not be reached within {timeoutSeconds}s — it may still be warming up. Re-check in a moment.";
_logger.LogWarning(ex, "MCP server health probe did not respond within {TimeoutSeconds}s (wake attempt failed)", timeoutSeconds);
check.Message = "MCP server unreachable (connection failed)";
_logger.LogWarning(ex, "MCP server health probe could not connect");
}
catch (Exception ex)
{
@@ -322,6 +349,24 @@ public class HealthCheckService
return check;
}
/// <summary>
/// An answer slower than this came from a replica that had just started, not from a warm
/// one, so the message can say so. 750ms is ~5x the warm maximum measured over 14 days
/// (4..148ms) and deliberately generous: those measurements come from a near-idle,
/// single-user system, and warm latency will rise under concurrency. It stays well below
/// the next observed cluster (2289ms), so the distinction remains sharp.
/// </summary>
private const long WarmHitThresholdMs = 750;
/// <summary>
/// Wording for a reachable MCP server. Extracted so the threshold can be pinned by a test
/// without a real delay. Both arms keep the word "reachable".
/// </summary>
internal static string McpReachableMessage(long elapsedMs) =>
elapsedMs > WarmHitThresholdMs
? $"MCP server reachable after a cold start ({elapsedMs}ms)"
: $"MCP server reachable ({elapsedMs}ms)";
/// <summary>
/// Best-effort parse of the MCP <c>/health</c> JSON body for its <c>version</c> field.
/// A missing or malformed body is non-fatal — the probe already proved reachability.
@@ -2336,6 +2336,7 @@ export interface HardwareRejectedResponse {
export interface HealthCheck {
name: string;
description: string;
/** One of: "healthy" (working), "warning" (degraded but reachable), "unhealthy" (failing), "unknown" (not determined), "warming" (health/mcp only — the scale-to-zero MCP container did not answer inside the probe budget and is still starting; the caller should re-check, and it must NOT be rated as a fault in any aggregate or overall banner). */
status: string;
message: string;
details?: Record<string, unknown>;
@@ -137,6 +137,13 @@ namespace AutopilotMonitor.Shared.Models
{
public string Name { get; set; } = string.Empty;
public string Description { get; set; } = string.Empty;
/// <summary>
/// One of: "healthy" (working), "warning" (degraded but reachable), "unhealthy"
/// (failing), "unknown" (not determined), "warming" (health/mcp only — the
/// scale-to-zero MCP container did not answer inside the probe budget and is still
/// starting; the caller should re-check, and it must NOT be rated as a fault in any
/// aggregate or overall banner).
/// </summary>
public string Status { get; set; } = "unknown";
public string Message { get; set; } = string.Empty;
public Dictionary<string, object>? Details { get; set; }
@@ -8,7 +8,15 @@ import { useGlobalAdminUi } from '@/hooks/useGlobalAdminUi';
import { useState, useEffect, useCallback } from 'react';
import * as signalR from '@microsoft/signalr';
import { api } from '@/lib/api';
import { isUrlDetail, visibleHealthChecks, visibleHealthDetails } from '@/lib/healthCheckView';
import {
isUrlDetail,
visibleHealthChecks,
visibleHealthDetails,
resolveMcpCardState,
MCP_WARMING_POLL_MS,
MCP_PROBE_TIMEOUT_MS,
} from '@/lib/healthCheckView';
import { useLatest } from '@/hooks/useLatest';
import type { DetailedHealthCheckResponse, HealthCheck, McpHealthCheckResponse } from '@/utils/wire-types.generated';
import { DocsLink } from "@/components/DocsLink";
import { DOCS_PATHS } from "@/lib/docsPaths";
@@ -81,6 +89,9 @@ export default function HealthCheckPage() {
// dashboard on it. The card renders a "checking" state immediately and updates in place.
const [mcpCheck, setMcpCheck] = useState<HealthCheck | null>(null);
const [mcpLoading, setMcpLoading] = useState(false);
// Counts only the automatic warming re-probes; reset solely by the Re-check button, so
// a reset inside performMcpCheck can never make the poll loop immortal.
const [mcpAttempts, setMcpAttempts] = useState(0);
// Portal build stamp: /version.json is a static file from this same origin (generated at
// build time, Cache-Control: no-store) — plain fetch, no auth. Card stays hidden when the
// file is absent (e.g. a dev server without the prebuild step).
@@ -113,14 +124,16 @@ export default function HealthCheckPage() {
// Separate, non-blocking probe for the MCP server. Runs concurrently with — and
// independently of — performHealthCheck so a cold-starting MCP container never holds
// up the other cards. Resolves to healthy / warning / unhealthy; the card shows a
// "checking" state until then.
// up the other cards. The server answers "warming" for a scaled-to-zero container
// instead of holding the request; the effect below then re-probes until it is up.
const performMcpCheck = useCallback(async () => {
setMcpLoading(true);
try {
let data: McpHealthCheckResponse;
try {
data = await fetchJson<McpHealthCheckResponse>(api.health.mcp(), getAccessToken);
data = await fetchJson<McpHealthCheckResponse>(api.health.mcp(), getAccessToken, {
signal: AbortSignal.timeout(MCP_PROBE_TIMEOUT_MS),
});
} catch (err) {
if (!(err instanceof ApiError)) throw err;
setMcpCheck({
@@ -135,18 +148,40 @@ export default function HealthCheckPage() {
}
if (data?.check) setMcpCheck(data.check);
} catch (error) {
// Network/token errors: surface on the card itself, never as a blocking page error.
// Our own client-side budget expiring means the same thing the server's does: the
// container is still starting. Keep it on the warming track so the re-poll continues.
const clientTimedOut = error instanceof DOMException && error.name === 'TimeoutError';
setMcpCheck({
name: 'MCP Server',
description: 'AI query interface availability',
status: 'warning',
message: error instanceof Error ? error.message : 'MCP status check could not complete',
status: clientTimedOut ? 'warming' : 'warning',
message: clientTimedOut
? 'MCP status check timed out — the server is likely still starting.'
: error instanceof Error ? error.message : 'MCP status check could not complete',
});
} finally {
setMcpLoading(false);
}
}, [getAccessToken]);
const mcpCardState = resolveMcpCardState({ check: mcpCheck, loading: mcpLoading, attempts: mcpAttempts });
const { display: mcpDisplay, ratedStatus: mcpRated, shouldPoll: mcpShouldPoll } = mcpCardState;
// Re-probe while the MCP container is warming, so the card turns green on its own.
// useLatest keeps the interval from being torn down and rebuilt whenever the callback
// identity changes — that would reset the cadence and desync the attempt counter.
const mcpProbeRef = useLatest(performMcpCheck);
const mcpLoadingRef = useLatest(mcpLoading);
useEffect(() => {
if (!mcpShouldPoll) return;
const id = window.setInterval(() => {
if (mcpLoadingRef.current) return; // never stack probes
setMcpAttempts(a => a + 1);
void mcpProbeRef.current();
}, MCP_WARMING_POLL_MS);
return () => window.clearInterval(id);
}, [mcpShouldPoll, mcpProbeRef, mcpLoadingRef]);
useEffect(() => {
if (!showBuildDetails) return;
let cancelled = false;
@@ -203,16 +238,17 @@ export default function HealthCheckPage() {
case 'healthy': return { bg: 'bg-green-50 dark:bg-green-900/20', border: 'border-green-200 dark:border-green-800', text: 'text-green-700 dark:text-green-400', accent: 'border-green-500', badge: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-400' };
case 'unhealthy': return { bg: 'bg-red-50 dark:bg-red-900/20', border: 'border-red-200 dark:border-red-800', text: 'text-red-700 dark:text-red-400', accent: 'border-red-500', badge: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-400' };
case 'warning': return { bg: 'bg-yellow-50 dark:bg-yellow-900/20', border: 'border-yellow-200 dark:border-yellow-800', text: 'text-yellow-700 dark:text-yellow-400', accent: 'border-yellow-500', badge: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/40 dark:text-yellow-400' };
case 'checking': return { bg: 'bg-blue-50 dark:bg-blue-900/20', border: 'border-blue-200 dark:border-blue-800', text: 'text-blue-700 dark:text-blue-400', accent: 'border-blue-500', badge: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-400' };
// "warming" shares the blue in-progress look: a scale-from-zero cold start is
// expected behaviour, and yellow would read as a fault.
case 'checking':
case 'warming': return { bg: 'bg-blue-50 dark:bg-blue-900/20', border: 'border-blue-200 dark:border-blue-800', text: 'text-blue-700 dark:text-blue-400', accent: 'border-blue-500', badge: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-400' };
default: return { bg: 'bg-gray-50 dark:bg-gray-800', border: 'border-gray-200 dark:border-gray-700', text: 'text-gray-700 dark:text-gray-300', accent: 'border-gray-500', badge: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300' };
}
};
const connStatus = getConnectionStatus();
// The MCP card only contributes to the banner/counts once it has resolved to a rated
// status. While it is still probing (null/loading) or not viewable (unknown), it stays
// neutral so a slow cold start never flips the overall banner to "warning".
const mcpRated = mcpCheck && mcpCheck.status !== 'unknown' ? mcpCheck.status : null;
// mcpRated comes from resolveMcpCardState: null while probing, warming or not viewable,
// so a cold start never flips the overall banner. See lib/healthCheckView.ts.
const totalChecks = healthResult ? healthResult.checks.length + 1 + (mcpRated ? 1 : 0) : 0;
const healthyChecks = healthResult
? healthResult.checks.filter(c => c.status === 'healthy').length
@@ -236,7 +272,7 @@ export default function HealthCheckPage() {
<h1 className="text-2xl font-normal text-gray-900 dark:text-white">System Health</h1>
<div className="flex flex-wrap items-center gap-2">
<button
onClick={() => { performHealthCheck(); performMcpCheck(); }}
onClick={() => { setMcpAttempts(0); performHealthCheck(); performMcpCheck(); }}
disabled={loading}
className="px-4 py-2 bg-green-600 text-white rounded-lg hover:bg-green-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors text-sm font-medium flex items-center gap-2"
>
@@ -446,12 +482,13 @@ export default function HealthCheckPage() {
})()}
</div>
{/* Individual Check Cards */}
{healthResult && (
<div>
{/* Individual Check Cards. The section is NOT gated on healthResult: the MCP card
runs on its own track (and re-polls itself through a cold start), so a failing
detailed check must not hide it — that would leave the poll running invisibly. */}
<div>
<h2 className="text-sm font-semibold text-gray-500 dark:text-gray-400 uppercase tracking-wider mb-3">Backend Services</h2>
<div className="grid gap-5 sm:grid-cols-2">
{visibleHealthChecks(healthResult.checks, operatorView).map((check, index) => {
{(healthResult ? visibleHealthChecks(healthResult.checks, operatorView) : []).map((check, index) => {
const colors = getStatusColor(check.status);
return (
<div key={index} className={`bg-white dark:bg-gray-800 rounded-lg shadow border-l-4 ${colors.accent}`}>
@@ -493,13 +530,11 @@ export default function HealthCheckPage() {
);
})}
{/* MCP Server — fetched on its own track; renders a "checking" state while the
probe (possibly waking the scaled-to-zero container) is in flight, then
updates in place without blocking any of the cards above. */}
{/* MCP Server — fetched on its own track; a scaled-to-zero container reports
"warming" and the card re-probes itself until it is up, without blocking
any of the cards above. State machine: lib/healthCheckView.ts. */}
{(() => {
const display = mcpLoading
? { name: 'MCP Server', description: 'AI query interface availability', status: 'checking', message: 'Probing MCP server — waking it from idle if needed…', details: undefined as Record<string, unknown> | undefined }
: (mcpCheck ?? { name: 'MCP Server', description: 'AI query interface availability', status: 'unknown', message: 'Not checked yet', details: undefined as Record<string, unknown> | undefined });
const display = mcpDisplay;
const colors = getStatusColor(display.status);
return (
<div className={`bg-white dark:bg-gray-800 rounded-lg shadow border-l-4 ${colors.accent}`}>
@@ -507,7 +542,7 @@ export default function HealthCheckPage() {
<div className="flex items-start justify-between mb-3">
<div className="flex items-center space-x-3">
<div className={`w-8 h-8 rounded-full ${colors.bg} flex items-center justify-center`}>
{display.status === 'checking' ? (
{display.status === 'checking' || display.status === 'warming' ? (
<svg className="w-5 h-5 text-blue-600 animate-spin" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={2}>
<path strokeLinecap="round" strokeLinejoin="round" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
@@ -531,7 +566,7 @@ export default function HealthCheckPage() {
</div>
</div>
<span className={`inline-flex items-center px-2.5 py-0.5 rounded-full text-xs font-medium ${colors.badge}`}>
{display.status === 'checking' ? 'checking…' : display.status}
{display.status === 'checking' ? 'checking…' : display.status === 'warming' ? 'warming…' : display.status}
</span>
</div>
@@ -546,9 +581,9 @@ export default function HealthCheckPage() {
);
})()}
</div>
</div>
)}
</div>
</div>
</div>
);
}
@@ -1,6 +1,12 @@
import { describe, expect, it } from "vitest";
import type { HealthCheck } from "@/utils/wire-types.generated";
import { isUrlDetail, visibleHealthChecks, visibleHealthDetails } from "../healthCheckView";
import {
isUrlDetail,
visibleHealthChecks,
visibleHealthDetails,
resolveMcpCardState,
MCP_WARMING_MAX_ATTEMPTS,
} from "../healthCheckView";
const checks: HealthCheck[] = [
{
@@ -90,3 +96,70 @@ describe("visibleHealthDetails", () => {
expect(visibleHealthDetails({ "Server URL": "https://mcp.example.com" }, false)).toBeUndefined();
});
});
describe("resolveMcpCardState", () => {
const warming = (message = "MCP server did not answer within 3s — it scales to zero when idle."): HealthCheck => ({
name: "MCP Server",
description: "AI query interface availability",
status: "warming",
message,
});
it("shows a checking card and rates nothing while the first probe is in flight", () => {
const s = resolveMcpCardState({ check: null, loading: true, attempts: 0 });
expect(s.display.status).toBe("checking");
expect(s.ratedStatus).toBeNull();
expect(s.shouldPoll).toBe(false);
});
it("stays neutral and not-yet-checked before anything ran", () => {
const s = resolveMcpCardState({ check: null, loading: false, attempts: 0 });
expect(s.display.status).toBe("unknown");
expect(s.ratedStatus).toBeNull();
expect(s.shouldPoll).toBe(false);
});
it("polls on a warming server and keeps it out of the banner", () => {
const s = resolveMcpCardState({ check: warming(), loading: false, attempts: 0 });
expect(s.display.status).toBe("warming");
expect(s.ratedStatus).toBeNull();
expect(s.shouldPoll).toBe(true);
expect(s.display.message).toContain("attempt 1 of 8");
});
it("keeps showing warming while the next probe runs, so the card cannot flicker", () => {
const s = resolveMcpCardState({ check: warming(), loading: true, attempts: 3 });
expect(s.display.status).toBe("warming");
expect(s.display.message).toContain("attempt 4 of 8");
expect(s.shouldPoll).toBe(true);
});
it("turns a never-arriving container into a real warning once the re-polls are spent", () => {
const s = resolveMcpCardState({ check: warming(), loading: false, attempts: MCP_WARMING_MAX_ATTEMPTS });
expect(s.display.status).toBe("warning");
expect(s.ratedStatus).toBe("warning");
expect(s.shouldPoll).toBe(false);
});
it("rates a healthy server and stops polling", () => {
const check: HealthCheck = { name: "MCP Server", description: "d", status: "healthy", message: "MCP server reachable (41ms)" };
const s = resolveMcpCardState({ check, loading: false, attempts: 2 });
expect(s.display).toBe(check);
expect(s.ratedStatus).toBe("healthy");
expect(s.shouldPoll).toBe(false);
});
it("keeps a not-permitted (unknown) card out of the banner", () => {
const check: HealthCheck = { name: "MCP Server", description: "d", status: "unknown", message: "Not permitted" };
const s = resolveMcpCardState({ check, loading: false, attempts: 0 });
expect(s.ratedStatus).toBeNull();
expect(s.shouldPoll).toBe(false);
});
it("rates an unhealthy server into the banner", () => {
const check: HealthCheck = { name: "MCP Server", description: "d", status: "unhealthy", message: "MCP server returned 503" };
const s = resolveMcpCardState({ check, loading: false, attempts: 0 });
expect(s.ratedStatus).toBe("unhealthy");
expect(s.shouldPoll).toBe(false);
});
});
@@ -28,6 +28,109 @@ export function visibleHealthChecks(checks: readonly HealthCheck[], operatorView
.map((c) => ({ ...c, details: visibleHealthDetails(c.details, false) }));
}
// ── MCP card: cold start, and how the card recovers from it ────────────────────
/**
* The MCP Container App runs minReplicas=0. A cold start is 20-30s of Container Apps
* activation, so the backend probe gives up after ~3s and answers status "warming"
* (see HealthCheckService.CheckMcpServerAsync) rather than holding the request. The card
* then re-polls on its own until the replica is up.
*
* 5s x 8 attempts ≈ 46s of coverage against a measured 13-25s activation (worst observed
* first byte 31.5s). Shorter intervals buy nothing — every attempt costs the backend a 3s
* held request — and more attempts would turn a genuinely broken container into a silent
* forever-spinner instead of an honest warning.
*/
export const MCP_WARMING_POLL_MS = 5_000;
export const MCP_WARMING_MAX_ATTEMPTS = 8;
/**
* Client-side budget for one probe. Comfortably above the backend's ~3s answer, and
* deliberately NOT tighter: authenticatedFetch reuses the caller's signal for its 401
* retry, so a nearly-spent budget would abort the retry on arrival.
*/
export const MCP_PROBE_TIMEOUT_MS = 10_000;
const MCP_CARD_BASE = {
name: "MCP Server",
description: "AI query interface availability",
} as const;
export interface McpCardState {
/** What the card renders. */
display: HealthCheck;
/** What the overall banner and the healthy/total counts may use — null = stay neutral. */
ratedStatus: string | null;
/** Whether the page should keep re-probing on the warming cadence. */
shouldPoll: boolean;
}
/**
* Single source of truth for the MCP card: display, banner rating and whether to keep
* polling. Pure so it can be tested without a DOM (the web suite runs on node).
*
* A cold start must never colour the overall banner — scaling from zero is expected
* behaviour, not a degraded platform — so "warming" rates as null exactly the way
* "unknown" already does. Only once the re-polls are exhausted does it become a real
* warning, because by then the container genuinely is not coming up.
*/
export function resolveMcpCardState(args: {
check: HealthCheck | null;
loading: boolean;
attempts: number;
}): McpCardState {
const { check, loading, attempts } = args;
if (check?.status === "warming") {
const exhausted = attempts >= MCP_WARMING_MAX_ATTEMPTS;
if (exhausted) {
return {
display: {
...MCP_CARD_BASE,
status: "warning",
message: `MCP server did not come up within ~${Math.round((MCP_WARMING_MAX_ATTEMPTS * MCP_WARMING_POLL_MS) / 1000)}s. Use Re-check, or the container may be failing to start.`,
},
ratedStatus: "warning",
shouldPoll: false,
};
}
// Deliberately keeps showing "warming" while the next probe is in flight: switching to
// "checking" every 5s would make the card flicker for the whole cold start.
return {
display: {
...MCP_CARD_BASE,
status: "warming",
message: `${check.message} Re-checking automatically (attempt ${attempts + 1} of ${MCP_WARMING_MAX_ATTEMPTS})…`,
details: check.details,
},
ratedStatus: null,
shouldPoll: true,
};
}
if (loading) {
return {
display: { ...MCP_CARD_BASE, status: "checking", message: "Probing MCP server…" },
ratedStatus: null,
shouldPoll: false,
};
}
if (!check) {
return {
display: { ...MCP_CARD_BASE, status: "unknown", message: "Not checked yet" },
ratedStatus: null,
shouldPoll: false,
};
}
return {
display: check,
ratedStatus: check.status === "unknown" ? null : check.status,
shouldPoll: false,
};
}
/** Removes URL-valued rows unless the operator view is on; undefined when nothing is left. */
export function visibleHealthDetails(
details: Record<string, unknown> | undefined,
@@ -10959,7 +10959,8 @@
"type": {
"kind": "primitive",
"name": "string"
}
},
"doc": "One of: \"healthy\" (working), \"warning\" (degraded but reachable), \"unhealthy\" (failing), \"unknown\" (not determined), \"warming\" (health/mcp only — the scale-to-zero MCP container did not answer inside the probe budget and is still starting; the caller should re-check, and it must NOT be rated as a fault in any aggregate or overall banner)."
},
{
"name": "message",
@@ -2336,6 +2336,7 @@ export interface HardwareRejectedResponse {
export interface HealthCheck {
name: string;
description: string;
/** One of: "healthy" (working), "warning" (degraded but reachable), "unhealthy" (failing), "unknown" (not determined), "warming" (health/mcp only — the scale-to-zero MCP container did not answer inside the probe budget and is still starting; the caller should re-check, and it must NOT be rated as a fault in any aggregate or overall banner). */
status: string;
message: string;
details?: Record<string, unknown>;