From 7680bdff7b58c2704893518a5969cdfe4a615ef7 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Mon, 27 Jul 2026 18:27:24 +0200 Subject: [PATCH] fix(web): disable RSC prefetch on nav links - portal freeze root cause Navbar + GlobalSidebar render the full navigation on every page, so the App Router's default viewport prefetch fired one ?_rsc= request per visible link against the SWA runtime (~10.4k requests / 14 days). That origin serves document, chunks and RSC payloads over HTTP/1.1 (6 connections per origin); when the runtime queued the prefetches (p95 896ms, tail up to 234s), they held every socket and the portal froze until the queue drained. Measured on the 2026-07-27 17:25 incident: /admin/presence?_rsc= at 151s while the backend API answered in <2.2s. All 16 nav links now set prefetch={false}. Every target route is a client component behind ProtectedRoute that fetches its own data after mount, so the prefetched RSC payload carried no benefit. New OKF doc docs/web/portal-navigation-prefetch.md records the full evidence chain and where browser telemetry lives (separate App Insights resource, not reachable via MCP query_backend_logs). Co-Authored-By: Claude Opus 5 (1M context) --- docs/index.md | 4 + docs/log.md | 2 + docs/web/portal-navigation-prefetch.md | 88 +++++++++++++++++++ .../components/GlobalSidebar.tsx | 20 ++++- .../components/Navbar.tsx | 12 ++- .../utils/page-lastmod.generated.ts | 8 +- 6 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 docs/web/portal-navigation-prefetch.md diff --git a/docs/index.md b/docs/index.md index 297ba271..4ec6f67d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,6 +41,10 @@ https://docs.autopilotmonitor.com (separate repository). * [MCP Docs Corpus](mcp/docs-corpus.md) - How the published customer documentation becomes the `search_docs` corpus: CRLF-safe heading chunking, build-time embedding as a third precomputed section, and why a docs edit needs an MCP redeploy. +# Web Portal + +* [Portal Navigation Prefetch & the SWA Runtime Bottleneck](web/portal-navigation-prefetch.md) - Why every navigation Link sets `prefetch={false}`: the App Router's viewport prefetch fans out one RSC request per visible link against an HTTP/1.1 Static Web Apps runtime, and its latency tail starves the six connections that also serve the document and the route chunks. Includes where browser telemetry actually lives. + # Security & Identity * [MCP OAuth Flow](mcp-oauth-flow.md) - Who authenticates where when connecting an AI client to the MCP server; two identities, three parties plus Entra ID. diff --git a/docs/log.md b/docs/log.md index 8bd3d3a1..6840732d 100644 --- a/docs/log.md +++ b/docs/log.md @@ -2,6 +2,8 @@ ## 2026-07-27 +* **Creation**: Added `web/portal-navigation-prefetch.md` (and a `Web Portal` section to the index) — root cause of the minute-long portal freezes. `GlobalSidebar` and `Navbar` render the whole navigation on every page, so the App Router's default viewport prefetch issued one `?_rsc=` request per visible link against `portal.autopilotmonitor.com` (~10,400 in 14 days). That origin serves the document, the route chunks and the RSC payloads over **HTTP/1.1**, so the six-connection-per-origin cap applies; when the Static Web Apps managed runtime queued the prefetches (p50 91 ms, p95 896 ms, but max 234 s / 21 samples over 30 s), the in-flight prefetches held every socket and the navigation the user wanted could not get one. Measured on the 2026-07-27 17:25 CEST incident: `/admin/presence?_rsc=` 151 s, three siblings finishing within 340 ms of each other at ~37.5 s, all `200`, while the backend API answered in 1.6–2.2 s and `browserTimings` showed `networkDuration` 16 ms against `processingDuration` 225 s. Every navigation Link now sets `prefetch={false}` — these routes are client components behind `ProtectedRoute` that fetch their own data, so the prefetched RSC payload carried no benefit. The doc also records that browser telemetry lives in a **separate** App Insights resource (`AUTOPILOT_MONITOR_WEB_APPINSIGHTS_ID`) that the MCP `query_backend_logs` tool cannot reach, which is why the freezes left no trace in the backend logs. + * **Creation**: Added `backend/rule-regression-radar.md` — F3 regression radar (analyze rules only; gather rules wait for their per-batch stats dedup fix): per (tenant, rule) daily hit rate from `RuleStats`, firing only when the trailing 7-day window has ≥5 hit sessions and ≥20 evaluated sessions, the rate is ≥2× the prior 28-day baseline AND the Wilson 95 % intervals are disjoint (deterministic `MetricsMath` primitive with hand-computed test vectors). Suppressed for deleted rules, rules younger than baseline+window (`CreatedAt` grace), rules edited inside the window (`UpdatedAt`), empty baselines, and the `RuleRegressionRadarDisabled` app-setting kill switch. Alert episodes live as `ruleregression|{ruleId}` tracker rows (one bell + one `RuleFrequencyRegression` ops event per episode; numbers refreshed while active; re-armed under 1.5× baseline, on fire-stop, or by the 30-day tracker retention). On-fire dimension correlation over osBuild/model/agentVersion/imeVersion with the "correlated, not necessarily causal" wording contract. Surfaces: `regressions[]` in the rule-stats response (MCP inherits), the analyze-rules page's 30-day sparkline and regression badge. * **Creation**: Added `backend/device-journeys.md` — F2 device history / First-Time-Right: the device key (trim + case-fold serial, junk/placeholder exclusion incl. the agent's `Unknown` WMI sentinel), journey grouping over the terminal set Succeeded/Failed/Incomplete (first success ends a journey, a >30-day gap from the previous attempt's CompletedAt starts one, WhiteGlove's single session row is ONE attempt, open sessions are never attempts), the `DeviceHistories` chain (cap 20, written inline at the terminal seam, healed by the sweep, deleted-session refs pruned tombstone-driven, deliberately not in the per-session deletion manifest) and the additive `DeviceJourneyAggregates` daily rows (window rate = ratio of sums — no rolling row needed, 180-day retention). Surfaces: the session-detail "Attempt N" banner with server-computed attempt numbers (virtual-attempt rule for live sessions), the Fleet Health First-time-right section (≥20-journey gate, attempt histogram, repeat-devices violator list), and the MCP `get_device_history` tool (serial mode / fleet FTR mode). diff --git a/docs/web/portal-navigation-prefetch.md b/docs/web/portal-navigation-prefetch.md new file mode 100644 index 00000000..c01fee6a --- /dev/null +++ b/docs/web/portal-navigation-prefetch.md @@ -0,0 +1,88 @@ +--- +type: Concept +title: Portal Navigation Prefetch & the SWA Runtime Bottleneck +description: Why every navigation Link in the portal sets prefetch={false} — the App Router's viewport prefetch fires one RSC request per visible link against an HTTP/1.1 Static Web Apps runtime, and its latency tail starves the connection pool that serves the document and the route chunks. +resource: src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx +tags: + - web + - performance + - nextjs + - static-web-apps + - observability +timestamp: 2026-07-27T18:00:00+02:00 +--- + +# Context + +The portal is a Next.js 15 App Router application hosted on Azure Static Web Apps. Every page under +the authenticated shell is a client component behind `ProtectedRoute` that fetches its own data from +the Function App (`autopilotmonitor-api-eu`) after mount. The RSC payload a route returns on +navigation therefore carries no page data — only the shell. + +`GlobalSidebar` and `Navbar` render the full navigation on every page. With the App Router's default +prefetch behaviour, each visible `` triggers a background `GET /?_rsc=` as soon as +it enters the viewport. + +# Mechanism + +Three properties combine into a portal-wide freeze: + +1. **Fan-out.** The navigation is fully rendered on each page load, so a single navigation issues one + RSC request per visible link — measured at ~10,400 requests over 14 days. +2. **Shared origin over HTTP/1.1.** `portal.autopilotmonitor.com` serves the document, the JS route + chunks *and* the RSC payloads, and it negotiates HTTP/1.1 — not HTTP/2. Browsers cap concurrent + connections per origin at six. +3. **A heavy latency tail on the SSR runtime.** RSC responses are prerendered and edge-cached + (`x-nextjs-cache: HIT`, `s-maxage=31536000`), so the median is fast. On a cache miss or cold start + the Static Web Apps managed runtime queues them. + +When the tail hits, the in-flight prefetches hold all six connections. The document request and the +route chunks of the page the user actually wants then have no socket available, and the tab shows a +loading spinner or a blank page until the queue drains. Nothing fails — the requests eventually +return `200`, which is why the failure leaves no error trace anywhere. + +# Evidence + +Incident of 2026-07-27, 17:25–17:29 CEST. Browser-side dependencies during the freeze: + +| Request | Duration | Status | +| --- | --- | --- | +| `/admin/presence?_rsc=…` | 151,461 ms | 200 | +| `/gather-rules?_rsc=…` | 52,283 ms | 200 | +| `/progress?_rsc=…` | 37,878 ms | 200 | +| `/analyze-rules?_rsc=…` | 37,559 ms | 200 | +| `/audit?_rsc=…` | 37,539 ms | 200 | +| Backend API, same window | 1,600–2,200 ms | 200 | + +The three ~37.5 s responses complete within 340 ms of each other — a queue draining, not independent +slow requests. The corresponding `browserTimings` row for `/geographic-performance` reports +`networkDuration` = 16 ms against `processingDuration` = 225,095 ms: the network delivered the +document immediately and the browser then waited nearly four minutes for the `load` event. + +Fourteen-day distribution of RSC prefetches: n = 10,428, p50 = 91 ms, p95 = 896 ms, max = 234,230 ms, +38 over 10 s, 21 over 30 s. + +# Diagnosing this class of problem + +The backend Application Insights resource is the wrong place to look — a freeze appears there as an +*absence* of requests, never as an error. Browser telemetry lives in a **separate** Application +Insights resource addressed by `AUTOPILOT_MONITOR_WEB_APPINSIGHTS_ID`, which the MCP +`query_backend_logs` tool does not reach. Query it through +`.claude/commands/scripts/query-appinsights.sh` with that ID exported, and read `dependencies` +(client-side fetches, including RSC prefetches) and `browserTimings` (the network/processing split of +a document load). + +# Contract + +Navigation `` elements in `GlobalSidebar` and `Navbar` set `prefetch={false}`. The prefetched +RSC payload buys nothing for these routes, so the only remaining cost of a click is the route's JS +chunk, which the browser caches after first use. + +Do not remove the prop to "speed up navigation" without first re-measuring the tail in the web +Application Insights resource — the prefetch is cheap only while the runtime is warm. + +# Citations + +* `src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx` — nav link rendering, rationale comment +* `src/Web/autopilot-monitor-web/components/Navbar.tsx` — top-bar and notification links +* [SWA and Oryx node pinning](../architecture.md) — hosting model for the web app diff --git a/src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx b/src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx index 3d74dc62..fd61edb3 100644 --- a/src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx +++ b/src/Web/autopilot-monitor-web/components/GlobalSidebar.tsx @@ -312,6 +312,15 @@ export function GlobalSidebar({ children }: { children: ReactNode }) { : "text-gray-600 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-gray-200"; }; + // Every nav Link below sets prefetch={false} on purpose. The sidebar renders the whole + // navigation on every page, so the App Router's default viewport prefetch fires one RSC + // request per visible link against the SWA-hosted Next.js runtime on each page load + // (measured: ~10k requests / 14 days). That host speaks HTTP/1.1, so those prefetches + // occupy the browser's 6 connections to the SAME origin that serves the document and the + // route chunks — when the runtime queues them (p95 ~0.9s but a tail reaching 150s+), the + // portal freezes until the queue drains. Every route here is a client component behind + // ProtectedRoute that fetches its own data after mount, so the prefetched RSC payload + // buys nothing. See docs/web/portal-navigation-prefetch.md. // --- Render a global nav link --- const renderGlobalItem = (item: NavItem, isGlobal = false) => { const active = isNavActive(item.href); @@ -322,6 +331,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) {
  • setMobileDrawerOpen(false)} className={`${base} px-2 py-2 justify-center relative group`} title={item.label} @@ -339,7 +349,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) { return (
  • - setMobileDrawerOpen(false)} className={`${base} px-3 py-1.5`}> + setMobileDrawerOpen(false)} className={`${base} px-3 py-1.5`}> {renderIcon(item.icon, "w-4 h-4")} {item.label} @@ -362,7 +372,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) { if (pageSectionsMode === "route" && item.href) { return (
  • - setMobileDrawerOpen(false)} className={`${base} px-2 py-2 justify-center relative group`} title={item.label}> + setMobileDrawerOpen(false)} className={`${base} px-2 py-2 justify-center relative group`} title={item.label}> {inner} {item.label} @@ -384,7 +394,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) { if (pageSectionsMode === "route" && item.href) { return (
  • - setMobileDrawerOpen(false)} className={`${base} px-3 py-1.5`}> + setMobileDrawerOpen(false)} className={`${base} px-3 py-1.5`}> {renderIcon(item.icon, "w-4 h-4")} {item.label} @@ -502,6 +512,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) {
  • setMobileDrawerOpen(false)} className={`flex items-center justify-center px-2 py-2 rounded-md text-sm transition-colors relative group ${ itemHasActive @@ -567,6 +578,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) {
  • setMobileDrawerOpen(false)} className={`block pl-10 pr-3 py-1 rounded-md text-[13px] transition-colors ${ subActive @@ -644,6 +656,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) {
  • setMobileDrawerOpen(false)} className={`flex items-center justify-center px-2 py-2 rounded-md text-sm transition-colors relative group ${ groupHasActive @@ -701,6 +714,7 @@ export function GlobalSidebar({ children }: { children: ReactNode }) {
  • setMobileDrawerOpen(false)} className={`block pl-10 pr-3 py-1 rounded-md text-[13px] transition-colors ${ active diff --git a/src/Web/autopilot-monitor-web/components/Navbar.tsx b/src/Web/autopilot-monitor-web/components/Navbar.tsx index 93fef427..e745f05c 100644 --- a/src/Web/autopilot-monitor-web/components/Navbar.tsx +++ b/src/Web/autopilot-monitor-web/components/Navbar.tsx @@ -159,7 +159,7 @@ export default function Navbar() {
    - + @@ -215,7 +215,7 @@ export default function Navbar() {
    {/* Logo and Title */}
    - + @@ -326,6 +326,7 @@ export default function Navbar() { {tn.href && ( { e.stopPropagation(); }} className="text-[10px] text-blue-600 hover:text-blue-800 font-medium underline" > @@ -384,6 +385,7 @@ export default function Navbar() { {notification.href && ( { e.stopPropagation(); markAsRead(notification.id); }} className="text-[10px] text-blue-600 hover:text-blue-800 font-medium underline" > @@ -518,6 +520,7 @@ export default function Navbar() { setShowHelp(false)} > @@ -529,6 +532,7 @@ export default function Navbar() { setShowHelp(false)} > @@ -706,11 +710,11 @@ export default function Navbar() { Service Announcements
    - { setShowOverflow(false); setOverflowSubmenu(null); }}> + { setShowOverflow(false); setOverflowSubmenu(null); }}> Privacy Policy - { setShowOverflow(false); setOverflowSubmenu(null); }}> + { setShowOverflow(false); setOverflowSubmenu(null); }}> Terms of Use diff --git a/src/Web/autopilot-monitor-web/utils/page-lastmod.generated.ts b/src/Web/autopilot-monitor-web/utils/page-lastmod.generated.ts index 834fc66c..364bab42 100644 --- a/src/Web/autopilot-monitor-web/utils/page-lastmod.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/page-lastmod.generated.ts @@ -3,12 +3,12 @@ * Run: node scripts/generate-lastmod.js */ export const PAGE_LASTMOD: Record = { - "/": "2026-07-21T13:42:03+02:00", - "/about": "2026-07-22T10:31:49+02:00", + "/": "2026-07-23T08:37:42+02:00", + "/about": "2026-07-25T12:43:42+02:00", "/roadmap": "2026-07-06T18:31:00+02:00", "/changelog": "2026-07-06T18:12:31+02:00", - "/privacy": "2026-07-21T18:00:18+02:00", - "/terms": "2026-07-21T18:00:18+02:00", + "/privacy": "2026-07-25T12:43:42+02:00", + "/terms": "2026-07-23T08:37:42+02:00", "/docs": "2026-07-06T18:12:31+02:00", "/docs/private-preview": "2026-07-06T18:12:31+02:00", "/docs/overview": "2026-07-06T18:12:31+02:00",