diff --git a/AGENTS.md b/AGENTS.md index f88860b3f..5a15fb92a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,19 @@ Examples: - `http://localhost:3000/url/gitbook.com/docs` - `http://localhost:3000/url/open-source.gitbook.io/midjourney` +### PPR routes + +PPR requests are normally resolved upstream and arrive with a large set of `x-gbo-*` headers, so they +can't be reproduced by hitting the dev server directly. `bun run dev:ppr` (from `packages/gitbook`) +starts a dev-only proxy on port 3001 that resolves the URL and injects those headers: + +``` +http://localhost:3001/url/ +``` + +Responses carry `x-gitbook-route-type: ppr` when the PPR route was used. Hot reload doesn't work +through the proxy (its websocket can't be forwarded), so keep using port 3000 while iterating. + ## Architecture ``` diff --git a/packages/gitbook/package.json b/packages/gitbook/package.json index bdca57942..e2716d383 100644 --- a/packages/gitbook/package.json +++ b/packages/gitbook/package.json @@ -122,6 +122,7 @@ "generate:fonts": "bun ./scripts/generate-font-faces.ts", "clean": "rm -rf ./.next && rm -rf ./public/~gitbook/static/icons && rm -rf ./public/~gitbook/static/math && rm -rf ./public/~gitbook/static/mermaid && rm -rf ./public/~gitbook/static/scalar && rm -rf ./public/~gitbook/static/fonts", "dev": "bun run generate:assets && env-cmd --silent -f ../../.env.local next --webpack", + "dev:ppr": "env-cmd --silent -f ../../.env.local bun scripts/ppr-dev-proxy.ts", "build": "bun run generate:assets && next build --webpack", "build:local": "bun run generate:assets && GITBOOK_URL=http://localhost:3000 next build --webpack", "check:css-browser-compatibility": "bun scripts/check-css-browser-compatibility.ts", diff --git a/packages/gitbook/scripts/ppr-dev-proxy.ts b/packages/gitbook/scripts/ppr-dev-proxy.ts new file mode 100644 index 000000000..d656a30c3 --- /dev/null +++ b/packages/gitbook/scripts/ppr-dev-proxy.ts @@ -0,0 +1,256 @@ +/** + * Dev-only proxy that stands in for the upstream layer resolving PPR requests. + * + * It resolves the incoming URL against the published-URLs API and forwards the request to the local + * app with the complete `x-gbo-*` header set, so PPR routes can be exercised locally: + * + * bun dev # app on :3000 + * bun run dev:ppr # this proxy on :3001 + * open http://localhost:3001/url/gitbook.com/docs + * + * Never deploy this. It trusts its input and caches API tokens in memory. + */ +import type { PublishedSiteContent, PublishedSiteContentLookup, Space } from '@gitbook/api'; +import { PPRRequestHeaders } from '../src/lib/ppr'; + +const PORT = Number(process.env.PPR_PROXY_PORT || 3001); +const UPSTREAM = process.env.PPR_UPSTREAM || 'http://localhost:3000'; +const API_URL = process.env.GITBOOK_API_URL || 'https://api.gitbook.com/cache'; +const API_TOKEN = process.env.GITBOOK_API_TOKEN; +const REVALIDATION_ID = process.env.PPR_REVALIDATION_ID; +const LOOKUP_TTL = Number(process.env.PPR_LOOKUP_TTL || 60) * 1000; + +const URL_PREFIX = '/url/'; +const PASSTHROUGH_PREFIXES = ['/_next/', '/~gitbook/static/']; + +function log(message: string) { + // biome-ignore lint/suspicious/noConsole: this is a CLI script + console.log(`[ppr-proxy] ${message}`); +} + +const cache = new Map(); +const inflight = new Map>(); + +/** + * Cache and dedupe an API call. A single page load fans out into many requests for the same site, + * and the site root lookup is shared by every page of a site. + */ +function cached(key: string, fetcher: () => Promise): Promise { + const entry = cache.get(key); + if (entry && entry.expiresAt > Date.now()) { + return Promise.resolve(entry.value as T); + } + + const pending = inflight.get(key); + if (pending) { + return pending as Promise; + } + + const promise = fetcher() + .then((value) => { + cache.set(key, { value, expiresAt: Date.now() + LOOKUP_TTL }); + return value; + }) + .finally(() => inflight.delete(key)); + + inflight.set(key, promise); + return promise; +} + +async function api(path: string, init: RequestInit, token = API_TOKEN): Promise { + const response = await fetch(`${API_URL}/v1${path}`, { + ...init, + headers: { + 'content-type': 'application/json', + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...init.headers, + }, + }); + + if (!response.ok) { + throw new Error(`${path}: ${response.status} ${await response.text()}`); + } + + return response.json() as Promise; +} + +function lookupPublishedURL(url: string): Promise { + return cached(`url:${url}`, () => + api('/urls/published', { + method: 'POST', + body: JSON.stringify({ url }), + }) + ); +} + +/** + * The lookup only carries a revision for preview URLs, so for a regular published URL we read the + * space's active revision using the short-lived token the lookup returned. + */ +function getActiveRevision(content: PublishedSiteContent): Promise { + return cached(`revision:${content.space}`, async () => { + const space = await api(`/spaces/${content.space}`, {}, content.apiToken); + return space.revision; + }); +} + +/** + * The `x-gbo-default-*` headers describe the site's default variant, which is the cache key of the + * shared header. Resolving the site root is the only way to get it from the published-URLs API. + */ +async function getDefaults(content: PublishedSiteContent) { + const fallback = { + siteSection: content.siteSection, + siteSpace: content.siteSpace, + space: content.space, + }; + + try { + const rootURL = `https://${new URL(content.canonicalUrl).host}${content.siteBasePath}`; + const root = await lookupPublishedURL(rootURL); + if ('redirect' in root) { + return fallback; + } + return { siteSection: root.siteSection, siteSpace: root.siteSpace, space: root.space }; + } catch { + return fallback; + } +} + +function setPPRHeaders( + headers: Headers, + content: PublishedSiteContent & { revision: string }, + defaults: { siteSection: string | undefined; siteSpace: string; space: string } +) { + headers.set(PPRRequestHeaders.Site, content.site); + headers.set(PPRRequestHeaders.SiteSection, content.siteSection ?? ''); + headers.set(PPRRequestHeaders.SiteSpace, content.siteSpace); + headers.set(PPRRequestHeaders.Space, content.space); + headers.set(PPRRequestHeaders.SiteBasePath, content.siteBasePath); + headers.set(PPRRequestHeaders.BasePath, content.basePath); + // An empty pathname is rejected; the root page is `/`. + headers.set(PPRRequestHeaders.Pathname, content.pathname || '/'); + headers.set(PPRRequestHeaders.Organization, content.organization); + headers.set(PPRRequestHeaders.ShareKey, content.shareKey ?? ''); + headers.set(PPRRequestHeaders.Complete, String(content.complete)); + headers.set(PPRRequestHeaders.ContextID, content.contextId ?? ''); + headers.set(PPRRequestHeaders.CanonicalURL, content.canonicalUrl); + // Anything other than '', 'true' or 'false' rejects the whole PPR request. + headers.set( + PPRRequestHeaders.Preview, + content.preview === undefined ? '' : String(content.preview) + ); + headers.set(PPRRequestHeaders.Revision, content.revision); + headers.set(PPRRequestHeaders.ChangeRequest, content.changeRequest ?? ''); + headers.set(PPRRequestHeaders.APIToken, content.apiToken); + headers.set(PPRRequestHeaders.RevalidationID, REVALIDATION_ID || content.revision); + // Unlike the other optional headers this one is checked with `has()`, so it must be sent even + // when the site has no sections. + headers.set(PPRRequestHeaders.DefaultSiteSection, defaults.siteSection ?? ''); + headers.set(PPRRequestHeaders.DefaultSiteSpace, defaults.siteSpace); + headers.set(PPRRequestHeaders.DefaultSpace, defaults.space); +} + +/** + * `fetch` decodes the response body, so the upstream framing headers no longer describe what we are + * about to send. Forwarding `content-encoding: gzip` with plain bytes renders as a blank page. + */ +const DECODED_RESPONSE_HEADERS = ['content-encoding', 'content-length', 'transfer-encoding']; + +async function forward(request: Request, url: URL, headers: Headers): Promise { + const hasBody = request.method !== 'GET' && request.method !== 'HEAD'; + + let response: Response; + try { + response = await fetch(new URL(url.pathname + url.search, UPSTREAM), { + method: request.method, + headers, + body: hasBody ? request.body : undefined, + redirect: 'manual', + ...(hasBody ? { duplex: 'half' } : {}), + } as RequestInit); + } catch (error) { + log(`${request.method} ${url.pathname} → upstream unreachable at ${UPSTREAM}`); + return new Response(`Upstream ${UPSTREAM} unreachable: ${error}`, { status: 502 }); + } + + const responseHeaders = new Headers(response.headers); + for (const name of DECODED_RESPONSE_HEADERS) { + responseHeaders.delete(name); + } + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers: responseHeaders, + }); +} + +async function handle(request: Request): Promise { + const url = new URL(request.url); + + // `fetch` can't perform an upgrade, so hot reload only works when hitting the app directly. + if (request.headers.get('upgrade') === 'websocket') { + return new Response('Websocket upgrades are not proxied', { status: 501 }); + } + + const headers = new Headers(request.headers); + + // Never let a client inject its own PPR headers. + for (const name of Object.values(PPRRequestHeaders)) { + headers.delete(name); + } + + if ( + !url.pathname.startsWith(URL_PREFIX) || + PASSTHROUGH_PREFIXES.some((prefix) => url.pathname.startsWith(prefix)) + ) { + return forward(request, url, headers); + } + + const publishedURL = `https://${url.pathname.slice(URL_PREFIX.length)}${url.search}`; + + const skip = (reason: string) => { + log(`${request.method} ${url.pathname} → no PPR: ${reason}`); + return forward(request, url, headers); + }; + + let content: PublishedSiteContent; + try { + const result = await lookupPublishedURL(publishedURL); + if ('redirect' in result) { + return skip(`lookup redirected to ${result.redirect}`); + } + content = result; + } catch (error) { + return skip(`lookup failed: ${error instanceof Error ? error.message : String(error)}`); + } + + let revision: string; + let defaults: Awaited>; + try { + [revision, defaults] = await Promise.all([ + content.revision ?? getActiveRevision(content), + getDefaults(content), + ]); + } catch (error) { + return skip(`no revision: ${error instanceof Error ? error.message : String(error)}`); + } + + setPPRHeaders(headers, { ...content, revision }, defaults); + + log( + `${request.method} ${url.pathname} → site=${content.site} space=${content.space} revision=${revision}` + ); + + return forward(request, url, headers); +} + +Bun.serve({ + port: PORT, + idleTimeout: 60, + fetch: handle, +}); + +log(`listening on http://localhost:${PORT} → ${UPSTREAM}`); +log(`try http://localhost:${PORT}/url/gitbook.com/docs`); diff --git a/packages/gitbook/src/app/utils.ts b/packages/gitbook/src/app/utils.ts index d4d07f2f5..7d82dc0ba 100644 --- a/packages/gitbook/src/app/utils.ts +++ b/packages/gitbook/src/app/utils.ts @@ -181,20 +181,29 @@ export function getPPRHeaderRouteParams(params: PPRRouteLayoutParams): RouteLayo return { ...routeParams, - siteData: encodeURIComponent( - rison.encode({ - ...siteURLData, - // For the header, we don't want to vary the cache by page path, so we set it to the root. - pathname: '/', - // For the header, we keep site section and space data from the PPR defaults, so that the header can be cached across all pages in a site. - siteSection: defaults.siteSection ?? undefined, - siteSpace: defaults.siteSpace, - space: defaults.space, - }) - ), + siteData: encodeSiteData({ + ...siteURLData, + // For the header, we don't want to vary the cache by page path, so we set it to the root. + pathname: '/', + // For the header, we keep site section and space data from the PPR defaults, so that the header can be cached across all pages in a site. + siteSection: defaults.siteSection ?? undefined, + siteSpace: defaults.siteSpace, + space: defaults.space, + }), }; } +/** rison can't encode undefined values, so they are dropped like the middleware does. */ +function encodeSiteData(siteURLData: Record): string { + return encodeURIComponent( + rison.encode( + Object.fromEntries( + Object.entries(siteURLData).filter(([_, value]) => typeof value !== 'undefined') + ) + ) + ); +} + /** * Project PPR params for the table of contents, keeping its current location data. * For the table of contents, we don't want to vary the cache by page path, so we set it to the root.