From fae34e56ac40f6684beab5031cc1f00e8df35cbc Mon Sep 17 00:00:00 2001 From: Viktor Renkema <49148610+viktorrenkema@users.noreply.github.com> Date: Wed, 21 May 2025 14:20:39 +0200 Subject: [PATCH 01/15] Calculate class more explicitly & separately to fix horizontal table cell alignment issue (#3252) --- .../src/components/DocumentView/Table/RecordColumnValue.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/gitbook/src/components/DocumentView/Table/RecordColumnValue.tsx b/packages/gitbook/src/components/DocumentView/Table/RecordColumnValue.tsx index aa1bf45b8..50dc0d827 100644 --- a/packages/gitbook/src/components/DocumentView/Table/RecordColumnValue.tsx +++ b/packages/gitbook/src/components/DocumentView/Table/RecordColumnValue.tsx @@ -115,7 +115,8 @@ export async function RecordColumnValue( return {''}; } - const horizontalAlignment = `[&_*]:${getColumnAlignment(definition)} ${getColumnAlignment(definition)}`; + const horizontalAlignment = getColumnAlignment(definition); + const childrenHorizontalAlignment = `[&_*]:${horizontalAlignment}`; return ( ( 'leading-normal', verticalAlignment, horizontalAlignment, + childrenHorizontalAlignment, ]} context={context} blockStyle={['w-full', 'max-w-[unset]']} From 19c20c0be35374d7126db17009a40d313795ced8 Mon Sep 17 00:00:00 2001 From: conico974 Date: Wed, 21 May 2025 15:39:52 +0200 Subject: [PATCH 02/15] Fix OGImage icon fetch on cloudflare (#3253) Co-authored-by: Nicolas Dorseuil --- packages/gitbook/src/routes/ogimage.tsx | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/packages/gitbook/src/routes/ogimage.tsx b/packages/gitbook/src/routes/ogimage.tsx index c1cfd9772..ee586cb1b 100644 --- a/packages/gitbook/src/routes/ogimage.tsx +++ b/packages/gitbook/src/routes/ogimage.tsx @@ -9,7 +9,6 @@ import { getAssetURL } from '@/lib/assets'; import { filterOutNullable } from '@/lib/typescript'; import { getCacheTag } from '@gitbook/cache-tags'; import type { GitBookSiteContext } from '@v2/lib/context'; -import { getCloudflareContext } from '@v2/lib/data/cloudflare'; import { getResizedImageURL } from '@v2/lib/images'; const googleFontsMap: { [fontName in CustomizationDefaultFont]: string } = { @@ -322,24 +321,6 @@ function logOnCloudflareOnly(message: string) { } } -/** - * Fetch a resource from the function itself. - * To avoid error with worker to worker requests in the same zone, we use the `WORKER_SELF_REFERENCE` binding. - */ -async function fetchSelf(url: string) { - const cloudflare = getCloudflareContext(); - if (cloudflare?.env.WORKER_SELF_REFERENCE) { - logOnCloudflareOnly(`Fetching self: ${url}`); - return await cloudflare.env.WORKER_SELF_REFERENCE.fetch( - // `getAssetURL` can return a relative URL, so we need to make it absolute - // the URL doesn't matter, as we're using the worker-self-reference binding - new URL(url, 'https://worker-self-reference/') - ); - } - - return await fetch(url); -} - /** * Read an image from a response as a base64 encoded string. */ @@ -363,6 +344,7 @@ const staticImagesCache = new Map(); * Read a static image and cache it in memory. */ async function readStaticImage(url: string) { + logOnCloudflareOnly(`Reading static image: ${url}, cache size: ${staticImagesCache.size}`); const cached = staticImagesCache.get(url); if (cached) { return cached; @@ -377,7 +359,7 @@ async function readStaticImage(url: string) { * Read an image from GitBook itself. */ async function readSelfImage(url: string) { - const response = await fetchSelf(url); + const response = await fetch(url); const image = await readImage(response); return image; } From a0c06a72cc8aa255bc8c068e6dca5a5701ff4314 Mon Sep 17 00:00:00 2001 From: "Nolann B." <100787331+nolannbiron@users.noreply.github.com> Date: Fri, 23 May 2025 09:13:20 +0200 Subject: [PATCH 03/15] Indent JSON python code sample (#3260) --- .changeset/rotten-seals-rush.md | 5 ++++ .../react-openapi/src/code-samples.test.ts | 4 +-- packages/react-openapi/src/code-samples.ts | 28 +++++++++++-------- 3 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 .changeset/rotten-seals-rush.md diff --git a/.changeset/rotten-seals-rush.md b/.changeset/rotten-seals-rush.md new file mode 100644 index 000000000..950e25880 --- /dev/null +++ b/.changeset/rotten-seals-rush.md @@ -0,0 +1,5 @@ +--- +'@gitbook/react-openapi': patch +--- + +Indent JSON python code sample diff --git a/packages/react-openapi/src/code-samples.test.ts b/packages/react-openapi/src/code-samples.test.ts index 7fd24bb3f..375cee84b 100644 --- a/packages/react-openapi/src/code-samples.test.ts +++ b/packages/react-openapi/src/code-samples.test.ts @@ -400,7 +400,7 @@ describe('python code sample generator', () => { const output = generator?.generate(input); expect(output).toBe( - 'import requests\n\nresponse = requests.get(\n "https://example.com/path",\n headers={"Content-Type":"application/x-www-form-urlencoded"},\n data={"key":"value"}\n)\n\ndata = response.json()' + 'import requests\n\nresponse = requests.get(\n "https://example.com/path",\n headers={"Content-Type":"application/x-www-form-urlencoded"},\n data={\n "key": "value"\n }\n)\n\ndata = response.json()' ); }); @@ -422,7 +422,7 @@ describe('python code sample generator', () => { const output = generator?.generate(input); expect(output).toBe( - 'import requests\n\nresponse = requests.get(\n "https://example.com/path",\n headers={"Content-Type":"application/json"},\n data=json.dumps({"key":"value","truethy":True,"falsey":False,"nullish":None})\n)\n\ndata = response.json()' + 'import requests\n\nresponse = requests.get(\n "https://example.com/path",\n headers={"Content-Type":"application/json"},\n data=json.dumps({\n "key": "value",\n "truethy": True,\n "falsey": False,\n "nullish": None\n })\n)\n\ndata = response.json()' ); }); diff --git a/packages/react-openapi/src/code-samples.ts b/packages/react-openapi/src/code-samples.ts index 50c6a9204..8d855bbb6 100644 --- a/packages/react-openapi/src/code-samples.ts +++ b/packages/react-openapi/src/code-samples.ts @@ -356,18 +356,22 @@ const BodyGenerators = { // Convert JSON to XML if needed body = JSON.stringify(convertBodyToXML(body)); } else { - body = stringifyOpenAPI(body, (_key, value) => { - switch (value) { - case true: - return '$$__TRUE__$$'; - case false: - return '$$__FALSE__$$'; - case null: - return '$$__NULL__$$'; - default: - return value; - } - }) + body = stringifyOpenAPI( + body, + (_key, value) => { + switch (value) { + case true: + return '$$__TRUE__$$'; + case false: + return '$$__FALSE__$$'; + case null: + return '$$__NULL__$$'; + default: + return value; + } + }, + 2 + ) .replaceAll('"$$__TRUE__$$"', 'True') .replaceAll('"$$__FALSE__$$"', 'False') .replaceAll('"$$__NULL__$$"', 'None'); From dc4268db64144c0e144d2541b317969a35fe5c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Fri, 23 May 2025 10:01:15 +0200 Subject: [PATCH 04/15] Compute correct urls for variants/sections when previewing unpublished site (#3259) --- .changeset/cool-seas-approve.md | 5 +++++ .../src/components/Header/SpacesDropdown.tsx | 14 +++++++++++++- .../SiteSections/encodeClientSiteSections.ts | 4 +++- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 .changeset/cool-seas-approve.md diff --git a/.changeset/cool-seas-approve.md b/.changeset/cool-seas-approve.md new file mode 100644 index 000000000..c97f692e6 --- /dev/null +++ b/.changeset/cool-seas-approve.md @@ -0,0 +1,5 @@ +--- +"gitbook": patch +--- + +Fix navigation between sections/variants when previewing a site in v2 diff --git a/packages/gitbook/src/components/Header/SpacesDropdown.tsx b/packages/gitbook/src/components/Header/SpacesDropdown.tsx index 9728216b7..3fc133523 100644 --- a/packages/gitbook/src/components/Header/SpacesDropdown.tsx +++ b/packages/gitbook/src/components/Header/SpacesDropdown.tsx @@ -2,6 +2,7 @@ import type { SiteSpace } from '@gitbook/api'; import { tcls } from '@/lib/tailwind'; +import { joinPath } from '@/lib/paths'; import type { GitBookSiteContext } from '@v2/lib/context'; import { DropdownChevron, DropdownMenu } from './DropdownMenu'; import { SpacesDropdownMenuItem } from './SpacesDropdownMenuItem'; @@ -74,7 +75,7 @@ export function SpacesDropdown(props: { title: otherSiteSpace.title, url: otherSiteSpace.urls.published ? linker.toLinkForContent(otherSiteSpace.urls.published) - : otherSiteSpace.space.urls.app, + : getFallbackSiteSpaceURL(otherSiteSpace, context), }} active={otherSiteSpace.id === siteSpace.id} /> @@ -82,3 +83,14 @@ export function SpacesDropdown(props: { ); } + +/** + * When the site is not published yet, `urls.published` is not available. + * To ensure navigation works in preview, we compute a relative URL from the siteSpace path. + */ +function getFallbackSiteSpaceURL(siteSpace: SiteSpace, context: GitBookSiteContext) { + const { linker, sections } = context; + return linker.toPathInSite( + sections?.current ? joinPath(sections.current.path, siteSpace.path) : siteSpace.path + ); +} diff --git a/packages/gitbook/src/components/SiteSections/encodeClientSiteSections.ts b/packages/gitbook/src/components/SiteSections/encodeClientSiteSections.ts index 79a007ecc..49b73a8e1 100644 --- a/packages/gitbook/src/components/SiteSections/encodeClientSiteSections.ts +++ b/packages/gitbook/src/components/SiteSections/encodeClientSiteSections.ts @@ -53,6 +53,8 @@ function encodeSection(context: GitBookSiteContext, section: SiteSection) { description: section.description, icon: section.icon, object: section.object, - url: section.urls.published ? linker.toLinkForContent(section.urls.published) : '', + url: section.urls.published + ? linker.toLinkForContent(section.urls.published) + : linker.toPathInSite(section.path), }; } From fa3eb076177571914ab4bd40c83757bba64e0bc7 Mon Sep 17 00:00:00 2001 From: conico974 Date: Fri, 23 May 2025 10:20:07 +0200 Subject: [PATCH 05/15] Cache fonts and static image in memory for OG Image (#3258) Co-authored-by: Nicolas Dorseuil --- .changeset/stupid-plums-perform.md | 6 +++++ packages/gitbook/src/routes/ogimage.tsx | 34 ++++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 .changeset/stupid-plums-perform.md diff --git a/.changeset/stupid-plums-perform.md b/.changeset/stupid-plums-perform.md new file mode 100644 index 000000000..34707627e --- /dev/null +++ b/.changeset/stupid-plums-perform.md @@ -0,0 +1,6 @@ +--- +"gitbook": patch +"gitbook-v2": patch +--- + +cache fonts and static image used in OGImage in memory diff --git a/packages/gitbook/src/routes/ogimage.tsx b/packages/gitbook/src/routes/ogimage.tsx index ee586cb1b..b086ffd1c 100644 --- a/packages/gitbook/src/routes/ogimage.tsx +++ b/packages/gitbook/src/routes/ogimage.tsx @@ -72,8 +72,12 @@ export async function serveOGImage(baseContext: GitBookSiteContext, params: Page const fonts = ( await Promise.all([ - loadGoogleFont({ fontFamily, text: regularText, weight: 400 }), - loadGoogleFont({ fontFamily, text: boldText, weight: 700 }), + getWithCache(`google-font:${fontFamily}:400`, () => + loadGoogleFont({ fontFamily, text: regularText, weight: 400 }) + ), + getWithCache(`google-font:${fontFamily}:700`, () => + loadGoogleFont({ fontFamily, text: boldText, weight: 700 }) + ), ]) ).filter(filterOutNullable); @@ -338,21 +342,27 @@ async function readImage(response: Response) { return `data:${contentType};base64,${base64}`; } -const staticImagesCache = new Map(); +// biome-ignore lint/suspicious/noExplicitAny: +const staticCache = new Map(); + +// Do we need to limit the in-memory cache size? I think given the usage, we should be fine. +async function getWithCache(key: string, fn: () => Promise) { + const cached = staticCache.get(key) as T; + if (cached) { + return Promise.resolve(cached); + } + + const result = await fn(); + staticCache.set(key, result); + return result; +} /** * Read a static image and cache it in memory. */ async function readStaticImage(url: string) { - logOnCloudflareOnly(`Reading static image: ${url}, cache size: ${staticImagesCache.size}`); - const cached = staticImagesCache.get(url); - if (cached) { - return cached; - } - - const image = await readSelfImage(url); - staticImagesCache.set(url, image); - return image; + logOnCloudflareOnly(`Reading static image: ${url}, cache size: ${staticCache.size}`); + return getWithCache(`static-image:${url}`, () => readSelfImage(url)); } /** From 6f8ecd611138ce1a148687696a01009aa12452a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Fri, 23 May 2025 13:16:43 +0200 Subject: [PATCH 06/15] Use more strict context ID for images during preview (#3261) --- packages/gitbook-v2/src/lib/data/index.ts | 1 - packages/gitbook-v2/src/lib/data/visitor.ts | 2 +- .../images/getImageResizingContextId.test.ts | 19 +++++++++++++++++ .../lib/images/getImageResizingContextId.ts | 6 +++++- packages/gitbook-v2/src/lib/preview.test.ts | 21 +++++++++++++++++++ packages/gitbook-v2/src/lib/preview.ts | 13 ++++++++++++ .../src/lib/{data => }/proxy.test.ts | 0 .../gitbook-v2/src/lib/{data => }/proxy.ts | 0 8 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 packages/gitbook-v2/src/lib/images/getImageResizingContextId.test.ts create mode 100644 packages/gitbook-v2/src/lib/preview.test.ts create mode 100644 packages/gitbook-v2/src/lib/preview.ts rename packages/gitbook-v2/src/lib/{data => }/proxy.test.ts (100%) rename packages/gitbook-v2/src/lib/{data => }/proxy.ts (100%) diff --git a/packages/gitbook-v2/src/lib/data/index.ts b/packages/gitbook-v2/src/lib/data/index.ts index d79049684..2e37e2fbb 100644 --- a/packages/gitbook-v2/src/lib/data/index.ts +++ b/packages/gitbook-v2/src/lib/data/index.ts @@ -4,5 +4,4 @@ export * from './pages'; export * from './urls'; export * from './errors'; export * from './lookup'; -export * from './proxy'; export * from './visitor'; diff --git a/packages/gitbook-v2/src/lib/data/visitor.ts b/packages/gitbook-v2/src/lib/data/visitor.ts index f93f0afe8..e59e32365 100644 --- a/packages/gitbook-v2/src/lib/data/visitor.ts +++ b/packages/gitbook-v2/src/lib/data/visitor.ts @@ -1,6 +1,6 @@ import { withLeadingSlash, withTrailingSlash } from '@/lib/paths'; import type { PublishedSiteContent } from '@gitbook/api'; -import { getProxyRequestIdentifier, isProxyRequest } from './proxy'; +import { getProxyRequestIdentifier, isProxyRequest } from '@v2/lib/proxy'; /** * Get the appropriate base path for the visitor authentication cookie. diff --git a/packages/gitbook-v2/src/lib/images/getImageResizingContextId.test.ts b/packages/gitbook-v2/src/lib/images/getImageResizingContextId.test.ts new file mode 100644 index 000000000..eaba01663 --- /dev/null +++ b/packages/gitbook-v2/src/lib/images/getImageResizingContextId.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'bun:test'; +import { getImageResizingContextId } from './getImageResizingContextId'; + +describe('getImageResizingContextId', () => { + it('should return proxy identifier for proxy requests', () => { + const proxyRequestURL = new URL('https://proxy.gitbook.site/sites/site_foo/hello/world'); + expect(getImageResizingContextId(proxyRequestURL)).toBe('sites/site_foo'); + }); + + it('should return preview identifier for preview requests', () => { + const previewRequestURL = new URL('https://preview/site_foo/hello/world'); + expect(getImageResizingContextId(previewRequestURL)).toBe('site_foo'); + }); + + it('should return host for regular requests', () => { + const regularRequestURL = new URL('https://example.com/docs/foo/hello/world'); + expect(getImageResizingContextId(regularRequestURL)).toBe('example.com'); + }); +}); diff --git a/packages/gitbook-v2/src/lib/images/getImageResizingContextId.ts b/packages/gitbook-v2/src/lib/images/getImageResizingContextId.ts index 82f922526..40594e2ae 100644 --- a/packages/gitbook-v2/src/lib/images/getImageResizingContextId.ts +++ b/packages/gitbook-v2/src/lib/images/getImageResizingContextId.ts @@ -1,4 +1,5 @@ -import { getProxyRequestIdentifier, isProxyRequest } from '../data'; +import { getPreviewRequestIdentifier, isPreviewRequest } from '@v2/lib/preview'; +import { getProxyRequestIdentifier, isProxyRequest } from '@v2/lib/proxy'; /** * Get the site identifier to use for image resizing for an incoming request. @@ -8,6 +9,9 @@ export function getImageResizingContextId(url: URL): string { if (isProxyRequest(url)) { return getProxyRequestIdentifier(url); } + if (isPreviewRequest(url)) { + return getPreviewRequestIdentifier(url); + } return url.host; } diff --git a/packages/gitbook-v2/src/lib/preview.test.ts b/packages/gitbook-v2/src/lib/preview.test.ts new file mode 100644 index 000000000..bbaf0402b --- /dev/null +++ b/packages/gitbook-v2/src/lib/preview.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'bun:test'; +import { getPreviewRequestIdentifier, isPreviewRequest } from './preview'; + +describe('isPreviewRequest', () => { + it('should return true for preview requests', () => { + const previewRequestURL = new URL('https://preview/site_foo/hello/world'); + expect(isPreviewRequest(previewRequestURL)).toBe(true); + }); + + it('should return false for non-preview requests', () => { + const nonPreviewRequestURL = new URL('https://example.com/docs/foo/hello/world'); + expect(isPreviewRequest(nonPreviewRequestURL)).toBe(false); + }); +}); + +describe('getPreviewRequestIdentifier', () => { + it('should return the correct identifier for preview requests', () => { + const previewRequestURL = new URL('https://preview/site_foo/hello/world'); + expect(getPreviewRequestIdentifier(previewRequestURL)).toBe('site_foo'); + }); +}); diff --git a/packages/gitbook-v2/src/lib/preview.ts b/packages/gitbook-v2/src/lib/preview.ts new file mode 100644 index 000000000..7094d1197 --- /dev/null +++ b/packages/gitbook-v2/src/lib/preview.ts @@ -0,0 +1,13 @@ +/** + * Check if the request to the site is a preview request. + */ +export function isPreviewRequest(requestURL: URL): boolean { + return requestURL.host === 'preview'; +} + +export function getPreviewRequestIdentifier(requestURL: URL): string { + // For preview requests, we extract the site ID from the pathname + // e.g. https://preview/site_id/... + const pathname = requestURL.pathname.slice(1).split('/'); + return pathname[0]; +} diff --git a/packages/gitbook-v2/src/lib/data/proxy.test.ts b/packages/gitbook-v2/src/lib/proxy.test.ts similarity index 100% rename from packages/gitbook-v2/src/lib/data/proxy.test.ts rename to packages/gitbook-v2/src/lib/proxy.test.ts diff --git a/packages/gitbook-v2/src/lib/data/proxy.ts b/packages/gitbook-v2/src/lib/proxy.ts similarity index 100% rename from packages/gitbook-v2/src/lib/data/proxy.ts rename to packages/gitbook-v2/src/lib/proxy.ts From af66ff71ed65b0e215377fcfa9424d9837c9e661 Mon Sep 17 00:00:00 2001 From: conico974 Date: Sun, 25 May 2025 16:55:45 +0200 Subject: [PATCH 07/15] Add a force-revalidate api route (#3263) Co-authored-by: Nicolas Dorseuil --- .changeset/gorgeous-cycles-cheat.md | 5 ++ packages/gitbook-v2/next-env.d.ts | 1 + .../pages/api/~gitbook/force-revalidate.ts | 49 +++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 .changeset/gorgeous-cycles-cheat.md create mode 100644 packages/gitbook-v2/src/pages/api/~gitbook/force-revalidate.ts diff --git a/.changeset/gorgeous-cycles-cheat.md b/.changeset/gorgeous-cycles-cheat.md new file mode 100644 index 000000000..d13c27765 --- /dev/null +++ b/.changeset/gorgeous-cycles-cheat.md @@ -0,0 +1,5 @@ +--- +"gitbook-v2": patch +--- + +add a force-revalidate api route to force bust the cache in case of errors diff --git a/packages/gitbook-v2/next-env.d.ts b/packages/gitbook-v2/next-env.d.ts index 1b3be0840..3cd7048ed 100644 --- a/packages/gitbook-v2/next-env.d.ts +++ b/packages/gitbook-v2/next-env.d.ts @@ -1,5 +1,6 @@ /// /// +/// // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/packages/gitbook-v2/src/pages/api/~gitbook/force-revalidate.ts b/packages/gitbook-v2/src/pages/api/~gitbook/force-revalidate.ts new file mode 100644 index 000000000..45e6c7eca --- /dev/null +++ b/packages/gitbook-v2/src/pages/api/~gitbook/force-revalidate.ts @@ -0,0 +1,49 @@ +import crypto from 'node:crypto'; +import type { NextApiRequest, NextApiResponse } from 'next'; + +interface JsonBody { + // The paths need to be the rewritten one, `res.revalidate` call don't go through the middleware + paths: string[]; +} + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + // Only allow POST requests + if (req.method !== 'POST') { + return res.status(405).json({ error: 'Method not allowed' }); + } + const signatureHeader = req.headers['x-gitbook-signature'] as string | undefined; + if (!signatureHeader) { + return res.status(400).json({ error: 'Missing signature header' }); + } + // We cannot use env from `@/v2/lib/env` here as it make it crash because of the import "server-only" in the file. + if (process.env.GITBOOK_SECRET) { + try { + const computedSignature = crypto + .createHmac('sha256', process.env.GITBOOK_SECRET) + .update(JSON.stringify(req.body)) + .digest('hex'); + + if (computedSignature === signatureHeader) { + const results = await Promise.allSettled( + (req.body as JsonBody).paths.map((path) => { + // biome-ignore lint/suspicious/noConsole: we want to log here + console.log(`Revalidating path: ${path}`); + return res.revalidate(path); + }) + ); + return res.status(200).json({ + success: results.every((result) => result.status === 'fulfilled'), + errors: results + .filter((result) => result.status === 'rejected') + .map((result) => (result as PromiseRejectedResult).reason), + }); + } + return res.status(401).json({ error: 'Invalid signature' }); + } catch (error) { + console.error('Error during revalidation:', error); + return res.status(400).json({ error: 'Invalid request or unable to parse JSON' }); + } + } + // If no secret is set, we do not allow revalidation + return res.status(403).json({ error: 'Revalidation is disabled' }); +} From 40df91a3639e8bfd93c589d5fb3c8ff552c5232e Mon Sep 17 00:00:00 2001 From: "Nolann B." <100787331+nolannbiron@users.noreply.github.com> Date: Mon, 26 May 2025 18:55:11 +0200 Subject: [PATCH 08/15] Deduplicate path parameters from OpenAPI spec (#3264) --- .changeset/violet-schools-care.md | 5 +++++ packages/react-openapi/src/OpenAPISpec.tsx | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .changeset/violet-schools-care.md diff --git a/.changeset/violet-schools-care.md b/.changeset/violet-schools-care.md new file mode 100644 index 000000000..e77a98b5c --- /dev/null +++ b/.changeset/violet-schools-care.md @@ -0,0 +1,5 @@ +--- +'@gitbook/react-openapi': patch +--- + +Deduplicate path parameters from OpenAPI spec diff --git a/packages/react-openapi/src/OpenAPISpec.tsx b/packages/react-openapi/src/OpenAPISpec.tsx index 49c41cda4..1e6e56269 100644 --- a/packages/react-openapi/src/OpenAPISpec.tsx +++ b/packages/react-openapi/src/OpenAPISpec.tsx @@ -18,7 +18,7 @@ export function OpenAPISpec(props: { const { operation } = data; - const parameters = operation.parameters ?? []; + const parameters = deduplicateParameters(operation.parameters ?? []); const parameterGroups = groupParameters(parameters, context); const securities = 'securities' in data ? data.securities : []; @@ -113,3 +113,23 @@ function getParameterGroupName(paramIn: string, context: OpenAPIClientContext): return paramIn; } } + +/** Deduplicate parameters by name and in. + * Some specs have both parameters define at path and operation level. + * We only want to display one of them. + */ +function deduplicateParameters(parameters: OpenAPI.Parameters): OpenAPI.Parameters { + const seen = new Set(); + + return parameters.filter((param) => { + const key = `${param.name}:${param.in}`; + + if (seen.has(key)) { + return false; + } + + seen.add(key); + + return true; + }); +} From 4c9a9d0416c91398198027f637b1aa7049f20899 Mon Sep 17 00:00:00 2001 From: "Nolann B." <100787331+nolannbiron@users.noreply.github.com> Date: Mon, 26 May 2025 20:28:26 +0200 Subject: [PATCH 09/15] Handle nested deprecated properties in generateSchemaExample (#3267) --- .changeset/thick-chefs-repeat.md | 5 +++++ .../src/generateSchemaExample.test.ts | 20 +++++++++++++++++++ .../src/generateSchemaExample.ts | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .changeset/thick-chefs-repeat.md diff --git a/.changeset/thick-chefs-repeat.md b/.changeset/thick-chefs-repeat.md new file mode 100644 index 000000000..8b08d50e3 --- /dev/null +++ b/.changeset/thick-chefs-repeat.md @@ -0,0 +1,5 @@ +--- +'@gitbook/react-openapi': patch +--- + +Handle nested deprecated properties in generateSchemaExample diff --git a/packages/react-openapi/src/generateSchemaExample.test.ts b/packages/react-openapi/src/generateSchemaExample.test.ts index 3181881b6..682c5f2f3 100644 --- a/packages/react-openapi/src/generateSchemaExample.test.ts +++ b/packages/react-openapi/src/generateSchemaExample.test.ts @@ -1017,4 +1017,24 @@ describe('generateSchemaExample', () => { }, }); }); + + it('handles deprecated properties', () => { + expect( + generateSchemaExample({ + type: 'object', + deprecated: true, + }) + ).toBeUndefined(); + }); + + it('handle nested deprecated properties', () => { + expect( + generateSchemaExample({ + type: 'array', + items: { + deprecated: true, + }, + }) + ).toBeUndefined(); + }); }); diff --git a/packages/react-openapi/src/generateSchemaExample.ts b/packages/react-openapi/src/generateSchemaExample.ts index 5a038db06..595f723d9 100644 --- a/packages/react-openapi/src/generateSchemaExample.ts +++ b/packages/react-openapi/src/generateSchemaExample.ts @@ -167,7 +167,7 @@ const getExampleFromSchema = ( const makeUpRandomData = !!options?.emptyString; // If the property is deprecated we don't show it in examples. - if (schema.deprecated) { + if (schema.deprecated || (schema.type === 'array' && schema.items?.deprecated)) { return undefined; } From 957afd967fc306c404046d27c240bed34bdd2115 Mon Sep 17 00:00:00 2001 From: "Nolann B." <100787331+nolannbiron@users.noreply.github.com> Date: Mon, 26 May 2025 20:54:07 +0200 Subject: [PATCH 10/15] Add authorization header for OAuth2 (#3266) --- .changeset/fast-trees-battle.md | 5 +++++ packages/react-openapi/src/OpenAPICodeSample.tsx | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/fast-trees-battle.md diff --git a/.changeset/fast-trees-battle.md b/.changeset/fast-trees-battle.md new file mode 100644 index 000000000..0e75fbbed --- /dev/null +++ b/.changeset/fast-trees-battle.md @@ -0,0 +1,5 @@ +--- +'@gitbook/react-openapi': patch +--- + +Add authorization header for OAuth2 diff --git a/packages/react-openapi/src/OpenAPICodeSample.tsx b/packages/react-openapi/src/OpenAPICodeSample.tsx index 8b67bbedc..8bffb6d9a 100644 --- a/packages/react-openapi/src/OpenAPICodeSample.tsx +++ b/packages/react-openapi/src/OpenAPICodeSample.tsx @@ -312,6 +312,11 @@ function getSecurityHeaders(securities: OpenAPIOperationData['securities']): { [name]: 'YOUR_API_KEY', }; } + case 'oauth2': { + return { + Authorization: 'Bearer YOUR_OAUTH2_TOKEN', + }; + } default: { return {}; } From 2932077bf90ca92eadbfd8b9c832c5fd6f018e0b Mon Sep 17 00:00:00 2001 From: conico974 Date: Tue, 27 May 2025 21:13:55 +0200 Subject: [PATCH 11/15] Remove trailing slash from linker (#3268) Co-authored-by: Nicolas Dorseuil --- .changeset/wise-gifts-smash.md | 5 +++++ packages/gitbook-v2/src/lib/links.test.ts | 22 +++++++++++++++++++++- packages/gitbook-v2/src/lib/links.ts | 6 +++++- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 .changeset/wise-gifts-smash.md diff --git a/.changeset/wise-gifts-smash.md b/.changeset/wise-gifts-smash.md new file mode 100644 index 000000000..32a85eecb --- /dev/null +++ b/.changeset/wise-gifts-smash.md @@ -0,0 +1,5 @@ +--- +"gitbook-v2": patch +--- + +remove trailing slash from linker diff --git a/packages/gitbook-v2/src/lib/links.test.ts b/packages/gitbook-v2/src/lib/links.test.ts index 26a2464f5..f73ec5ee8 100644 --- a/packages/gitbook-v2/src/lib/links.test.ts +++ b/packages/gitbook-v2/src/lib/links.test.ts @@ -19,7 +19,7 @@ const siteGitBookIO = createLinker({ siteBasePath: '/sitename/', }); -describe('toPathInContent', () => { +describe('toPathInSpace', () => { it('should return the correct path', () => { expect(root.toPathInSpace('some/path')).toBe('/some/path'); expect(variantInSection.toPathInSpace('some/path')).toBe('/section/variant/some/path'); @@ -29,6 +29,16 @@ describe('toPathInContent', () => { expect(root.toPathInSpace('/some/path')).toBe('/some/path'); expect(variantInSection.toPathInSpace('/some/path')).toBe('/section/variant/some/path'); }); + + it('should remove the trailing slash', () => { + expect(root.toPathInSpace('some/path/')).toBe('/some/path'); + expect(variantInSection.toPathInSpace('some/path/')).toBe('/section/variant/some/path'); + }); + + it('should not add a trailing slash', () => { + expect(root.toPathInSpace('')).toBe(''); + expect(variantInSection.toPathInSpace('')).toBe('/section/variant'); + }); }); describe('toPathInSite', () => { @@ -36,6 +46,16 @@ describe('toPathInSite', () => { expect(root.toPathInSite('some/path')).toBe('/some/path'); expect(siteGitBookIO.toPathInSite('some/path')).toBe('/sitename/some/path'); }); + + it('should remove the trailing slash', () => { + expect(root.toPathInSite('some/path/')).toBe('/some/path'); + expect(siteGitBookIO.toPathInSite('some/path/')).toBe('/sitename/some/path'); + }); + + it('should not add a trailing slash', () => { + expect(root.toPathInSite('')).toBe(''); + expect(siteGitBookIO.toPathInSite('')).toBe('/sitename'); + }); }); describe('toRelativePathInSite', () => { diff --git a/packages/gitbook-v2/src/lib/links.ts b/packages/gitbook-v2/src/lib/links.ts index b84565d1e..a64bda541 100644 --- a/packages/gitbook-v2/src/lib/links.ts +++ b/packages/gitbook-v2/src/lib/links.ts @@ -128,5 +128,9 @@ export function createLinker( function joinPaths(prefix: string, path: string): string { const prefixPath = prefix.endsWith('/') ? prefix : `${prefix}/`; const suffixPath = path.startsWith('/') ? path.slice(1) : path; - return prefixPath + suffixPath; + return removeTrailingSlash(prefixPath + suffixPath); +} + +function removeTrailingSlash(path: string): string { + return path.endsWith('/') ? path.slice(0, -1) : path; } From b3a7ad6d2cfc1a55fd3c15c0baefcd39bd3228cf Mon Sep 17 00:00:00 2001 From: conico974 Date: Wed, 28 May 2025 17:07:33 +0200 Subject: [PATCH 12/15] Fix href being empty in TOC (#3269) Co-authored-by: Nicolas Dorseuil --- .changeset/cool-jars-matter.md | 5 +++++ .../src/components/TableOfContents/PageDocumentItem.tsx | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .changeset/cool-jars-matter.md diff --git a/.changeset/cool-jars-matter.md b/.changeset/cool-jars-matter.md new file mode 100644 index 000000000..ff0c934f3 --- /dev/null +++ b/.changeset/cool-jars-matter.md @@ -0,0 +1,5 @@ +--- +"gitbook": patch +--- + +fix href being empty in TOC diff --git a/packages/gitbook/src/components/TableOfContents/PageDocumentItem.tsx b/packages/gitbook/src/components/TableOfContents/PageDocumentItem.tsx index 2ff3289d7..4875633ac 100644 --- a/packages/gitbook/src/components/TableOfContents/PageDocumentItem.tsx +++ b/packages/gitbook/src/components/TableOfContents/PageDocumentItem.tsx @@ -17,7 +17,11 @@ export async function PageDocumentItem(props: { context: GitBookSiteContext; }) { const { rootPages, page, context } = props; - const href = context.linker.toPathForPage({ pages: rootPages, page }); + let href = context.linker.toPathForPage({ pages: rootPages, page }); + // toPathForPage can returns an empty path, this will cause all links to point to the current page. + if (href === '') { + href = '/'; + } return (
  • From df2fa42bda09a85a3ca2412ed898e6e26cf77586 Mon Sep 17 00:00:00 2001 From: conico974 Date: Fri, 30 May 2025 14:31:21 +0200 Subject: [PATCH 13/15] Chores: create server worker (#3265) Co-authored-by: Nicolas Dorseuil --- .../composite/deploy-cloudflare/action.yaml | 11 ++++++ .../openNext/customWorkers/default.js | 9 +++++ .../customWorkers/defaultWrangler.jsonc | 38 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 packages/gitbook-v2/openNext/customWorkers/default.js create mode 100644 packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc diff --git a/.github/composite/deploy-cloudflare/action.yaml b/.github/composite/deploy-cloudflare/action.yaml index fbc98fc82..1cd19c4d5 100644 --- a/.github/composite/deploy-cloudflare/action.yaml +++ b/.github/composite/deploy-cloudflare/action.yaml @@ -73,6 +73,17 @@ runs: wranglerVersion: '4.10.0' environment: ${{ inputs.environment }} command: ${{ inputs.deploy == 'true' && 'deploy' || format('versions upload --tag {0} --message "{1}"', inputs.commitTag, inputs.commitMessage) }} --config ./packages/gitbook-v2/wrangler.jsonc + + - name: Temporary deploy server CF worker + uses: cloudflare/wrangler-action@v3.14.0 + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + workingDirectory: ./ + wranglerVersion: '4.10.0' + environment: ${{ inputs.environment }} + command: 'deploy --config ./packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc' + - name: Outputs shell: bash env: diff --git a/packages/gitbook-v2/openNext/customWorkers/default.js b/packages/gitbook-v2/openNext/customWorkers/default.js new file mode 100644 index 000000000..6e2f1af82 --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/default.js @@ -0,0 +1,9 @@ +export default { + async fetch() { + return new Response('Hello World', { + headers: { + 'Content-Type': 'text/plain', + }, + }); + }, +}; diff --git a/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc b/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc new file mode 100644 index 000000000..5ade8d025 --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc @@ -0,0 +1,38 @@ +{ + "main": "default.js", + "name": "gitbook-open-v2-server", + "compatibility_date": "2025-04-14", + "compatibility_flags": [ + "nodejs_compat", + "allow_importable_env", + "global_fetch_strictly_public" + ], + "observability": { + "enabled": true + }, + "vars": { + "NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true" + }, + "env": { + "dev": { + "vars": { + "STAGE": "dev" + } + }, + "preview": { + "vars": { + "STAGE": "preview" + } + }, + "staging": { + "vars": { + "STAGE": "staging" + } + }, + "production": { + "vars": { + "STAGE": "production" + } + } + } +} From 666e8426e1a39fffdb4de586bad633a1a2049365 Mon Sep 17 00:00:00 2001 From: conico974 Date: Fri, 30 May 2025 14:56:00 +0200 Subject: [PATCH 14/15] External middleware in cloudflare (#3254) Co-authored-by: Nicolas Dorseuil --- .../gradual-deploy-cloudflare/action.yaml | 83 ++++++++ .../composite/deploy-cloudflare/action.yaml | 54 ++++- packages/gitbook-v2/open-next.config.ts | 53 ++--- .../openNext/customWorkers/default.js | 16 +- .../customWorkers/defaultWrangler.jsonc | 96 ++++++++- .../openNext/customWorkers/middleware.js | 42 ++++ .../customWorkers/middlewareWrangler.jsonc | 185 ++++++++++++++++++ .../customWorkers/script/updateWrangler.ts | 26 +++ packages/gitbook-v2/openNext/queue.ts | 17 -- .../gitbook-v2/openNext/queue/middleware.ts | 21 ++ packages/gitbook-v2/openNext/queue/server.ts | 9 + .../openNext/tagCache/middleware.ts | 76 +++++++ packages/gitbook-v2/package.json | 2 + packages/gitbook-v2/src/lib/data/api.ts | 23 +-- 14 files changed, 616 insertions(+), 87 deletions(-) create mode 100644 .github/actions/gradual-deploy-cloudflare/action.yaml create mode 100644 packages/gitbook-v2/openNext/customWorkers/middleware.js create mode 100644 packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc create mode 100644 packages/gitbook-v2/openNext/customWorkers/script/updateWrangler.ts delete mode 100644 packages/gitbook-v2/openNext/queue.ts create mode 100644 packages/gitbook-v2/openNext/queue/middleware.ts create mode 100644 packages/gitbook-v2/openNext/queue/server.ts create mode 100644 packages/gitbook-v2/openNext/tagCache/middleware.ts diff --git a/.github/actions/gradual-deploy-cloudflare/action.yaml b/.github/actions/gradual-deploy-cloudflare/action.yaml new file mode 100644 index 000000000..eff064a75 --- /dev/null +++ b/.github/actions/gradual-deploy-cloudflare/action.yaml @@ -0,0 +1,83 @@ +name: Gradual Deploy to Cloudflare +description: Use gradual deployment to deploy to Cloudflare. This action will upload the middleware and server versions to Cloudflare and kept them bound together +inputs: + apiToken: + description: 'Cloudflare API token' + required: true + accountId: + description: 'Cloudflare account ID' + required: true + environment: + description: 'Cloudflare environment to deploy to (staging, production, preview)' + required: true + middlewareVersionId: + description: 'Middleware version ID to deploy' + required: true + serverVersionId: + description: 'Server version ID to deploy' + required: true +outputs: + deployment-url: + description: "Deployment URL" + value: ${{ steps.deploy_middleware.outputs.deployment-url }} +runs: + using: 'composite' + steps: + - id: wrangler_status + name: Check wrangler deployment status + uses: cloudflare/wrangler-action@v3.14.0 + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + workingDirectory: ./ + wranglerVersion: '4.10.0' + environment: ${{ inputs.environment }} + command: deployments status --config ./packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc + + # This step is used to get the version ID that is currently deployed to Cloudflare. + - id: extract_current_version + name: Extract current version + shell: bash + run: | + version_id=$(echo "${{ steps.wrangler_status.outputs.command-output }}" | grep -A 3 "(100%)" | grep -oP '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}') + echo "version_id=$version_id" >> $GITHUB_OUTPUT + + - id: deploy_server + name: Deploy server to Cloudflare at 0% + uses: cloudflare/wrangler-action@v3.14.0 + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + workingDirectory: ./ + wranglerVersion: '4.10.0' + environment: ${{ inputs.environment }} + command: versions deploy ${{ steps.extract_current_version.outputs.version_id }}@100% ${{ inputs.serverVersionId }}@0% -y --config ./packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc + + # Since we use version overrides headers, we can directly deploy the middleware to 100%. + - id: deploy_middleware + name: Deploy middleware to Cloudflare at 100% + uses: cloudflare/wrangler-action@v3.14.0 + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + workingDirectory: ./ + wranglerVersion: '4.10.0' + environment: ${{ inputs.environment }} + command: versions deploy ${{ inputs.middlewareVersionId }}@100% -y --config ./packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc + + - name: Deploy server to Cloudflare at 100% + uses: cloudflare/wrangler-action@v3.14.0 + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + workingDirectory: ./ + wranglerVersion: '4.10.0' + environment: ${{ inputs.environment }} + command: versions deploy ${{ inputs.serverVersionId }}@100% -y --config ./packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc + + - name: Outputs + shell: bash + env: + DEPLOYMENT_URL: ${{ steps.deploy_middleware.outputs.deployment-url }} + run: | + echo "URL: ${{ steps.deploy_middleware.outputs.deployment-url }}" \ No newline at end of file diff --git a/.github/composite/deploy-cloudflare/action.yaml b/.github/composite/deploy-cloudflare/action.yaml index 1cd19c4d5..d1d0d6633 100644 --- a/.github/composite/deploy-cloudflare/action.yaml +++ b/.github/composite/deploy-cloudflare/action.yaml @@ -28,7 +28,7 @@ inputs: outputs: deployment-url: description: "Deployment URL" - value: ${{ steps.deploy.outputs.deployment-url }} + value: ${{ steps.upload_middleware.outputs.deployment-url }} runs: using: 'composite' steps: @@ -63,8 +63,9 @@ runs: env: GITBOOK_RUNTIME: cloudflare shell: bash - - id: deploy - name: Deploy to Cloudflare + + - id: upload_server + name: Upload server to Cloudflare uses: cloudflare/wrangler-action@v3.14.0 with: apiToken: ${{ inputs.apiToken }} @@ -72,9 +73,22 @@ runs: workingDirectory: ./ wranglerVersion: '4.10.0' environment: ${{ inputs.environment }} - command: ${{ inputs.deploy == 'true' && 'deploy' || format('versions upload --tag {0} --message "{1}"', inputs.commitTag, inputs.commitMessage) }} --config ./packages/gitbook-v2/wrangler.jsonc - - - name: Temporary deploy server CF worker + command: ${{ format('versions upload --tag {0} --message "{1}"', inputs.commitTag, inputs.commitMessage) }} --config ./packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc + + - name: Extract server version worker ID + shell: bash + id: extract_server_version_id + run: | + version_id=$(echo '${{ steps.upload_server.outputs.command-output }}' | grep "Worker Version ID" | awk '{print $4}') + echo "version_id=$version_id" >> $GITHUB_OUTPUT + + - name: Run updateWrangler scripts + shell: bash + run: | + bun run ./packages/gitbook-v2/openNext/customWorkers/script/updateWrangler.ts ${{ steps.extract_server_version_id.outputs.version_id }} + + - id: upload_middleware + name: Upload middleware to Cloudflare uses: cloudflare/wrangler-action@v3.14.0 with: apiToken: ${{ inputs.apiToken }} @@ -82,11 +96,33 @@ runs: workingDirectory: ./ wranglerVersion: '4.10.0' environment: ${{ inputs.environment }} - command: 'deploy --config ./packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc' + command: ${{ format('versions upload --tag {0} --message "{1}"', inputs.commitTag, inputs.commitMessage) }} --config ./packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc + - name: Extract middleware version worker ID + shell: bash + id: extract_middleware_version_id + run: | + version_id=$(echo '${{ steps.upload_middleware.outputs.command-output }}' | grep "Worker Version ID" | awk '{print $4}') + echo "version_id=$version_id" >> $GITHUB_OUTPUT + + - name: Deploy server and middleware to Cloudflare + if: ${{ inputs.deploy == 'true' }} + uses: ./.github/actions/gradual-deploy-cloudflare + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + opServiceAccount: ${{ inputs.opServiceAccount }} + opItem: ${{ inputs.opItem }} + environment: ${{ inputs.environment }} + serverVersionId: ${{ steps.extract_server_version_id.outputs.version_id }} + middlewareVersionId: ${{ steps.extract_middleware_version_id.outputs.version_id }} + deploy: ${{ inputs.deploy }} + + - name: Outputs shell: bash env: - DEPLOYMENT_URL: ${{ steps.deploy.outputs.deployment-url }} + DEPLOYMENT_URL: ${{ steps.upload_middleware.outputs.deployment-url }} run: | - echo "URL: ${{ steps.deploy.outputs.deployment-url }}" \ No newline at end of file + echo "URL: ${{ steps.upload_middleware.outputs.deployment-url }}" + echo "Output server: ${{ steps.upload_server.outputs.command-output }}" \ No newline at end of file diff --git a/packages/gitbook-v2/open-next.config.ts b/packages/gitbook-v2/open-next.config.ts index d35c9ef03..959834739 100644 --- a/packages/gitbook-v2/open-next.config.ts +++ b/packages/gitbook-v2/open-next.config.ts @@ -1,26 +1,29 @@ -import { defineCloudflareConfig } from '@opennextjs/cloudflare'; -import doShardedTagCache from '@opennextjs/cloudflare/overrides/tag-cache/do-sharded-tag-cache'; -import { - softTagFilter, - withFilter, -} from '@opennextjs/cloudflare/overrides/tag-cache/tag-cache-filter'; +import type { OpenNextConfig } from '@opennextjs/cloudflare'; -export default defineCloudflareConfig({ - incrementalCache: () => import('./openNext/incrementalCache').then((m) => m.default), - tagCache: withFilter({ - tagCache: doShardedTagCache({ - baseShardSize: 12, - regionalCache: true, - shardReplication: { - numberOfSoftReplicas: 2, - numberOfHardReplicas: 1, - }, - }), - // We don't use `revalidatePath`, so we filter out soft tags - filterFn: softTagFilter, - }), - queue: () => import('./openNext/queue').then((m) => m.default), - - // Performance improvements as we don't use PPR - enableCacheInterception: true, -}); +export default { + default: { + override: { + wrapper: 'cloudflare-node', + converter: 'edge', + proxyExternalRequest: 'fetch', + queue: () => import('./openNext/queue/server').then((m) => m.default), + incrementalCache: () => import('./openNext/incrementalCache').then((m) => m.default), + tagCache: () => import('./openNext/tagCache/middleware').then((m) => m.default), + }, + }, + middleware: { + external: true, + override: { + wrapper: 'cloudflare-edge', + converter: 'edge', + proxyExternalRequest: 'fetch', + queue: () => import('./openNext/queue/middleware').then((m) => m.default), + incrementalCache: () => import('./openNext/incrementalCache').then((m) => m.default), + tagCache: () => import('./openNext/tagCache/middleware').then((m) => m.default), + }, + }, + dangerous: { + enableCacheInterception: true, + }, + edgeExternals: ['node:crypto'], +} satisfies OpenNextConfig; diff --git a/packages/gitbook-v2/openNext/customWorkers/default.js b/packages/gitbook-v2/openNext/customWorkers/default.js index 6e2f1af82..bb673e172 100644 --- a/packages/gitbook-v2/openNext/customWorkers/default.js +++ b/packages/gitbook-v2/openNext/customWorkers/default.js @@ -1,9 +1,15 @@ +import { runWithCloudflareRequestContext } from '../../.open-next/cloudflare/init.js'; + export default { - async fetch() { - return new Response('Hello World', { - headers: { - 'Content-Type': 'text/plain', - }, + async fetch(request, env, ctx) { + return runWithCloudflareRequestContext(request, env, ctx, async () => { + // We can't move the handler import to the top level, otherwise the runtime will not be properly initialized + const { handler } = await import( + '../../.open-next/server-functions/default/handler.mjs' + ); + + // - `Request`s are handled by the Next server + return handler(request, env, ctx); }); }, }; diff --git a/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc b/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc index 5ade8d025..3fff07d7c 100644 --- a/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc +++ b/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc @@ -17,22 +17,100 @@ "dev": { "vars": { "STAGE": "dev" - } + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-preview" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-server-dev" + } + ] }, "preview": { "vars": { - "STAGE": "preview" - } + "STAGE": "preview", + // Just as a test for the preview environment to check that everything works + "NEXT_PRIVATE_DEBUG_CACHE": "true" + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-preview" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-server-preview" + } + ] + // No durable objects on preview, as they block the generation of preview URLs + // and we don't need tags invalidation on preview }, "staging": { - "vars": { - "STAGE": "staging" - } + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-staging" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-server-staging" + } + ], + "durable_objects": { + "bindings": [ + // We do not need to define migrations for external DOs, + // In fact, defining migrations for external DOs will crash + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache", + "script_name": "gitbook-open-v2-staging" + } + ] + }, + "tail_consumers": [ + { + "service": "gitbook-x-staging-tail" + } + ] }, "production": { - "vars": { - "STAGE": "production" - } + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-production" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-server-production" + } + ], + "durable_objects": { + "bindings": [ + { + // We do not need to define migrations for external DOs, + // In fact, defining migrations for external DOs will crash + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache", + "script_name": "gitbook-open-v2-production" + } + ] + }, + "tail_consumers": [ + { + "service": "gitbook-x-prod-tail" + } + ] } } } diff --git a/packages/gitbook-v2/openNext/customWorkers/middleware.js b/packages/gitbook-v2/openNext/customWorkers/middleware.js new file mode 100644 index 000000000..78a84a976 --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/middleware.js @@ -0,0 +1,42 @@ +import { WorkerEntrypoint } from 'cloudflare:workers'; +import { runWithCloudflareRequestContext } from '../../.open-next/cloudflare/init.js'; + +import { handler as middlewareHandler } from '../../.open-next/middleware/handler.mjs'; + +export { DOQueueHandler } from '../../.open-next/.build/durable-objects/queue.js'; + +export { DOShardedTagCache } from '../../.open-next/.build/durable-objects/sharded-tag-cache.js'; + +export default class extends WorkerEntrypoint { + async fetch(request) { + return runWithCloudflareRequestContext(request, this.env, this.ctx, async () => { + // - `Request`s are handled by the Next server + const reqOrResp = await middlewareHandler(request, this.env, this.ctx); + if (reqOrResp instanceof Response) { + return reqOrResp; + } + + if (this.env.STAGE !== 'preview') { + // https://developers.cloudflare.com/workers/configuration/versions-and-deployments/gradual-deployments/#version-affinity + reqOrResp.headers.set( + 'Cloudflare-Workers-Version-Overrides', + `gitbook-open-v2-${this.env.STAGE}="${this.env.WORKER_VERSION_ID}"` + ); + return this.env.DEFAULT_WORKER?.fetch(reqOrResp, { + cf: { + cacheEverything: false, + }, + }); + } + // If we are in preview mode, we need to send the request to the preview URL + const modifiedUrl = new URL(reqOrResp.url); + modifiedUrl.hostname = this.env.PREVIEW_HOSTNAME; + const nextRequest = new Request(modifiedUrl, reqOrResp); + return fetch(nextRequest, { + cf: { + cacheEverything: false, + }, + }); + }); + } +} diff --git a/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc b/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc new file mode 100644 index 000000000..4870c89d5 --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc @@ -0,0 +1,185 @@ +{ + "main": "middleware.js", + "name": "gitbook-open-v2", + "compatibility_date": "2025-04-14", + "compatibility_flags": [ + "nodejs_compat", + "allow_importable_env", + "global_fetch_strictly_public" + ], + "assets": { + "directory": "../../.open-next/assets", + "binding": "ASSETS" + }, + "observability": { + "enabled": true + }, + "vars": { + "NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true" + }, + "env": { + "dev": { + "vars": { + "STAGE": "dev", + "NEXT_PRIVATE_DEBUG_CACHE": "true" + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-preview" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-dev" + }, + { + "binding": "DEFAULT_WORKER", + "service": "gitbook-open-v2-server-dev" + } + ] + }, + "preview": { + "vars": { + "STAGE": "preview", + "PREVIEW_HOSTNAME": "TO_REPLACE", + "WORKER_VERSION_ID": "TO_REPLACE" + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-preview" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-preview" + }, + { + "binding": "DEFAULT_WORKER", + "service": "gitbook-open-v2-server-preview" + } + ] + // No durable objects on preview, as they block the generation of preview URLs + // and we don't need tags invalidation on preview + }, + "staging": { + "vars": { + "STAGE": "staging", + "WORKER_VERSION_ID": "TO_REPLACE" + }, + "routes": [ + { + "pattern": "open-2c.gitbook-staging.com/*", + "zone_name": "gitbook-staging.com" + }, + { + "pattern": "static-2c.gitbook-staging.com/*", + "zone_name": "gitbook-staging.com" + } + ], + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-staging" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-staging" + }, + { + "binding": "DEFAULT_WORKER", + "service": "gitbook-open-v2-server-staging" + } + ], + "tail_consumers": [ + { + "service": "gitbook-x-staging-tail" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache"] + } + ] + }, + "production": { + "vars": { + // This is a bit misleading, but it means that we can have 500 concurrent revalidations + // This means that we'll have up to 100 durable objects instance running at the same time + "MAX_REVALIDATE_CONCURRENCY": "100", + // Temporary variable to find the issue once deployed + // TODO: remove this once the issue is fixed + "DEBUG_CLOUDFLARE": "true", + "WORKER_VERSION_ID": "TO_REPLACE", + "STAGE": "production" + }, + "routes": [ + { + "pattern": "open-2c.gitbook.com/*", + "zone_name": "gitbook.com" + }, + { + "pattern": "static-2c.gitbook.com/*", + "zone_name": "gitbook.com" + } + ], + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-production" + } + ], + "services": [ + { + "binding": "WORKER_SELF_REFERENCE", + "service": "gitbook-open-v2-production" + }, + { + "binding": "DEFAULT_WORKER", + "service": "gitbook-open-v2-server-production" + } + ], + "tail_consumers": [ + { + "service": "gitbook-x-prod-tail" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache"] + } + ] + } + } +} diff --git a/packages/gitbook-v2/openNext/customWorkers/script/updateWrangler.ts b/packages/gitbook-v2/openNext/customWorkers/script/updateWrangler.ts new file mode 100644 index 000000000..0fdbf6cc7 --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/script/updateWrangler.ts @@ -0,0 +1,26 @@ +// In this script, we use the args from the cli to update the PREVIEW_URL vars in the wrangler config file for the middleware +import fs from 'node:fs'; +import path from 'node:path'; + +const wranglerConfigPath = path.join(__dirname, '../middlewareWrangler.jsonc'); + +const file = fs.readFileSync(wranglerConfigPath, 'utf-8'); + +const args = process.argv.slice(2); +// The versionId is in the format xxx-xxx-xxx-xxx, we need the first part to reconstruct the preview URL +const versionId = args[0]; + +// The preview URL is in the format https://-gitbook-open-v2-server-preview.gitbook.workers.dev +const previewHostname = `${versionId.split('-')[0]}-gitbook-open-v2-server-preview.gitbook.workers.dev`; + +let updatedFile = file.replace( + /"PREVIEW_HOSTNAME": "TO_REPLACE"/, + `"PREVIEW_HOSTNAME": "${previewHostname}"` +); + +updatedFile = updatedFile.replaceAll( + /"WORKER_VERSION_ID": "TO_REPLACE"/g, + `"WORKER_VERSION_ID": "${versionId}"` +); + +fs.writeFileSync(wranglerConfigPath, updatedFile); diff --git a/packages/gitbook-v2/openNext/queue.ts b/packages/gitbook-v2/openNext/queue.ts deleted file mode 100644 index ab33c479d..000000000 --- a/packages/gitbook-v2/openNext/queue.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { Queue } from '@opennextjs/aws/types/overrides.js'; -import { getCloudflareContext } from '@opennextjs/cloudflare'; -import doQueue from '@opennextjs/cloudflare/overrides/queue/do-queue'; -import memoryQueue from '@opennextjs/cloudflare/overrides/queue/memory-queue'; - -interface Env { - IS_PREVIEW?: string; -} - -export default { - name: 'GitbookISRQueue', - send: async (msg) => { - const { ctx, env } = getCloudflareContext(); - const isPreview = (env as Env).IS_PREVIEW === 'true'; - ctx.waitUntil(isPreview ? memoryQueue.send(msg) : doQueue.send(msg)); - }, -} satisfies Queue; diff --git a/packages/gitbook-v2/openNext/queue/middleware.ts b/packages/gitbook-v2/openNext/queue/middleware.ts new file mode 100644 index 000000000..2a14dc1d8 --- /dev/null +++ b/packages/gitbook-v2/openNext/queue/middleware.ts @@ -0,0 +1,21 @@ +import { trace } from '@/lib/tracing'; +import type { Queue } from '@opennextjs/aws/types/overrides.js'; +import { getCloudflareContext } from '@opennextjs/cloudflare'; +import doQueue from '@opennextjs/cloudflare/overrides/queue/do-queue'; +import memoryQueue from '@opennextjs/cloudflare/overrides/queue/memory-queue'; + +interface Env { + STAGE?: string; +} + +export default { + name: 'GitbookISRQueue', + send: async (msg) => { + return trace({ operation: 'gitbookISRQueueSend', name: msg.MessageBody.url }, async () => { + const { ctx, env } = getCloudflareContext(); + const hasDurableObject = + (env as Env).STAGE !== 'dev' && (env as Env).STAGE !== 'preview'; + ctx.waitUntil(hasDurableObject ? memoryQueue.send(msg) : doQueue.send(msg)); + }); + }, +} satisfies Queue; diff --git a/packages/gitbook-v2/openNext/queue/server.ts b/packages/gitbook-v2/openNext/queue/server.ts new file mode 100644 index 000000000..9a5b3b689 --- /dev/null +++ b/packages/gitbook-v2/openNext/queue/server.ts @@ -0,0 +1,9 @@ +import type { Queue } from '@opennextjs/aws/types/overrides.js'; + +export default { + name: 'GitbookISRQueue', + send: async (msg) => { + // We should never reach this point in the server. If that's the case, we should log it. + console.warn('GitbookISRQueue: send called on server side, this should not happen.', msg); + }, +} satisfies Queue; diff --git a/packages/gitbook-v2/openNext/tagCache/middleware.ts b/packages/gitbook-v2/openNext/tagCache/middleware.ts new file mode 100644 index 000000000..173b5dd28 --- /dev/null +++ b/packages/gitbook-v2/openNext/tagCache/middleware.ts @@ -0,0 +1,76 @@ +import { trace } from '@/lib/tracing'; +import type { NextModeTagCache } from '@opennextjs/aws/types/overrides.js'; +import doShardedTagCache from '@opennextjs/cloudflare/overrides/tag-cache/do-sharded-tag-cache'; +import { softTagFilter } from '@opennextjs/cloudflare/overrides/tag-cache/tag-cache-filter'; + +const originalTagCache = doShardedTagCache({ + baseShardSize: 12, + regionalCache: true, + shardReplication: { + numberOfSoftReplicas: 2, + numberOfHardReplicas: 1, + }, +}); + +export default { + name: 'GitbookTagCache', + mode: 'nextMode', + getLastRevalidated: async (tags: string[]) => { + const tagsToCheck = tags.filter(softTagFilter); + if (tagsToCheck.length === 0) { + // If we reach here, it probably means that there is an issue that we'll need to address. + console.warn( + 'getLastRevalidated - No valid tags to check for last revalidation, original tags:', + tags + ); + return 0; // If no tags to check, return 0 + } + return trace( + { + operation: 'gitbookTagCacheGetLastRevalidated', + name: tagsToCheck.join(', '), + }, + async () => { + return await originalTagCache.getLastRevalidated(tagsToCheck); + } + ); + }, + hasBeenRevalidated: async (tags: string[]) => { + const tagsToCheck = tags.filter(softTagFilter); + if (tagsToCheck.length === 0) { + // If we reach here, it probably means that there is an issue that we'll need to address. + console.warn( + 'hasBeenRevalidated - No valid tags to check for revalidation, original tags:', + tags + ); + return false; // If no tags to check, return false + } + return trace( + { + operation: 'gitbookTagCacheHasBeenRevalidated', + name: tagsToCheck.join(', '), + }, + async () => { + const result = await originalTagCache.hasBeenRevalidated(tagsToCheck); + return result; + } + ); + }, + writeTags: async (tags: string[]) => { + return trace( + { + operation: 'gitbookTagCacheWriteTags', + name: tags.join(', '), + }, + async () => { + const tagsToWrite = tags.filter(softTagFilter); + if (tagsToWrite.length === 0) { + console.warn('writeTags - No valid tags to write'); + return; // If no tags to write, exit early + } + // Write only the filtered tags + await originalTagCache.writeTags(tagsToWrite); + } + ); + }, +} satisfies NextModeTagCache; diff --git a/packages/gitbook-v2/package.json b/packages/gitbook-v2/package.json index 5561b7f8d..224d55e30 100644 --- a/packages/gitbook-v2/package.json +++ b/packages/gitbook-v2/package.json @@ -30,6 +30,8 @@ "start": "next start", "build:v2:cloudflare": "opennextjs-cloudflare build", "dev:v2:cloudflare": "wrangler dev --port 8771 --env preview", + "dev:v2:cf:middleware": "wrangler dev --port 8771 --inspector-port 9230 --env dev --config ./openNext/customWorkers/middlewareWrangler.jsonc", + "dev:v2:cf:server": "wrangler dev --port 8772 --env dev --config ./openNext/customWorkers/defaultWrangler.jsonc", "unit": "bun test", "typecheck": "tsc --noEmit" } diff --git a/packages/gitbook-v2/src/lib/data/api.ts b/packages/gitbook-v2/src/lib/data/api.ts index 7a2d03d16..f8ae6d271 100644 --- a/packages/gitbook-v2/src/lib/data/api.ts +++ b/packages/gitbook-v2/src/lib/data/api.ts @@ -2,14 +2,13 @@ import { trace } from '@/lib/tracing'; import { type ComputedContentSource, GitBookAPI, - type GitBookAPIServiceBinding, type HttpResponse, type RenderIntegrationUI, } from '@gitbook/api'; import { getCacheTag, getComputedContentSourceCacheTags } from '@gitbook/cache-tags'; import { GITBOOK_API_TOKEN, GITBOOK_API_URL, GITBOOK_USER_AGENT } from '@v2/lib/env'; import { unstable_cacheLife as cacheLife, unstable_cacheTag as cacheTag } from 'next/cache'; -import { getCloudflareContext, getCloudflareRequestGlobal } from './cloudflare'; +import { getCloudflareRequestGlobal } from './cloudflare'; import { DataFetcherError, wrapDataFetcherError } from './errors'; import { withCacheKey, withoutConcurrentExecution } from './memoize'; import type { GitBookDataFetcher } from './types'; @@ -828,36 +827,16 @@ async function* streamAIResponse( } } -let loggedServiceBinding = false; - /** * Create a new API client. */ export function apiClient(input: DataFetcherInput = { apiToken: null }) { const { apiToken } = input; - let serviceBinding: GitBookAPIServiceBinding | undefined; - - const cloudflareContext = getCloudflareContext(); - if (cloudflareContext) { - // @ts-expect-error - serviceBinding = cloudflareContext.env.GITBOOK_API as GitBookAPIServiceBinding | undefined; - if (!loggedServiceBinding) { - loggedServiceBinding = true; - if (serviceBinding) { - // biome-ignore lint/suspicious/noConsole: we want to log here - console.log(`using service binding for the API (${GITBOOK_API_URL})`); - } else { - // biome-ignore lint/suspicious/noConsole: we want to log here - console.warn(`no service binding for the API (${GITBOOK_API_URL})`); - } - } - } const api = new GitBookAPI({ authToken: apiToken || GITBOOK_API_TOKEN || undefined, endpoint: GITBOOK_API_URL, userAgent: GITBOOK_USER_AGENT, - serviceBinding, }); return api; From a1da4f023d7d491dde259f526c66d922d5015645 Mon Sep 17 00:00:00 2001 From: conico974 Date: Fri, 30 May 2025 16:19:11 +0200 Subject: [PATCH 15/15] Move DO to a 3rd worker (#3270) Co-authored-by: Nicolas Dorseuil --- .../composite/deploy-cloudflare/action.yaml | 10 ++ .../openNext/customWorkers/default.js | 21 +++ .../customWorkers/defaultWrangler.jsonc | 65 +++++++-- .../gitbook-v2/openNext/customWorkers/do.js | 38 ++++++ .../openNext/customWorkers/doWrangler.jsonc | 127 ++++++++++++++++++ .../customWorkers/middlewareWrangler.jsonc | 49 +++++-- .../gitbook-v2/openNext/incrementalCache.ts | 22 ++- .../openNext/tagCache/middleware.ts | 6 +- 8 files changed, 315 insertions(+), 23 deletions(-) create mode 100644 packages/gitbook-v2/openNext/customWorkers/do.js create mode 100644 packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc diff --git a/.github/composite/deploy-cloudflare/action.yaml b/.github/composite/deploy-cloudflare/action.yaml index d1d0d6633..7e66a8bf3 100644 --- a/.github/composite/deploy-cloudflare/action.yaml +++ b/.github/composite/deploy-cloudflare/action.yaml @@ -64,6 +64,16 @@ runs: GITBOOK_RUNTIME: cloudflare shell: bash + - name: Upload the DO worker + uses: cloudflare/wrangler-action@v3.14.0 + with: + apiToken: ${{ inputs.apiToken }} + accountId: ${{ inputs.accountId }} + workingDirectory: ./ + wranglerVersion: '4.10.0' + environment: ${{ inputs.environment }} + command: deploy --config ./packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc + - id: upload_server name: Upload server to Cloudflare uses: cloudflare/wrangler-action@v3.14.0 diff --git a/packages/gitbook-v2/openNext/customWorkers/default.js b/packages/gitbook-v2/openNext/customWorkers/default.js index bb673e172..535c21816 100644 --- a/packages/gitbook-v2/openNext/customWorkers/default.js +++ b/packages/gitbook-v2/openNext/customWorkers/default.js @@ -1,5 +1,26 @@ import { runWithCloudflareRequestContext } from '../../.open-next/cloudflare/init.js'; +import { DurableObject } from 'cloudflare:workers'; + +// Only needed to run locally, in prod we'll use the one from do.js +export class R2WriteBuffer extends DurableObject { + writePromise; + + async write(cacheKey, value) { + // We are already writing to this key + if (this.writePromise) { + return; + } + + this.writePromise = this.env.NEXT_INC_CACHE_R2_BUCKET.put(cacheKey, value); + this.ctx.waitUntil( + this.writePromise.finally(() => { + this.writePromise = undefined; + }) + ); + } +} + export default { async fetch(request, env, ctx) { return runWithCloudflareRequestContext(request, env, ctx, async () => { diff --git a/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc b/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc index 3fff07d7c..e036db1c4 100644 --- a/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc +++ b/packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc @@ -29,6 +29,20 @@ "binding": "WORKER_SELF_REFERENCE", "service": "gitbook-open-v2-server-dev" } + ], + "durable_objects": { + "bindings": [ + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["R2WriteBuffer"] + } ] }, "preview": { @@ -48,9 +62,26 @@ "binding": "WORKER_SELF_REFERENCE", "service": "gitbook-open-v2-server-preview" } - ] - // No durable objects on preview, as they block the generation of preview URLs - // and we don't need tags invalidation on preview + ], + "durable_objects": { + "bindings": [ + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer", + "script_name": "gitbook-open-v2-do-preview" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache", + "script_name": "gitbook-open-v2-do-preview" + }, + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler", + "script_name": "gitbook-open-v2-do-preview" + } + ] + } }, "staging": { "r2_buckets": [ @@ -67,12 +98,20 @@ ], "durable_objects": { "bindings": [ - // We do not need to define migrations for external DOs, - // In fact, defining migrations for external DOs will crash + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer", + "script_name": "gitbook-open-v2-do-staging" + }, { "name": "NEXT_TAG_CACHE_DO_SHARDED", "class_name": "DOShardedTagCache", - "script_name": "gitbook-open-v2-staging" + "script_name": "gitbook-open-v2-do-staging" + }, + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler", + "script_name": "gitbook-open-v2-do-staging" } ] }, @@ -98,11 +137,19 @@ "durable_objects": { "bindings": [ { - // We do not need to define migrations for external DOs, - // In fact, defining migrations for external DOs will crash + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer", + "script_name": "gitbook-open-v2-do-production" + }, + { "name": "NEXT_TAG_CACHE_DO_SHARDED", "class_name": "DOShardedTagCache", - "script_name": "gitbook-open-v2-production" + "script_name": "gitbook-open-v2-do-production" + }, + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler", + "script_name": "gitbook-open-v2-do-production" } ] }, diff --git a/packages/gitbook-v2/openNext/customWorkers/do.js b/packages/gitbook-v2/openNext/customWorkers/do.js new file mode 100644 index 000000000..04f3cf3be --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/do.js @@ -0,0 +1,38 @@ +// This worker only purposes it to host the different DO that we will need in the other workers. +import { DurableObject } from 'cloudflare:workers'; + +// `use cache` could cause multiple writes to the same key to happen concurrently, there is a limit of 1 write per key/second +// so we need to buffer writes to the R2 bucket to avoid hitting this limit. +export class R2WriteBuffer extends DurableObject { + writePromise; + + async write(cacheKey, value) { + // We are already writing to this key + if (this.writePromise) { + return; + } + + this.writePromise = this.env.NEXT_INC_CACHE_R2_BUCKET.put(cacheKey, value); + this.ctx.waitUntil( + this.writePromise.finally(() => { + this.writePromise = undefined; + }) + ); + } +} + +export { DOQueueHandler } from '../../.open-next/.build/durable-objects/queue.js'; + +export { DOShardedTagCache } from '../../.open-next/.build/durable-objects/sharded-tag-cache.js'; + +export default { + async fetch() { + // This worker does not handle any requests, it only provides Durable Objects + return new Response('This worker is not meant to handle requests directly', { + status: 400, + headers: { + 'Content-Type': 'text/plain', + }, + }); + }, +}; diff --git a/packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc b/packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc new file mode 100644 index 000000000..f5d2f726b --- /dev/null +++ b/packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc @@ -0,0 +1,127 @@ +{ + "main": "do.js", + "name": "gitbook-open-v2-do", + "compatibility_date": "2025-04-14", + "compatibility_flags": [ + "nodejs_compat", + "allow_importable_env", + "global_fetch_strictly_public" + ], + "observability": { + "enabled": true + }, + "env": { + "preview": { + "vars": { + "STAGE": "preview", + "NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true" + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-preview" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache" + }, + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache", "R2WriteBuffer"] + } + ] + }, + "staging": { + "vars": { + "STAGE": "staging", + "NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true" + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-staging" + } + ], + "tail_consumers": [ + { + "service": "gitbook-x-staging-tail" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache" + }, + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache", "R2WriteBuffer"] + } + ] + }, + "production": { + "vars": { + "NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true", + "STAGE": "production" + }, + "r2_buckets": [ + { + "binding": "NEXT_INC_CACHE_R2_BUCKET", + "bucket_name": "gitbook-open-v2-cache-production" + } + ], + "tail_consumers": [ + { + "service": "gitbook-x-prod-tail" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache" + }, + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache", "R2WriteBuffer"] + } + ] + } + } +} diff --git a/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc b/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc index 4870c89d5..09e48afcc 100644 --- a/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc +++ b/packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc @@ -61,9 +61,26 @@ "binding": "DEFAULT_WORKER", "service": "gitbook-open-v2-server-preview" } - ] - // No durable objects on preview, as they block the generation of preview URLs - // and we don't need tags invalidation on preview + ], + "durable_objects": { + "bindings": [ + { + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer", + "script_name": "gitbook-open-v2-do-preview" + }, + { + "name": "NEXT_TAG_CACHE_DO_SHARDED", + "class_name": "DOShardedTagCache", + "script_name": "gitbook-open-v2-do-preview" + }, + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler", + "script_name": "gitbook-open-v2-do-preview" + } + ] + } }, "staging": { "vars": { @@ -104,12 +121,19 @@ "durable_objects": { "bindings": [ { - "name": "NEXT_CACHE_DO_QUEUE", - "class_name": "DOQueueHandler" + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer", + "script_name": "gitbook-open-v2-do-staging" }, { "name": "NEXT_TAG_CACHE_DO_SHARDED", - "class_name": "DOShardedTagCache" + "class_name": "DOShardedTagCache", + "script_name": "gitbook-open-v2-do-staging" + }, + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler", + "script_name": "gitbook-open-v2-do-staging" } ] }, @@ -165,12 +189,19 @@ "durable_objects": { "bindings": [ { - "name": "NEXT_CACHE_DO_QUEUE", - "class_name": "DOQueueHandler" + "name": "WRITE_BUFFER", + "class_name": "R2WriteBuffer", + "script_name": "gitbook-open-v2-do-production" }, { "name": "NEXT_TAG_CACHE_DO_SHARDED", - "class_name": "DOShardedTagCache" + "class_name": "DOShardedTagCache", + "script_name": "gitbook-open-v2-do-production" + }, + { + "name": "NEXT_CACHE_DO_QUEUE", + "class_name": "DOQueueHandler", + "script_name": "gitbook-open-v2-do-production" } ] }, diff --git a/packages/gitbook-v2/openNext/incrementalCache.ts b/packages/gitbook-v2/openNext/incrementalCache.ts index d4a662af0..28d1d6b8a 100644 --- a/packages/gitbook-v2/openNext/incrementalCache.ts +++ b/packages/gitbook-v2/openNext/incrementalCache.ts @@ -9,6 +9,8 @@ import type { } from '@opennextjs/aws/types/overrides.js'; import { getCloudflareContext } from '@opennextjs/cloudflare'; +import type { DurableObjectNamespace, Rpc } from '@cloudflare/workers-types'; + export const BINDING_NAME = 'NEXT_INC_CACHE_R2_BUCKET'; export const DEFAULT_PREFIX = 'incremental-cache'; @@ -79,12 +81,10 @@ class GitbookIncrementalCache implements IncrementalCache { }, async (span) => { span.setAttribute('cacheType', cacheType ?? 'cache'); - const r2 = getCloudflareContext().env[BINDING_NAME]; const localCache = await this.getCacheInstance(); - if (!r2) throw new Error('No R2 bucket'); try { - await r2.put(cacheKey, JSON.stringify(value)); + await this.writeToR2(cacheKey, JSON.stringify(value)); //TODO: Check if there is any places where we don't have tags // Ideally we should always have tags, but in case we don't, we need to decide how to handle it @@ -145,6 +145,22 @@ class GitbookIncrementalCache implements IncrementalCache { ); } + async writeToR2(key: string, value: string): Promise { + const env = getCloudflareContext().env as { + WRITE_BUFFER: DurableObjectNamespace< + Rpc.DurableObjectBranded & { + write: (key: string, value: string) => Promise; + } + >; + }; + const id = env.WRITE_BUFFER.idFromName(key); + + // A stub is a client used to invoke methods on the Durable Object + const stub = env.WRITE_BUFFER.get(id); + + await stub.write(key, value); + } + async getCacheInstance(): Promise { if (this.localCache) return this.localCache; this.localCache = await caches.open('incremental-cache'); diff --git a/packages/gitbook-v2/openNext/tagCache/middleware.ts b/packages/gitbook-v2/openNext/tagCache/middleware.ts index 173b5dd28..398ceee0d 100644 --- a/packages/gitbook-v2/openNext/tagCache/middleware.ts +++ b/packages/gitbook-v2/openNext/tagCache/middleware.ts @@ -6,6 +6,8 @@ import { softTagFilter } from '@opennextjs/cloudflare/overrides/tag-cache/tag-ca const originalTagCache = doShardedTagCache({ baseShardSize: 12, regionalCache: true, + // We can probably increase this value even further + regionalCacheTtlSec: 60, shardReplication: { numberOfSoftReplicas: 2, numberOfHardReplicas: 1, @@ -35,7 +37,7 @@ export default { } ); }, - hasBeenRevalidated: async (tags: string[]) => { + hasBeenRevalidated: async (tags: string[], lastModified?: number) => { const tagsToCheck = tags.filter(softTagFilter); if (tagsToCheck.length === 0) { // If we reach here, it probably means that there is an issue that we'll need to address. @@ -51,7 +53,7 @@ export default { name: tagsToCheck.join(', '), }, async () => { - const result = await originalTagCache.hasBeenRevalidated(tagsToCheck); + const result = await originalTagCache.hasBeenRevalidated(tagsToCheck, lastModified); return result; } );