From 50653abd08ae6467e7643e7cb5caaa20e39d60a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samy=20Pess=C3=A9?= Date: Sat, 4 Apr 2026 12:35:43 +0200 Subject: [PATCH] Improve AI agent detection and readability by outputting 200 for page not found (#4164) --- .changeset/jolly-apples-search.md | 5 + .changeset/olive-mugs-invite.md | 5 + bun.lock | 15 +- packages/gitbook/package.json | 3 +- packages/gitbook/src/lib/pages.test.ts | 72 ++++++++++ packages/gitbook/src/lib/pages.ts | 152 ++++++++++++++++++++ packages/gitbook/src/middleware.ts | 26 ++-- packages/gitbook/src/routes/markdownPage.ts | 51 +++++-- packages/gitbook/tests/llms.test.ts | 20 +++ packages/gitbook/tests/markdown.test.ts | 11 ++ 10 files changed, 329 insertions(+), 31 deletions(-) create mode 100644 .changeset/jolly-apples-search.md create mode 100644 .changeset/olive-mugs-invite.md diff --git a/.changeset/jolly-apples-search.md b/.changeset/jolly-apples-search.md new file mode 100644 index 000000000..9db0b2347 --- /dev/null +++ b/.changeset/jolly-apples-search.md @@ -0,0 +1,5 @@ +--- +"gitbook": minor +--- + +Redirect sitemap.md and .well-known/sitemap.md to llms.txt. diff --git a/.changeset/olive-mugs-invite.md b/.changeset/olive-mugs-invite.md new file mode 100644 index 000000000..579f27cfe --- /dev/null +++ b/.changeset/olive-mugs-invite.md @@ -0,0 +1,5 @@ +--- +"gitbook": minor +--- + +Improve AI agent detection and readability by outputting 200 for page not found. diff --git a/bun.lock b/bun.lock index 1bb4d7e0f..6d773d3e1 100644 --- a/bun.lock +++ b/bun.lock @@ -7,7 +7,7 @@ "devDependencies": { "@biomejs/biome": "^1.9.4", "@changesets/cli": "^2.30.0", - "turbo": "^2.9.1", + "turbo": "^2.9.2", "vercel": "50.37.3", }, }, @@ -139,6 +139,7 @@ "@sindresorhus/fnv1a": "^3.1.0", "@tailwindcss/container-queries": "^0.1.1", "@tusbar/cache-control": "^1.0.2", + "@vercel/agent-readability": "^0.2.1", "ai": "^4.2.2", "assert-never": "catalog:", "bidc": "catalog:", @@ -150,6 +151,7 @@ "js-cookie": "^3.0.5", "jsontoxml": "^1.0.1", "jwt-decode": "^4.0.0", + "leven": "^4.1.0", "mcp-handler": "^1.0.2", "mdast-util-from-markdown": "^2.0.2", "mdast-util-frontmatter": "^2.0.1", @@ -160,7 +162,6 @@ "micromark-extension-frontmatter": "^2.0.0", "micromark-extension-gfm": "^3.0.0", "motion": "^12.23.24", - "negotiator": "^1.0.0", "next": "15.4.11", "next-themes": "^0.4.6", "nuqs": "^2.2.3", @@ -1718,6 +1719,8 @@ "@ungap/structured-clone": ["@ungap/structured-clone@1.2.0", "", {}, "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="], + "@vercel/agent-readability": ["@vercel/agent-readability@0.2.1", "", { "peerDependencies": { "next": ">=14" }, "optionalPeers": ["next"], "bin": { "agent-readability": "dist/cli/index.cjs" } }, "sha512-ShT7BzIS/dwKompii8tm5do+NR1g4xL5M3wM7S01xsH6yuYQ7wiTPZEcmHMFLHCsAQg45/mD0hgrufpS3NVunw=="], + "@vercel/backends": ["@vercel/backends@0.0.53", "", { "dependencies": { "@vercel/build-utils": "13.12.0", "@vercel/nft": "1.5.0", "execa": "3.2.0", "fs-extra": "11.1.0", "oxc-transform": "0.111.0", "path-to-regexp": "8.3.0", "resolve.exports": "2.0.3", "rolldown": "1.0.0-rc.1", "srvx": "0.8.9", "tsx": "4.21.0", "zod": "3.22.4" }, "peerDependencies": { "typescript": "^4.0.0 || ^5.0.0" } }, "sha512-f8JK9Jatlco6kT/NsLYVR88iqS7oZxUXzXW8TofLmyTT5o+OEXSY2hzAQSjul6r0qz/9yAhtQkMhcHG8Ne01ZQ=="], "@vercel/blob": ["@vercel/blob@2.3.0", "", { "dependencies": { "async-retry": "^1.3.3", "is-buffer": "^2.0.5", "is-node-process": "^1.2.0", "throttleit": "^2.1.0", "undici": "^6.23.0" } }, "sha512-oYWiJbWRQ7gz9Mj0X/NHFJ3OcLMOBzq/2b3j6zeNrQmtFo6dHwU8FAwNpxVIYddVMd+g8eqEi7iRueYx8FtM0Q=="], @@ -2612,7 +2615,7 @@ "layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="], - "leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="], + "leven": ["leven@4.1.0", "", {}, "sha512-KZ9W9nWDT7rF7Dazg8xyLHGLrmpgq2nVNFUckhqdW3szVP6YhCpp/RAnpmVExA9JvrMynjwSLVrEj3AepHR6ew=="], "lightningcss": ["lightningcss@1.30.1", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-darwin-arm64": "1.30.1", "lightningcss-darwin-x64": "1.30.1", "lightningcss-freebsd-x64": "1.30.1", "lightningcss-linux-arm-gnueabihf": "1.30.1", "lightningcss-linux-arm64-gnu": "1.30.1", "lightningcss-linux-arm64-musl": "1.30.1", "lightningcss-linux-x64-gnu": "1.30.1", "lightningcss-linux-x64-musl": "1.30.1", "lightningcss-win32-arm64-msvc": "1.30.1", "lightningcss-win32-x64-msvc": "1.30.1" } }, "sha512-xi6IyHML+c9+Q3W0S4fCQJOym42pyurFiJUHEcEyHS0CeKzia4yZDEsLlqOFykxOdHpNy0NmvVO31vcSqAxJCg=="], @@ -4360,6 +4363,8 @@ "@scalar/object-utils/@scalar/helpers": ["@scalar/helpers@0.1.1", "", {}, "sha512-eJjuCI5djqU0adTwrHvpDf0xuwNRpwZinCfJ03QjnmIFBM9Ch0u3tn/0EZOhcVNbdEyJ+3yvSVy1dCmmvgtpvQ=="], + "@scalar/openapi-parser/leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="], + "@scalar/openapi-types/zod": ["zod@4.1.11", "", {}, "sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg=="], "@scalar/postman-to-openapi/@scalar/helpers": ["@scalar/helpers@0.0.12", "", {}, "sha512-4NDmHShyi1hrVRsJCdRZT/FIpy+/5PFbVbQLRYX/pjpu5cYqHBj9s6n5RI6gGDXEBHAIFi63g9FC6Isgr66l1Q=="], @@ -5418,6 +5423,8 @@ "@scalar/api-client/@scalar/openapi-parser/@scalar/openapi-upgrader": ["@scalar/openapi-upgrader@0.1.3", "", { "dependencies": { "@scalar/openapi-types": "0.5.0" } }, "sha512-iROhcgy3vge6zsviPtoTLHale0nYt1PLhuMmJweQwJ0U23ZYyYhV5xgHtAd0OBEXuqT6rjYbJFvKOJZmJOwpNQ=="], + "@scalar/api-client/@scalar/openapi-parser/leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="], + "@scalar/api-client/@scalar/workspace-store/@scalar/openapi-upgrader": ["@scalar/openapi-upgrader@0.1.3", "", { "dependencies": { "@scalar/openapi-types": "0.5.0" } }, "sha512-iROhcgy3vge6zsviPtoTLHale0nYt1PLhuMmJweQwJ0U23ZYyYhV5xgHtAd0OBEXuqT6rjYbJFvKOJZmJOwpNQ=="], "@scalar/components/@scalar/oas-utils/@scalar/json-magic": ["@scalar/json-magic@0.6.1", "", { "dependencies": { "@scalar/helpers": "0.0.12", "yaml": "2.8.0" } }, "sha512-HJMPY5dUU3EXVS4EkjAFXo+uCrby/YFu/gljKDQnhYWRy5zQ0sJWrOEDcHS8nLoJRCIRD5tiVpCxnUnSb6OoAQ=="], @@ -5452,6 +5459,8 @@ "@scalar/import/@scalar/openapi-parser/@scalar/openapi-upgrader": ["@scalar/openapi-upgrader@0.1.3", "", { "dependencies": { "@scalar/openapi-types": "0.5.0" } }, "sha512-iROhcgy3vge6zsviPtoTLHale0nYt1PLhuMmJweQwJ0U23ZYyYhV5xgHtAd0OBEXuqT6rjYbJFvKOJZmJOwpNQ=="], + "@scalar/import/@scalar/openapi-parser/leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="], + "@scalar/postman-to-openapi/@scalar/oas-utils/@scalar/json-magic": ["@scalar/json-magic@0.6.1", "", { "dependencies": { "@scalar/helpers": "0.0.12", "yaml": "2.8.0" } }, "sha512-HJMPY5dUU3EXVS4EkjAFXo+uCrby/YFu/gljKDQnhYWRy5zQ0sJWrOEDcHS8nLoJRCIRD5tiVpCxnUnSb6OoAQ=="], "@scalar/postman-to-openapi/@scalar/oas-utils/@scalar/object-utils": ["@scalar/object-utils@1.2.8", "", { "dependencies": { "@scalar/helpers": "0.0.12", "flatted": "^3.3.3", "just-clone": "^6.2.0", "ts-deepmerge": "^7.0.1", "type-fest": "5.0.0" } }, "sha512-FjPxEg7Hw0tz2iTKvi+gYt+luZK0TqhX50hUIBuaYa/Ja/OMuKLp9QHhB5U68F1L55CZNP4vwoNNfXeYWnVEZg=="], diff --git a/packages/gitbook/package.json b/packages/gitbook/package.json index 92575c3fd..faa8bf2dd 100644 --- a/packages/gitbook/package.json +++ b/packages/gitbook/package.json @@ -31,6 +31,7 @@ "@sindresorhus/fnv1a": "^3.1.0", "@tailwindcss/container-queries": "^0.1.1", "@tusbar/cache-control": "^1.0.2", + "@vercel/agent-readability": "^0.2.1", "ai": "^4.2.2", "assert-never": "catalog:", "bidc": "catalog:", @@ -52,7 +53,7 @@ "micromark-extension-frontmatter": "^2.0.0", "micromark-extension-gfm": "^3.0.0", "motion": "^12.23.24", - "negotiator": "^1.0.0", + "leven": "^4.1.0", "next": "15.4.11", "next-themes": "^0.4.6", "nuqs": "^2.2.3", diff --git a/packages/gitbook/src/lib/pages.test.ts b/packages/gitbook/src/lib/pages.test.ts index ae9f955c5..18c9a94b5 100644 --- a/packages/gitbook/src/lib/pages.test.ts +++ b/packages/gitbook/src/lib/pages.test.ts @@ -7,6 +7,7 @@ import { import { extractPagePath, + getSimilarPages, resolveFirstDocument, resolvePagePath, resolvePagePathDocumentOrGroup, @@ -292,3 +293,74 @@ describe('resolvePagePath', () => { }); }); }); + +describe('getSimilarPages', () => { + it('returns the closest matches for a typoed path', () => { + const pages: RevisionPage[] = [ + createDocumentPage('install', 'guides/installation'), + createDocumentPage('config', 'guides/configuration'), + createDocumentPage('auth', 'api/authentication'), + ]; + + expect(getSimilarPages(pages, 'guides/installtion', 2).map((page) => page.id)).toEqual([ + 'install', + 'config', + ]); + }); + + it('prefers pages with the same path hierarchy', () => { + const pages: RevisionPage[] = [ + createDocumentPage('api-auth', 'api/reference/authentication'), + createDocumentPage('guide-auth', 'guides/authentication'), + createDocumentPage('api-errors', 'api/reference/errors'), + ]; + + expect( + getSimilarPages(pages, 'api/reference/authentcation', 1).map((page) => page.id) + ).toEqual(['api-auth']); + }); + + it('ignores hidden pages', () => { + const pages: RevisionPage[] = [ + createDocumentPage('hidden', 'private-api', true), + createDocumentPage('versioned', 'private-api-v2'), + createDocumentPage('public', 'public-api'), + ]; + + const similar = getSimilarPages(pages, 'privte-api', 2); + + expect(similar.map((page) => page.id)).toEqual(['versioned', 'public']); + expect(similar.some((page) => page.id === 'hidden')).toBe(false); + }); +}); + +function createDocumentPage(id: string, path: string, hidden = false): RevisionPage { + const slug = path.split('/').at(-1) ?? path; + + return { + id, + title: path, + kind: 'sheet', + type: 'document', + hidden, + urls: { + app: `https://app.gitbook.com/s/fvBF1lEt2CVd4RTffSOk/${path}`, + }, + path, + slug, + pages: [], + tags: [], + layout: { + cover: true, + coverSize: RevisionPageLayoutOptionsCoverSize.Full, + title: true, + description: true, + tableOfContents: true, + outline: true, + pagination: true, + width: RevisionPageLayoutOptionsWidth.Default, + metadata: true, + tags: true, + }, + }; +} diff --git a/packages/gitbook/src/lib/pages.ts b/packages/gitbook/src/lib/pages.ts index 5318d92a0..08ab395b8 100644 --- a/packages/gitbook/src/lib/pages.ts +++ b/packages/gitbook/src/lib/pages.ts @@ -5,6 +5,7 @@ import { type RevisionPageGroup, RevisionPageType, } from '@gitbook/api'; +import leven from 'leven'; import { removeLeadingSlash, removeTrailingSlash } from './paths'; export type AncestorRevisionPage = RevisionPageDocument | RevisionPageGroup; @@ -208,6 +209,157 @@ export function resolveFirstDocument( return; } +/** + * Get a similar pages to a given path. This is used to suggest similar pages when a page is not found. + */ +export function getSimilarPages( + rootPages: RevisionPage[], + pagePath: string, + topK = 5 +): RevisionPageDocument[] { + if (topK <= 0) { + return []; + } + + const pages = flattenPages(rootPages, (page) => !page.hidden); + if (pages.length === 0) { + return []; + } + + const inputPath = createComparablePagePath(pagePath); + if (!inputPath.path) { + return pages.slice(0, topK); + } + + const firstPageId = resolveFirstDocument(rootPages, [])?.page.id; + + return pages + .map((page, index) => { + const candidatePaths = new Set([page.path]); + if (page.id === firstPageId) { + candidatePaths.add(''); + } + + let score = 0; + for (const candidatePath of candidatePaths) { + score = Math.max( + score, + scoreComparablePagePaths(inputPath, createComparablePagePath(candidatePath)) + ); + } + + return { page, index, score }; + }) + .sort((left, right) => right.score - left.score || left.index - right.index) + .slice(0, topK) + .map(({ page }) => page); +} + +type ComparablePagePath = { + path: string; + segments: string[]; + tokens: string[]; + lastSegment: string; +}; + +function createComparablePagePath(path: string): ComparablePagePath { + const normalizedPath = removeTrailingSlash(removeLeadingSlash(path.trim().toLowerCase())) + .split('/') + .filter(Boolean) + .join('/'); + const segments = normalizedPath ? normalizedPath.split('/') : []; + const tokens = segments.flatMap((segment) => segment.split(/[-_\s]+/g)).filter(Boolean); + + return { + path: normalizedPath, + segments, + tokens, + lastSegment: segments.at(-1) ?? normalizedPath, + }; +} + +function scoreComparablePagePaths( + inputPath: ComparablePagePath, + candidatePath: ComparablePagePath +): number { + const fullPathSimilarity = getLevenshteinSimilarity(inputPath.path, candidatePath.path); + const segmentSimilarity = getAlignedSegmentSimilarity( + inputPath.segments, + candidatePath.segments + ); + const lastSegmentSimilarity = getLevenshteinSimilarity( + inputPath.lastSegment, + candidatePath.lastSegment + ); + const tokenOverlap = getJaccardSimilarity(inputPath.tokens, candidatePath.tokens); + + let score = + fullPathSimilarity * 0.4 + + segmentSimilarity * 0.3 + + lastSegmentSimilarity * 0.2 + + tokenOverlap * 0.1; + + if ( + candidatePath.path && + (inputPath.path.startsWith(`${candidatePath.path}/`) || + candidatePath.path.startsWith(`${inputPath.path}/`)) + ) { + score += 0.05; + } + + if (inputPath.lastSegment && inputPath.lastSegment === candidatePath.lastSegment) { + score += 0.1; + } + + return score; +} + +function getAlignedSegmentSimilarity(left: string[], right: string[]): number { + const length = Math.max(left.length, right.length); + if (length === 0) { + return 1; + } + + let score = 0; + for (let index = 0; index < length; index += 1) { + score += getLevenshteinSimilarity(left[index] ?? '', right[index] ?? ''); + } + + return score / length; +} + +function getJaccardSimilarity(left: string[], right: string[]): number { + const leftSet = new Set(left); + const rightSet = new Set(right); + const union = new Set([...leftSet, ...rightSet]); + + if (union.size === 0) { + return 1; + } + + let intersection = 0; + for (const token of leftSet) { + if (rightSet.has(token)) { + intersection += 1; + } + } + + return intersection / union.size; +} + +function getLevenshteinSimilarity(left: string, right: string): number { + if (left === right) { + return 1; + } + + const maxLength = Math.max(left.length, right.length); + if (maxLength === 0) { + return 1; + } + + return 1 - leven(left, right) / maxLength; +} + function resolvePageDocument( page: RevisionPage, ancestors: AncestorRevisionPage[] diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index 28dbeeb8d..eaefc8869 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -3,7 +3,7 @@ import { SiteInsightsDisplayContext, SiteInsightsLLMSVariant, } from '@gitbook/api'; -import Negotiator from 'negotiator'; +import { shouldServeMarkdown } from '@vercel/agent-readability'; import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; import rison from 'rison'; @@ -640,6 +640,10 @@ const RSS_PATH_REGEX = /^((\S+)\/)?rss\.xml$/; const MARKDOWN_PATH_REGEX = /\.md$/; const LLMS_FULL_PATH_REGEX = /^llms-full\.txt\/\d+$/; const EMBED_PAGE_PATH_REGEX = /^~gitbook\/embed\/page(\/(\S*))?$/; +const PATH_ALIASES: Record = { + 'sitemap.md': 'llms.txt', + '.well-known/sitemap.md': 'llms.txt', +}; /** * Encode path in a site content. @@ -653,7 +657,7 @@ function encodePathInSiteContent( routeType?: 'static' | 'dynamic'; events?: ServerInsightsEventInput[] | undefined; } { - const pathname = removeLeadingSlash(removeTrailingSlash(rawPathname)); + let pathname = removeLeadingSlash(removeTrailingSlash(rawPathname)); if (pathname.match(/^~gitbook\/ogimage\/\S+$/)) { return { pathname }; @@ -701,6 +705,7 @@ function encodePathInSiteContent( }; } + pathname = PATH_ALIASES[pathname] || pathname; switch (pathname) { case '~gitbook/embed': case '~gitbook/embed/assistant': @@ -740,9 +745,9 @@ function encodePathInSiteContent( // PDF, search and auth routes are always dynamic as they depend on the request. return { pathname, routeType: 'dynamic' }; default: { - // If the pathname is a markdown file or the request is accepting markdown, + // If the pathname is a markdown file or the request is ing markdown, // we rewrite it to ~gitbook/markdown/:pathname - if (pathname.match(MARKDOWN_PATH_REGEX) || isMarkdownPreferred(request)) { + if (pathname.match(MARKDOWN_PATH_REGEX) || shouldServeMarkdown(request).serve) { const pagePathWithoutMD = pathname.replace(MARKDOWN_PATH_REGEX, ''); return { pathname: `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`, @@ -797,16 +802,3 @@ async function writeResponseCookies( return response; } - -const MARKDOWN_MEDIA_TYPES = ['text/markdown']; - -/** - * Test if a request is requesting a markdown version of the page. - */ -function isMarkdownPreferred(request: Request): boolean { - const negotiator = new Negotiator({ - headers: Object.fromEntries(request.headers.entries()), - }); - const mediaTypes = negotiator.mediaTypes(); - return MARKDOWN_MEDIA_TYPES.some((mediaType) => mediaTypes.includes(mediaType)); -} diff --git a/packages/gitbook/src/routes/markdownPage.ts b/packages/gitbook/src/routes/markdownPage.ts index bdd6b06fb..758e75199 100644 --- a/packages/gitbook/src/routes/markdownPage.ts +++ b/packages/gitbook/src/routes/markdownPage.ts @@ -1,7 +1,7 @@ import type { GitBookSiteContext } from '@/lib/context'; -import { DataFetcherError, getExposableError } from '@/lib/data'; +import { getExposableError } from '@/lib/data'; import { getMarkdownForPage } from '@/lib/markdownPage'; -import { resolvePagePathDocumentOrGroup } from '@/lib/pages'; +import { getSimilarPages, resolvePagePathDocumentOrGroup } from '@/lib/pages'; /** * Serve a markdown version of a page. @@ -11,17 +11,14 @@ export async function servePageMarkdown(context: GitBookSiteContext, pagePath: s try { const pageLookup = resolvePagePathDocumentOrGroup(context.revision.pages, pagePath); if (!pageLookup) { - throw new DataFetcherError(`Page "${pagePath}" not found`, 404); + // Generates a markdown body for missing pages. Return this with a 200 status (not 404) because agents discard 404 response bodies. + const md = generateNotFoundMarkdown(context, pagePath); + // Return as 200 so agents read the body + return serveMarkdown(md); } const markdown = await getMarkdownForPage(context, pageLookup); - - return new Response(markdown, { - headers: { - 'Content-Type': 'text/markdown; charset=utf-8', - 'X-Robots-Tag': 'noindex', - }, - }); + return serveMarkdown(markdown); } catch (error) { const exposable = getExposableError(error); return new Response(exposable.message, { @@ -32,3 +29,37 @@ export async function servePageMarkdown(context: GitBookSiteContext, pagePath: s }); } } + +function generateNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) { + const similarPages = getSimilarPages(context.revision.pages, pagePath, 5); + const sitemapUrl = context.linker.toAbsoluteURL(context.linker.toPathInSite('sitemap.md')); + const fullContentUrl = context.linker.toAbsoluteURL( + context.linker.toPathInSite('llms-full.txt') + ); + + return `# Page Not Found + +The URL \`${pagePath}\` does not exist. + +You might be looking for one of these pages: +${similarPages.map((page) => `- [${page.title}](${context.linker.toAbsoluteURL(context.linker.toPathInSpace(page.path))}.md)`).join('\n')} + +## How to find the correct page + +1. **Browse the full index**: [/sitemap.md](${sitemapUrl}) - Complete documentation index +2. **View the full content**: [/llms-full.txt](${fullContentUrl}) - Full content export + +## Tips for requesting documentation + +- For markdown responses, append \`.md\` to URLs (e.g., \`${context.linker.toPathInSpace(similarPages[0]?.path ?? 'docs/example')}.md\`) +- Use \`Accept: text/markdown\` header for content negotiation`; +} + +function serveMarkdown(markdown: string) { + return new Response(markdown, { + headers: { + 'Content-Type': 'text/markdown; charset=utf-8', + 'X-Robots-Tag': 'noindex', + }, + }); +} diff --git a/packages/gitbook/tests/llms.test.ts b/packages/gitbook/tests/llms.test.ts index b37f7f666..8b4ea0112 100644 --- a/packages/gitbook/tests/llms.test.ts +++ b/packages/gitbook/tests/llms.test.ts @@ -26,6 +26,26 @@ describe('llms.txt', () => { expect(response.headers.get('content-type')).toContain('text/markdown'); expect(await response.text()).toContain('# E2E Tests GitBook Open'); }); + + it('should expose llms.txt from sitemap.md', async () => { + const response = await fetch( + getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/sitemap.md') + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(await response.text()).toContain('# E2E Tests GitBook Open'); + }); + + it('should expose llms.txt from .well-known/sitemap.md', async () => { + const response = await fetch( + getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/.well-known/sitemap.md') + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(await response.text()).toContain('# E2E Tests GitBook Open'); + }); }); describe('llms-full.txt', () => { diff --git a/packages/gitbook/tests/markdown.test.ts b/packages/gitbook/tests/markdown.test.ts index 7ccfe1bb0..fa50dcd7e 100644 --- a/packages/gitbook/tests/markdown.test.ts +++ b/packages/gitbook/tests/markdown.test.ts @@ -27,3 +27,14 @@ it('should expose a markdown page with the accept header', async () => { expect(response.headers.get('x-robots-tag')).toBe('noindex'); expect(await response.text()).toContain('# Text page'); }); + +it('should should return a 200 for a page not found', async () => { + const response = await fetch( + getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/missing-page.md') + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBe('noindex'); + expect(await response.text()).toContain('# Page Not Found'); +});