mirror of
https://github.com/GitbookIO/gitbook.git
synced 2026-09-23 02:53:29 +00:00
Improve AI agent detection and readability by outputting 200 for page not found (#4164)
This commit is contained in:
Samy Pessé
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"gitbook": minor
|
||||
---
|
||||
|
||||
Redirect sitemap.md and .well-known/sitemap.md to llms.txt.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"gitbook": minor
|
||||
---
|
||||
|
||||
Improve AI agent detection and readability by outputting 200 for page not found.
|
||||
@@ -7,7 +7,7 @@
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.4",
|
||||
"@changesets/cli": "^2.30.0",
|
||||
"turbo": "^2.9.1",
|
||||
"turbo": "^2.9.2",
|
||||
"vercel": "50.37.3",
|
||||
},
|
||||
},
|
||||
@@ -139,6 +139,7 @@
|
||||
"@sindresorhus/fnv1a": "^3.1.0",
|
||||
"@tailwindcss/container-queries": "^0.1.1",
|
||||
"@tusbar/cache-control": "^1.0.2",
|
||||
"@vercel/agent-readability": "^0.2.1",
|
||||
"ai": "^4.2.2",
|
||||
"assert-never": "catalog:",
|
||||
"bidc": "catalog:",
|
||||
@@ -150,6 +151,7 @@
|
||||
"js-cookie": "^3.0.5",
|
||||
"jsontoxml": "^1.0.1",
|
||||
"jwt-decode": "^4.0.0",
|
||||
"leven": "^4.1.0",
|
||||
"mcp-handler": "^1.0.2",
|
||||
"mdast-util-from-markdown": "^2.0.2",
|
||||
"mdast-util-frontmatter": "^2.0.1",
|
||||
@@ -160,7 +162,6 @@
|
||||
"micromark-extension-frontmatter": "^2.0.0",
|
||||
"micromark-extension-gfm": "^3.0.0",
|
||||
"motion": "^12.23.24",
|
||||
"negotiator": "^1.0.0",
|
||||
"next": "15.4.11",
|
||||
"next-themes": "^0.4.6",
|
||||
"nuqs": "^2.2.3",
|
||||
@@ -1718,6 +1719,8 @@
|
||||
|
||||
"@ungap/structured-clone": ["@ungap/structured-clone@1.2.0", "", {}, "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="],
|
||||
|
||||
"@vercel/agent-readability": ["@vercel/agent-readability@0.2.1", "", { "peerDependencies": { "next": ">=14" }, "optionalPeers": ["next"], "bin": { "agent-readability": "dist/cli/index.cjs" } }, "sha512-ShT7BzIS/dwKompii8tm5do+NR1g4xL5M3wM7S01xsH6yuYQ7wiTPZEcmHMFLHCsAQg45/mD0hgrufpS3NVunw=="],
|
||||
|
||||
"@vercel/backends": ["@vercel/backends@0.0.53", "", { "dependencies": { "@vercel/build-utils": "13.12.0", "@vercel/nft": "1.5.0", "execa": "3.2.0", "fs-extra": "11.1.0", "oxc-transform": "0.111.0", "path-to-regexp": "8.3.0", "resolve.exports": "2.0.3", "rolldown": "1.0.0-rc.1", "srvx": "0.8.9", "tsx": "4.21.0", "zod": "3.22.4" }, "peerDependencies": { "typescript": "^4.0.0 || ^5.0.0" } }, "sha512-f8JK9Jatlco6kT/NsLYVR88iqS7oZxUXzXW8TofLmyTT5o+OEXSY2hzAQSjul6r0qz/9yAhtQkMhcHG8Ne01ZQ=="],
|
||||
|
||||
"@vercel/blob": ["@vercel/blob@2.3.0", "", { "dependencies": { "async-retry": "^1.3.3", "is-buffer": "^2.0.5", "is-node-process": "^1.2.0", "throttleit": "^2.1.0", "undici": "^6.23.0" } }, "sha512-oYWiJbWRQ7gz9Mj0X/NHFJ3OcLMOBzq/2b3j6zeNrQmtFo6dHwU8FAwNpxVIYddVMd+g8eqEi7iRueYx8FtM0Q=="],
|
||||
@@ -2612,7 +2615,7 @@
|
||||
|
||||
"layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="],
|
||||
|
||||
"leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="],
|
||||
"leven": ["leven@4.1.0", "", {}, "sha512-KZ9W9nWDT7rF7Dazg8xyLHGLrmpgq2nVNFUckhqdW3szVP6YhCpp/RAnpmVExA9JvrMynjwSLVrEj3AepHR6ew=="],
|
||||
|
||||
"lightningcss": ["lightningcss@1.30.1", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-darwin-arm64": "1.30.1", "lightningcss-darwin-x64": "1.30.1", "lightningcss-freebsd-x64": "1.30.1", "lightningcss-linux-arm-gnueabihf": "1.30.1", "lightningcss-linux-arm64-gnu": "1.30.1", "lightningcss-linux-arm64-musl": "1.30.1", "lightningcss-linux-x64-gnu": "1.30.1", "lightningcss-linux-x64-musl": "1.30.1", "lightningcss-win32-arm64-msvc": "1.30.1", "lightningcss-win32-x64-msvc": "1.30.1" } }, "sha512-xi6IyHML+c9+Q3W0S4fCQJOym42pyurFiJUHEcEyHS0CeKzia4yZDEsLlqOFykxOdHpNy0NmvVO31vcSqAxJCg=="],
|
||||
|
||||
@@ -4360,6 +4363,8 @@
|
||||
|
||||
"@scalar/object-utils/@scalar/helpers": ["@scalar/helpers@0.1.1", "", {}, "sha512-eJjuCI5djqU0adTwrHvpDf0xuwNRpwZinCfJ03QjnmIFBM9Ch0u3tn/0EZOhcVNbdEyJ+3yvSVy1dCmmvgtpvQ=="],
|
||||
|
||||
"@scalar/openapi-parser/leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="],
|
||||
|
||||
"@scalar/openapi-types/zod": ["zod@4.1.11", "", {}, "sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg=="],
|
||||
|
||||
"@scalar/postman-to-openapi/@scalar/helpers": ["@scalar/helpers@0.0.12", "", {}, "sha512-4NDmHShyi1hrVRsJCdRZT/FIpy+/5PFbVbQLRYX/pjpu5cYqHBj9s6n5RI6gGDXEBHAIFi63g9FC6Isgr66l1Q=="],
|
||||
@@ -5418,6 +5423,8 @@
|
||||
|
||||
"@scalar/api-client/@scalar/openapi-parser/@scalar/openapi-upgrader": ["@scalar/openapi-upgrader@0.1.3", "", { "dependencies": { "@scalar/openapi-types": "0.5.0" } }, "sha512-iROhcgy3vge6zsviPtoTLHale0nYt1PLhuMmJweQwJ0U23ZYyYhV5xgHtAd0OBEXuqT6rjYbJFvKOJZmJOwpNQ=="],
|
||||
|
||||
"@scalar/api-client/@scalar/openapi-parser/leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="],
|
||||
|
||||
"@scalar/api-client/@scalar/workspace-store/@scalar/openapi-upgrader": ["@scalar/openapi-upgrader@0.1.3", "", { "dependencies": { "@scalar/openapi-types": "0.5.0" } }, "sha512-iROhcgy3vge6zsviPtoTLHale0nYt1PLhuMmJweQwJ0U23ZYyYhV5xgHtAd0OBEXuqT6rjYbJFvKOJZmJOwpNQ=="],
|
||||
|
||||
"@scalar/components/@scalar/oas-utils/@scalar/json-magic": ["@scalar/json-magic@0.6.1", "", { "dependencies": { "@scalar/helpers": "0.0.12", "yaml": "2.8.0" } }, "sha512-HJMPY5dUU3EXVS4EkjAFXo+uCrby/YFu/gljKDQnhYWRy5zQ0sJWrOEDcHS8nLoJRCIRD5tiVpCxnUnSb6OoAQ=="],
|
||||
@@ -5452,6 +5459,8 @@
|
||||
|
||||
"@scalar/import/@scalar/openapi-parser/@scalar/openapi-upgrader": ["@scalar/openapi-upgrader@0.1.3", "", { "dependencies": { "@scalar/openapi-types": "0.5.0" } }, "sha512-iROhcgy3vge6zsviPtoTLHale0nYt1PLhuMmJweQwJ0U23ZYyYhV5xgHtAd0OBEXuqT6rjYbJFvKOJZmJOwpNQ=="],
|
||||
|
||||
"@scalar/import/@scalar/openapi-parser/leven": ["leven@4.0.0", "", {}, "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw=="],
|
||||
|
||||
"@scalar/postman-to-openapi/@scalar/oas-utils/@scalar/json-magic": ["@scalar/json-magic@0.6.1", "", { "dependencies": { "@scalar/helpers": "0.0.12", "yaml": "2.8.0" } }, "sha512-HJMPY5dUU3EXVS4EkjAFXo+uCrby/YFu/gljKDQnhYWRy5zQ0sJWrOEDcHS8nLoJRCIRD5tiVpCxnUnSb6OoAQ=="],
|
||||
|
||||
"@scalar/postman-to-openapi/@scalar/oas-utils/@scalar/object-utils": ["@scalar/object-utils@1.2.8", "", { "dependencies": { "@scalar/helpers": "0.0.12", "flatted": "^3.3.3", "just-clone": "^6.2.0", "ts-deepmerge": "^7.0.1", "type-fest": "5.0.0" } }, "sha512-FjPxEg7Hw0tz2iTKvi+gYt+luZK0TqhX50hUIBuaYa/Ja/OMuKLp9QHhB5U68F1L55CZNP4vwoNNfXeYWnVEZg=="],
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
"@sindresorhus/fnv1a": "^3.1.0",
|
||||
"@tailwindcss/container-queries": "^0.1.1",
|
||||
"@tusbar/cache-control": "^1.0.2",
|
||||
"@vercel/agent-readability": "^0.2.1",
|
||||
"ai": "^4.2.2",
|
||||
"assert-never": "catalog:",
|
||||
"bidc": "catalog:",
|
||||
@@ -52,7 +53,7 @@
|
||||
"micromark-extension-frontmatter": "^2.0.0",
|
||||
"micromark-extension-gfm": "^3.0.0",
|
||||
"motion": "^12.23.24",
|
||||
"negotiator": "^1.0.0",
|
||||
"leven": "^4.1.0",
|
||||
"next": "15.4.11",
|
||||
"next-themes": "^0.4.6",
|
||||
"nuqs": "^2.2.3",
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
|
||||
import {
|
||||
extractPagePath,
|
||||
getSimilarPages,
|
||||
resolveFirstDocument,
|
||||
resolvePagePath,
|
||||
resolvePagePathDocumentOrGroup,
|
||||
@@ -292,3 +293,74 @@ describe('resolvePagePath', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getSimilarPages', () => {
|
||||
it('returns the closest matches for a typoed path', () => {
|
||||
const pages: RevisionPage[] = [
|
||||
createDocumentPage('install', 'guides/installation'),
|
||||
createDocumentPage('config', 'guides/configuration'),
|
||||
createDocumentPage('auth', 'api/authentication'),
|
||||
];
|
||||
|
||||
expect(getSimilarPages(pages, 'guides/installtion', 2).map((page) => page.id)).toEqual([
|
||||
'install',
|
||||
'config',
|
||||
]);
|
||||
});
|
||||
|
||||
it('prefers pages with the same path hierarchy', () => {
|
||||
const pages: RevisionPage[] = [
|
||||
createDocumentPage('api-auth', 'api/reference/authentication'),
|
||||
createDocumentPage('guide-auth', 'guides/authentication'),
|
||||
createDocumentPage('api-errors', 'api/reference/errors'),
|
||||
];
|
||||
|
||||
expect(
|
||||
getSimilarPages(pages, 'api/reference/authentcation', 1).map((page) => page.id)
|
||||
).toEqual(['api-auth']);
|
||||
});
|
||||
|
||||
it('ignores hidden pages', () => {
|
||||
const pages: RevisionPage[] = [
|
||||
createDocumentPage('hidden', 'private-api', true),
|
||||
createDocumentPage('versioned', 'private-api-v2'),
|
||||
createDocumentPage('public', 'public-api'),
|
||||
];
|
||||
|
||||
const similar = getSimilarPages(pages, 'privte-api', 2);
|
||||
|
||||
expect(similar.map((page) => page.id)).toEqual(['versioned', 'public']);
|
||||
expect(similar.some((page) => page.id === 'hidden')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
function createDocumentPage(id: string, path: string, hidden = false): RevisionPage {
|
||||
const slug = path.split('/').at(-1) ?? path;
|
||||
|
||||
return {
|
||||
id,
|
||||
title: path,
|
||||
kind: 'sheet',
|
||||
type: 'document',
|
||||
hidden,
|
||||
urls: {
|
||||
app: `https://app.gitbook.com/s/fvBF1lEt2CVd4RTffSOk/${path}`,
|
||||
},
|
||||
path,
|
||||
slug,
|
||||
pages: [],
|
||||
tags: [],
|
||||
layout: {
|
||||
cover: true,
|
||||
coverSize: RevisionPageLayoutOptionsCoverSize.Full,
|
||||
title: true,
|
||||
description: true,
|
||||
tableOfContents: true,
|
||||
outline: true,
|
||||
pagination: true,
|
||||
width: RevisionPageLayoutOptionsWidth.Default,
|
||||
metadata: true,
|
||||
tags: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
type RevisionPageGroup,
|
||||
RevisionPageType,
|
||||
} from '@gitbook/api';
|
||||
import leven from 'leven';
|
||||
import { removeLeadingSlash, removeTrailingSlash } from './paths';
|
||||
|
||||
export type AncestorRevisionPage = RevisionPageDocument | RevisionPageGroup;
|
||||
@@ -208,6 +209,157 @@ export function resolveFirstDocument(
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a similar pages to a given path. This is used to suggest similar pages when a page is not found.
|
||||
*/
|
||||
export function getSimilarPages(
|
||||
rootPages: RevisionPage[],
|
||||
pagePath: string,
|
||||
topK = 5
|
||||
): RevisionPageDocument[] {
|
||||
if (topK <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const pages = flattenPages(rootPages, (page) => !page.hidden);
|
||||
if (pages.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const inputPath = createComparablePagePath(pagePath);
|
||||
if (!inputPath.path) {
|
||||
return pages.slice(0, topK);
|
||||
}
|
||||
|
||||
const firstPageId = resolveFirstDocument(rootPages, [])?.page.id;
|
||||
|
||||
return pages
|
||||
.map((page, index) => {
|
||||
const candidatePaths = new Set([page.path]);
|
||||
if (page.id === firstPageId) {
|
||||
candidatePaths.add('');
|
||||
}
|
||||
|
||||
let score = 0;
|
||||
for (const candidatePath of candidatePaths) {
|
||||
score = Math.max(
|
||||
score,
|
||||
scoreComparablePagePaths(inputPath, createComparablePagePath(candidatePath))
|
||||
);
|
||||
}
|
||||
|
||||
return { page, index, score };
|
||||
})
|
||||
.sort((left, right) => right.score - left.score || left.index - right.index)
|
||||
.slice(0, topK)
|
||||
.map(({ page }) => page);
|
||||
}
|
||||
|
||||
type ComparablePagePath = {
|
||||
path: string;
|
||||
segments: string[];
|
||||
tokens: string[];
|
||||
lastSegment: string;
|
||||
};
|
||||
|
||||
function createComparablePagePath(path: string): ComparablePagePath {
|
||||
const normalizedPath = removeTrailingSlash(removeLeadingSlash(path.trim().toLowerCase()))
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.join('/');
|
||||
const segments = normalizedPath ? normalizedPath.split('/') : [];
|
||||
const tokens = segments.flatMap((segment) => segment.split(/[-_\s]+/g)).filter(Boolean);
|
||||
|
||||
return {
|
||||
path: normalizedPath,
|
||||
segments,
|
||||
tokens,
|
||||
lastSegment: segments.at(-1) ?? normalizedPath,
|
||||
};
|
||||
}
|
||||
|
||||
function scoreComparablePagePaths(
|
||||
inputPath: ComparablePagePath,
|
||||
candidatePath: ComparablePagePath
|
||||
): number {
|
||||
const fullPathSimilarity = getLevenshteinSimilarity(inputPath.path, candidatePath.path);
|
||||
const segmentSimilarity = getAlignedSegmentSimilarity(
|
||||
inputPath.segments,
|
||||
candidatePath.segments
|
||||
);
|
||||
const lastSegmentSimilarity = getLevenshteinSimilarity(
|
||||
inputPath.lastSegment,
|
||||
candidatePath.lastSegment
|
||||
);
|
||||
const tokenOverlap = getJaccardSimilarity(inputPath.tokens, candidatePath.tokens);
|
||||
|
||||
let score =
|
||||
fullPathSimilarity * 0.4 +
|
||||
segmentSimilarity * 0.3 +
|
||||
lastSegmentSimilarity * 0.2 +
|
||||
tokenOverlap * 0.1;
|
||||
|
||||
if (
|
||||
candidatePath.path &&
|
||||
(inputPath.path.startsWith(`${candidatePath.path}/`) ||
|
||||
candidatePath.path.startsWith(`${inputPath.path}/`))
|
||||
) {
|
||||
score += 0.05;
|
||||
}
|
||||
|
||||
if (inputPath.lastSegment && inputPath.lastSegment === candidatePath.lastSegment) {
|
||||
score += 0.1;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
function getAlignedSegmentSimilarity(left: string[], right: string[]): number {
|
||||
const length = Math.max(left.length, right.length);
|
||||
if (length === 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
let score = 0;
|
||||
for (let index = 0; index < length; index += 1) {
|
||||
score += getLevenshteinSimilarity(left[index] ?? '', right[index] ?? '');
|
||||
}
|
||||
|
||||
return score / length;
|
||||
}
|
||||
|
||||
function getJaccardSimilarity(left: string[], right: string[]): number {
|
||||
const leftSet = new Set(left);
|
||||
const rightSet = new Set(right);
|
||||
const union = new Set([...leftSet, ...rightSet]);
|
||||
|
||||
if (union.size === 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
let intersection = 0;
|
||||
for (const token of leftSet) {
|
||||
if (rightSet.has(token)) {
|
||||
intersection += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return intersection / union.size;
|
||||
}
|
||||
|
||||
function getLevenshteinSimilarity(left: string, right: string): number {
|
||||
if (left === right) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const maxLength = Math.max(left.length, right.length);
|
||||
if (maxLength === 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 1 - leven(left, right) / maxLength;
|
||||
}
|
||||
|
||||
function resolvePageDocument(
|
||||
page: RevisionPage,
|
||||
ancestors: AncestorRevisionPage[]
|
||||
|
||||
@@ -3,7 +3,7 @@ import {
|
||||
SiteInsightsDisplayContext,
|
||||
SiteInsightsLLMSVariant,
|
||||
} from '@gitbook/api';
|
||||
import Negotiator from 'negotiator';
|
||||
import { shouldServeMarkdown } from '@vercel/agent-readability';
|
||||
import type { NextRequest } from 'next/server';
|
||||
import { NextResponse } from 'next/server';
|
||||
import rison from 'rison';
|
||||
@@ -640,6 +640,10 @@ const RSS_PATH_REGEX = /^((\S+)\/)?rss\.xml$/;
|
||||
const MARKDOWN_PATH_REGEX = /\.md$/;
|
||||
const LLMS_FULL_PATH_REGEX = /^llms-full\.txt\/\d+$/;
|
||||
const EMBED_PAGE_PATH_REGEX = /^~gitbook\/embed\/page(\/(\S*))?$/;
|
||||
const PATH_ALIASES: Record<string, string> = {
|
||||
'sitemap.md': 'llms.txt',
|
||||
'.well-known/sitemap.md': 'llms.txt',
|
||||
};
|
||||
|
||||
/**
|
||||
* Encode path in a site content.
|
||||
@@ -653,7 +657,7 @@ function encodePathInSiteContent(
|
||||
routeType?: 'static' | 'dynamic';
|
||||
events?: ServerInsightsEventInput[] | undefined;
|
||||
} {
|
||||
const pathname = removeLeadingSlash(removeTrailingSlash(rawPathname));
|
||||
let pathname = removeLeadingSlash(removeTrailingSlash(rawPathname));
|
||||
|
||||
if (pathname.match(/^~gitbook\/ogimage\/\S+$/)) {
|
||||
return { pathname };
|
||||
@@ -701,6 +705,7 @@ function encodePathInSiteContent(
|
||||
};
|
||||
}
|
||||
|
||||
pathname = PATH_ALIASES[pathname] || pathname;
|
||||
switch (pathname) {
|
||||
case '~gitbook/embed':
|
||||
case '~gitbook/embed/assistant':
|
||||
@@ -740,9 +745,9 @@ function encodePathInSiteContent(
|
||||
// PDF, search and auth routes are always dynamic as they depend on the request.
|
||||
return { pathname, routeType: 'dynamic' };
|
||||
default: {
|
||||
// If the pathname is a markdown file or the request is accepting markdown,
|
||||
// If the pathname is a markdown file or the request is ing markdown,
|
||||
// we rewrite it to ~gitbook/markdown/:pathname
|
||||
if (pathname.match(MARKDOWN_PATH_REGEX) || isMarkdownPreferred(request)) {
|
||||
if (pathname.match(MARKDOWN_PATH_REGEX) || shouldServeMarkdown(request).serve) {
|
||||
const pagePathWithoutMD = pathname.replace(MARKDOWN_PATH_REGEX, '');
|
||||
return {
|
||||
pathname: `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`,
|
||||
@@ -797,16 +802,3 @@ async function writeResponseCookies<R extends NextResponse>(
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
const MARKDOWN_MEDIA_TYPES = ['text/markdown'];
|
||||
|
||||
/**
|
||||
* Test if a request is requesting a markdown version of the page.
|
||||
*/
|
||||
function isMarkdownPreferred(request: Request): boolean {
|
||||
const negotiator = new Negotiator({
|
||||
headers: Object.fromEntries(request.headers.entries()),
|
||||
});
|
||||
const mediaTypes = negotiator.mediaTypes();
|
||||
return MARKDOWN_MEDIA_TYPES.some((mediaType) => mediaTypes.includes(mediaType));
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { GitBookSiteContext } from '@/lib/context';
|
||||
import { DataFetcherError, getExposableError } from '@/lib/data';
|
||||
import { getExposableError } from '@/lib/data';
|
||||
import { getMarkdownForPage } from '@/lib/markdownPage';
|
||||
import { resolvePagePathDocumentOrGroup } from '@/lib/pages';
|
||||
import { getSimilarPages, resolvePagePathDocumentOrGroup } from '@/lib/pages';
|
||||
|
||||
/**
|
||||
* Serve a markdown version of a page.
|
||||
@@ -11,17 +11,14 @@ export async function servePageMarkdown(context: GitBookSiteContext, pagePath: s
|
||||
try {
|
||||
const pageLookup = resolvePagePathDocumentOrGroup(context.revision.pages, pagePath);
|
||||
if (!pageLookup) {
|
||||
throw new DataFetcherError(`Page "${pagePath}" not found`, 404);
|
||||
// Generates a markdown body for missing pages. Return this with a 200 status (not 404) because agents discard 404 response bodies.
|
||||
const md = generateNotFoundMarkdown(context, pagePath);
|
||||
// Return as 200 so agents read the body
|
||||
return serveMarkdown(md);
|
||||
}
|
||||
|
||||
const markdown = await getMarkdownForPage(context, pageLookup);
|
||||
|
||||
return new Response(markdown, {
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'X-Robots-Tag': 'noindex',
|
||||
},
|
||||
});
|
||||
return serveMarkdown(markdown);
|
||||
} catch (error) {
|
||||
const exposable = getExposableError(error);
|
||||
return new Response(exposable.message, {
|
||||
@@ -32,3 +29,37 @@ export async function servePageMarkdown(context: GitBookSiteContext, pagePath: s
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function generateNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) {
|
||||
const similarPages = getSimilarPages(context.revision.pages, pagePath, 5);
|
||||
const sitemapUrl = context.linker.toAbsoluteURL(context.linker.toPathInSite('sitemap.md'));
|
||||
const fullContentUrl = context.linker.toAbsoluteURL(
|
||||
context.linker.toPathInSite('llms-full.txt')
|
||||
);
|
||||
|
||||
return `# Page Not Found
|
||||
|
||||
The URL \`${pagePath}\` does not exist.
|
||||
|
||||
You might be looking for one of these pages:
|
||||
${similarPages.map((page) => `- [${page.title}](${context.linker.toAbsoluteURL(context.linker.toPathInSpace(page.path))}.md)`).join('\n')}
|
||||
|
||||
## How to find the correct page
|
||||
|
||||
1. **Browse the full index**: [/sitemap.md](${sitemapUrl}) - Complete documentation index
|
||||
2. **View the full content**: [/llms-full.txt](${fullContentUrl}) - Full content export
|
||||
|
||||
## Tips for requesting documentation
|
||||
|
||||
- For markdown responses, append \`.md\` to URLs (e.g., \`${context.linker.toPathInSpace(similarPages[0]?.path ?? 'docs/example')}.md\`)
|
||||
- Use \`Accept: text/markdown\` header for content negotiation`;
|
||||
}
|
||||
|
||||
function serveMarkdown(markdown: string) {
|
||||
return new Response(markdown, {
|
||||
headers: {
|
||||
'Content-Type': 'text/markdown; charset=utf-8',
|
||||
'X-Robots-Tag': 'noindex',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -26,6 +26,26 @@ describe('llms.txt', () => {
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# E2E Tests GitBook Open');
|
||||
});
|
||||
|
||||
it('should expose llms.txt from sitemap.md', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/sitemap.md')
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# E2E Tests GitBook Open');
|
||||
});
|
||||
|
||||
it('should expose llms.txt from .well-known/sitemap.md', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/.well-known/sitemap.md')
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# E2E Tests GitBook Open');
|
||||
});
|
||||
});
|
||||
|
||||
describe('llms-full.txt', () => {
|
||||
|
||||
@@ -27,3 +27,14 @@ it('should expose a markdown page with the accept header', async () => {
|
||||
expect(response.headers.get('x-robots-tag')).toBe('noindex');
|
||||
expect(await response.text()).toContain('# Text page');
|
||||
});
|
||||
|
||||
it('should should return a 200 for a page not found', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/missing-page.md')
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(response.headers.get('x-robots-tag')).toBe('noindex');
|
||||
expect(await response.text()).toContain('# Page Not Found');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user