mirror of
https://github.com/GitbookIO/gitbook.git
synced 2026-11-18 14:20:18 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| eb6d3f85bb | |||
| 6d056cdd76 |
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"gitbook": patch
|
||||
---
|
||||
|
||||
Split llms.txt into per-section and per-page indexes when it exceeds 100,000 characters, and point pages to their llms.txt with a `describedby` link header.
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
import { type RouteParams, getPagePathFromParams, getStaticSiteContext } from '@/app/utils';
|
||||
import { serveLLMsTxt } from '@/routes/llms';
|
||||
|
||||
export const dynamic = 'force-static';
|
||||
|
||||
export async function GET(_request: NextRequest, { params }: { params: Promise<RouteParams> }) {
|
||||
const { context } = await getStaticSiteContext(await params);
|
||||
const pathname = getPagePathFromParams(await params);
|
||||
|
||||
return serveLLMsTxt(context, pathname);
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
|
||||
import { type LLMsIndexFile, type LLMsIndexNode, renderLLMsIndex } from './llms-index';
|
||||
|
||||
const ROOT = 'https://docs.example.com/llms.txt';
|
||||
|
||||
function page(path: string, children: LLMsIndexNode[] = []): LLMsIndexNode {
|
||||
return {
|
||||
kind: 'page',
|
||||
title: `Page ${path}`,
|
||||
description: `About ${path}`,
|
||||
url: `https://docs.example.com/${path}.md`,
|
||||
indexURL: `https://docs.example.com/${path}/llms.txt`,
|
||||
children,
|
||||
};
|
||||
}
|
||||
|
||||
function section(path: string, children: LLMsIndexNode[], indexURL?: string): LLMsIndexNode {
|
||||
return {
|
||||
kind: 'structure',
|
||||
title: `Section ${path}`,
|
||||
indexURL: indexURL ?? `https://docs.example.com/${path}/llms.txt`,
|
||||
children,
|
||||
};
|
||||
}
|
||||
|
||||
function pages(prefix: string, count: number): LLMsIndexNode[] {
|
||||
return Array.from({ length: count }, (_, index) => page(`${prefix}/page-${index}`));
|
||||
}
|
||||
|
||||
function file(nodes: LLMsIndexNode[], footer?: string): LLMsIndexFile {
|
||||
return { title: 'Docs', url: ROOT, nodes, footer };
|
||||
}
|
||||
|
||||
describe('renderLLMsIndex', () => {
|
||||
it('renders everything inline when it fits', () => {
|
||||
const output = renderLLMsIndex(
|
||||
file([section('guides', [page('guides/start', [page('guides/start/install')])])])
|
||||
);
|
||||
|
||||
expect(output).toBe(
|
||||
[
|
||||
'# Docs',
|
||||
'',
|
||||
'## Section guides',
|
||||
'',
|
||||
'- [Page guides/start](https://docs.example.com/guides/start.md): About guides/start',
|
||||
' - [Page guides/start/install](https://docs.example.com/guides/start/install.md): About guides/start/install',
|
||||
'',
|
||||
].join('\n')
|
||||
);
|
||||
});
|
||||
|
||||
it('splits out the largest sections first', () => {
|
||||
const output = renderLLMsIndex(
|
||||
file([section('small', pages('small', 2)), section('large', pages('large', 50))]),
|
||||
3_000
|
||||
);
|
||||
|
||||
expect(output.length).toBeLessThanOrEqual(3_000);
|
||||
expect(output).toContain('## Section small');
|
||||
expect(output).toContain('Page small/page-1');
|
||||
expect(output).toContain(
|
||||
'## Section large\n\n- [Section large](https://docs.example.com/large/llms.txt)'
|
||||
);
|
||||
expect(output).not.toContain('Page large/page-1');
|
||||
});
|
||||
|
||||
it('never links a node to the file it is rendered in', () => {
|
||||
const output = renderLLMsIndex(
|
||||
file([section('default', [page('a', pages('a', 30)), page('b', pages('b', 5))], ROOT)]),
|
||||
2_000
|
||||
);
|
||||
|
||||
expect(output.length).toBeLessThanOrEqual(2_000);
|
||||
expect(output).toContain('## Section default');
|
||||
expect(output).toContain('- [Page a](https://docs.example.com/a.md): About a');
|
||||
expect(output).toContain(' - [Pages under Page a](https://docs.example.com/a/llms.txt)');
|
||||
expect(output).not.toContain(ROOT);
|
||||
});
|
||||
|
||||
it('serves an oversized index when nothing can be split out', () => {
|
||||
const output = renderLLMsIndex(file(pages('flat', 50)), 1_000);
|
||||
|
||||
expect(output.length).toBeGreaterThan(1_000);
|
||||
expect(output).toContain('Page flat/page-49');
|
||||
});
|
||||
|
||||
it('counts the footer in the size', () => {
|
||||
const nodes = [section('a', pages('a', 10)), section('b', pages('b', 10))];
|
||||
const withoutFooter = renderLLMsIndex(file(nodes), 2_000);
|
||||
const withFooter = renderLLMsIndex(file(nodes, 'x'.repeat(1_000)), 2_000);
|
||||
|
||||
expect(withoutFooter).toContain('Page a/page-1');
|
||||
expect(withFooter.length).toBeLessThanOrEqual(2_000);
|
||||
expect(withFooter).toContain('- [Section a](https://docs.example.com/a/llms.txt)');
|
||||
});
|
||||
|
||||
it('fits large sites within the default limit', () => {
|
||||
const nodes = Array.from({ length: 20 }, (_, sectionIndex) =>
|
||||
section(
|
||||
`s${sectionIndex}`,
|
||||
Array.from({ length: 20 }, (_, groupIndex) =>
|
||||
page(
|
||||
`s${sectionIndex}/g${groupIndex}`,
|
||||
pages(`s${sectionIndex}/g${groupIndex}`, 10)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
const output = renderLLMsIndex(file(nodes));
|
||||
expect(output.length).toBeLessThanOrEqual(100_000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,291 @@
|
||||
import type { Heading, List, ListItem, Paragraph, Root, RootContent } from 'mdast';
|
||||
import { toMarkdown } from 'mdast-util-to-markdown';
|
||||
|
||||
/**
|
||||
* Agents truncate or skip llms.txt files past this size, so larger indexes are split into
|
||||
* scoped child files (see https://llmstxt.org/ on files at subpaths).
|
||||
*/
|
||||
export const LLMS_TXT_MAX_CHARACTERS = 100_000;
|
||||
|
||||
export type LLMsIndexNode = {
|
||||
/** A navigation level (section group, section, site space) rendered as a heading, or a page rendered as a list item. */
|
||||
kind: 'structure' | 'page';
|
||||
title: string;
|
||||
description?: string;
|
||||
/** Markdown URL of the page itself. */
|
||||
url?: string;
|
||||
/** URL of the llms.txt scoped to this node, when it can be split out. */
|
||||
indexURL?: string;
|
||||
children: LLMsIndexNode[];
|
||||
};
|
||||
|
||||
export type LLMsIndexFile = {
|
||||
title: string;
|
||||
/** Rendered as the summary blockquote below the title. */
|
||||
summary?: string;
|
||||
/** URL this file is served at, so nodes sharing it are never collapsed into a link to itself. */
|
||||
url: string;
|
||||
nodes: LLMsIndexNode[];
|
||||
/** Appended after the index, and counted in its size. */
|
||||
footer?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Render an llms.txt index, collapsing the nodes that save the most space into links to
|
||||
* their own llms.txt until the file fits within the limit.
|
||||
*/
|
||||
export function renderLLMsIndex(
|
||||
file: LLMsIndexFile,
|
||||
limit: number = LLMS_TXT_MAX_CHARACTERS
|
||||
): string {
|
||||
const collapsed = new Set<LLMsIndexNode>();
|
||||
const candidates: LLMsIndexNode[] = [];
|
||||
const visit = (node: LLMsIndexNode) => {
|
||||
// A node sharing this file's URL would collapse into a link to the file itself.
|
||||
if (node.indexURL && node.indexURL !== file.url && node.children.length > 0) {
|
||||
candidates.push(node);
|
||||
}
|
||||
node.children.forEach(visit);
|
||||
};
|
||||
file.nodes.forEach(visit);
|
||||
|
||||
const ownSizes = new Map<LLMsIndexNode, number>();
|
||||
const collapsedSizes = new Map<LLMsIndexNode, number>();
|
||||
const ownSize = (node: LLMsIndexNode) =>
|
||||
getCached(ownSizes, node, () => measureNode(node, false));
|
||||
const collapsedSize = (node: LLMsIndexNode) =>
|
||||
getCached(collapsedSizes, node, () => measureNode(node, true));
|
||||
const baseSize = renderFile(file, collapsed, []).length;
|
||||
|
||||
// Sizes are estimated per node to stay linear in the number of pages; the exact render
|
||||
// only runs once the estimate fits.
|
||||
while (true) {
|
||||
const sizes = new Map<LLMsIndexNode, number>();
|
||||
const sizeOf = (node: LLMsIndexNode): number => {
|
||||
const size = collapsed.has(node)
|
||||
? collapsedSize(node)
|
||||
: node.children.reduce((total, child) => total + sizeOf(child), ownSize(node));
|
||||
sizes.set(node, size);
|
||||
return size;
|
||||
};
|
||||
const estimated = file.nodes.reduce((total, node) => total + sizeOf(node), baseSize);
|
||||
|
||||
if (estimated <= limit) {
|
||||
const output = renderFile(file, collapsed, file.nodes);
|
||||
if (output.length <= limit) {
|
||||
return output;
|
||||
}
|
||||
}
|
||||
|
||||
let best: LLMsIndexNode | null = null;
|
||||
let bestSaving = 0;
|
||||
for (const node of candidates) {
|
||||
const size = sizes.get(node);
|
||||
// Nodes under a collapsed ancestor weren't measured in this pass.
|
||||
if (size === undefined || collapsed.has(node)) {
|
||||
continue;
|
||||
}
|
||||
const saving = size - collapsedSize(node);
|
||||
if (saving > bestSaving) {
|
||||
best = node;
|
||||
bestSaving = saving;
|
||||
}
|
||||
}
|
||||
|
||||
if (!best) {
|
||||
// Nothing left to split out: serve the smallest index we can build.
|
||||
return renderFile(file, collapsed, file.nodes);
|
||||
}
|
||||
collapsed.add(best);
|
||||
}
|
||||
}
|
||||
|
||||
function getCached<K, V>(cache: Map<K, V>, key: K, compute: () => V): V {
|
||||
let value = cache.get(key);
|
||||
if (value === undefined) {
|
||||
value = compute();
|
||||
cache.set(key, value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Approximate the characters a node adds on its own, excluding its children.
|
||||
*/
|
||||
function measureNode(node: LLMsIndexNode, asCollapsed: boolean): number {
|
||||
const blocks =
|
||||
node.kind === 'structure'
|
||||
? asCollapsed
|
||||
? toCollapsedStructureBlocks(node, 2)
|
||||
: toStructureBlocks(node, 2, new Set(), false)
|
||||
: [toList([asCollapsed ? toCollapsedItem(node) : toPageItem(node, new Set(), false)])];
|
||||
// Nested list items are indented, which the standalone render doesn't account for.
|
||||
return toMarkdown({ type: 'root', children: blocks }, { bullet: '-' }).length + 4;
|
||||
}
|
||||
|
||||
function renderFile(
|
||||
file: LLMsIndexFile,
|
||||
collapsed: Set<LLMsIndexNode>,
|
||||
nodes: LLMsIndexNode[]
|
||||
): string {
|
||||
const tree: Root = {
|
||||
type: 'root',
|
||||
children: [
|
||||
{ type: 'heading', depth: 1, children: [{ type: 'text', value: file.title }] },
|
||||
...(file.summary
|
||||
? [
|
||||
{
|
||||
type: 'blockquote' as const,
|
||||
children: [toParagraph(file.summary)],
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...toBlocks(nodes, 2, collapsed),
|
||||
],
|
||||
};
|
||||
|
||||
let output = toMarkdown(tree, { bullet: '-' });
|
||||
if (file.footer) {
|
||||
output += `\n\n---\n\n${file.footer}`;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render sibling nodes, grouping consecutive list items into a single list.
|
||||
*/
|
||||
function toBlocks(
|
||||
nodes: LLMsIndexNode[],
|
||||
depth: number,
|
||||
collapsed: Set<LLMsIndexNode>
|
||||
): RootContent[] {
|
||||
const blocks: RootContent[] = [];
|
||||
let items: ListItem[] = [];
|
||||
const flush = () => {
|
||||
if (items.length > 0) {
|
||||
blocks.push(toList(items));
|
||||
items = [];
|
||||
}
|
||||
};
|
||||
|
||||
for (const node of nodes) {
|
||||
if (node.kind === 'structure') {
|
||||
flush();
|
||||
blocks.push(
|
||||
...(collapsed.has(node)
|
||||
? toCollapsedStructureBlocks(node, depth)
|
||||
: toStructureBlocks(node, depth, collapsed, true))
|
||||
);
|
||||
} else if (collapsed.has(node)) {
|
||||
items.push(toCollapsedItem(node));
|
||||
} else {
|
||||
items.push(toPageItem(node, collapsed, true));
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function toStructureBlocks(
|
||||
node: LLMsIndexNode,
|
||||
depth: number,
|
||||
collapsed: Set<LLMsIndexNode>,
|
||||
withChildren: boolean
|
||||
): RootContent[] {
|
||||
return [
|
||||
{
|
||||
type: 'heading',
|
||||
depth: Math.min(depth, 6) as Heading['depth'],
|
||||
children: [{ type: 'text', value: node.title }],
|
||||
},
|
||||
...(node.description ? [toParagraph(node.description)] : []),
|
||||
...(withChildren ? toBlocks(node.children, depth + 1, collapsed) : []),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the heading so the link isn't read as part of the previous section's list.
|
||||
*/
|
||||
function toCollapsedStructureBlocks(node: LLMsIndexNode, depth: number): RootContent[] {
|
||||
return [
|
||||
{
|
||||
type: 'heading',
|
||||
depth: Math.min(depth, 6) as Heading['depth'],
|
||||
children: [{ type: 'text', value: node.title }],
|
||||
},
|
||||
toList([toCollapsedItem(node)]),
|
||||
];
|
||||
}
|
||||
|
||||
function toPageItem(
|
||||
node: LLMsIndexNode,
|
||||
collapsed: Set<LLMsIndexNode>,
|
||||
withChildren: boolean
|
||||
): ListItem {
|
||||
const children: ListItem['children'] = [toLinkParagraph(node, node.url)];
|
||||
if (withChildren && node.children.length > 0) {
|
||||
children.push(
|
||||
toList(
|
||||
node.children.map((child) =>
|
||||
collapsed.has(child)
|
||||
? toCollapsedItem(child)
|
||||
: toPageItem(child, collapsed, true)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
return { type: 'listItem', spread: false, children };
|
||||
}
|
||||
|
||||
/**
|
||||
* A page that is also a document keeps its own link, with its subtree moved to its index.
|
||||
*/
|
||||
function toCollapsedItem(node: LLMsIndexNode): ListItem {
|
||||
if (node.kind === 'page' && node.url) {
|
||||
return {
|
||||
type: 'listItem',
|
||||
spread: false,
|
||||
children: [
|
||||
toLinkParagraph(node, node.url),
|
||||
toList([
|
||||
{
|
||||
type: 'listItem',
|
||||
spread: false,
|
||||
children: [
|
||||
toLinkParagraph({ title: `Pages under ${node.title}` }, node.indexURL),
|
||||
],
|
||||
},
|
||||
]),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'listItem',
|
||||
spread: false,
|
||||
children: [toLinkParagraph(node, node.indexURL)],
|
||||
};
|
||||
}
|
||||
|
||||
function toLinkParagraph(
|
||||
node: Pick<LLMsIndexNode, 'title' | 'description'>,
|
||||
url: string | undefined
|
||||
): Paragraph {
|
||||
const children: Paragraph['children'] = [
|
||||
url
|
||||
? { type: 'link', url, children: [{ type: 'text', value: node.title }] }
|
||||
: { type: 'text', value: node.title },
|
||||
];
|
||||
if (node.description) {
|
||||
children.push({ type: 'text', value: `: ${node.description}` });
|
||||
}
|
||||
return { type: 'paragraph', children };
|
||||
}
|
||||
|
||||
function toParagraph(value: string): Paragraph {
|
||||
return { type: 'paragraph', children: [{ type: 'text', value }] };
|
||||
}
|
||||
|
||||
function toList(children: ListItem[]): List {
|
||||
return { type: 'list', spread: false, children };
|
||||
}
|
||||
@@ -47,3 +47,30 @@ function flattenPages(
|
||||
export function getIndexablePages(rootPages: RevisionPage[]) {
|
||||
return flattenPages(rootPages, (page, ancestors) => isPageIndexable(ancestors, page));
|
||||
}
|
||||
|
||||
export type PageTreeEntry = {
|
||||
page: RevisionPageDocument | RevisionPageGroup;
|
||||
children: PageTreeEntry[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Get all indexable pages from a revision as a tree, keeping groups that contain indexable pages.
|
||||
*/
|
||||
export function getIndexablePageTree(
|
||||
rootPages: RevisionPage[],
|
||||
ancestors: (RevisionPageDocument | RevisionPageGroup)[] = []
|
||||
): PageTreeEntry[] {
|
||||
return rootPages.flatMap((page): PageTreeEntry[] => {
|
||||
if (page.type !== 'document' && page.type !== 'group') {
|
||||
return [];
|
||||
}
|
||||
if (!isPageIndexable(ancestors, page)) {
|
||||
return [];
|
||||
}
|
||||
const children = getIndexablePageTree(page.pages, [...ancestors, page]);
|
||||
if (page.type === 'group' && children.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return [{ page, children }];
|
||||
});
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ import {
|
||||
isOAuthProtectedResourceMetadataRequestForAuthEndpoint,
|
||||
isOAuthProtectedResourceRequest,
|
||||
} from '@/lib/oauth-protected';
|
||||
import { removeLeadingSlash, removeTrailingSlash } from '@/lib/paths';
|
||||
import { removeLeadingSlash, removeTrailingSlash, withTrailingSlash } from '@/lib/paths';
|
||||
import {
|
||||
getPreviewCookieResponse,
|
||||
getPreviewRequestIdentifier,
|
||||
@@ -496,6 +496,7 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
|
||||
events,
|
||||
isAiAgent,
|
||||
askVisitorAgent,
|
||||
isPage,
|
||||
} = encodePathInSiteContent(siteURLData, request);
|
||||
routeType = routeTypeFromPathname ?? routeType;
|
||||
// Only set for markdown routes, so it becomes part of their static cache key.
|
||||
@@ -609,6 +610,14 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
|
||||
}
|
||||
|
||||
// AI related headers
|
||||
if (isPage) {
|
||||
// Point agents to the most specific llms.txt that is always served for this page.
|
||||
const llmsTxtURL = new URL(
|
||||
'llms.txt',
|
||||
`${siteCanonicalURL.origin}${withTrailingSlash(siteURLData.basePath)}`
|
||||
);
|
||||
response.headers.append('link', `<${llmsTxtURL.toString()}>; rel="describedby"`);
|
||||
}
|
||||
// This one is technically useless, but is used by a bunch of scoring systems
|
||||
response.headers.set(
|
||||
'vary',
|
||||
@@ -802,6 +811,7 @@ function getSiteURLFromRequest(request: NextRequest): URLWithMode | null {
|
||||
const RSS_PATH_REGEX = /^((\S+)\/)?rss\.xml$/;
|
||||
const MARKDOWN_PATH_REGEX = /\.md$/;
|
||||
const LLMS_FULL_PATH_REGEX = /^llms-full\.txt\/\d+$/;
|
||||
const PAGE_LLMS_PATH_REGEX = /^(\S+)\/llms\.txt$/;
|
||||
const EMBED_PAGE_PATH_REGEX = /^~gitbook\/embed\/page(\/(\S*))?$/;
|
||||
const PATH_ALIASES: Record<string, string> = {
|
||||
'sitemap.md': 'llms.txt',
|
||||
@@ -847,6 +857,8 @@ function encodePathInSiteContent(
|
||||
isAiAgent?: boolean;
|
||||
/** Only set for the ask route, where the API classifies the asker from these headers. */
|
||||
askVisitorAgent?: AskVisitorAgent;
|
||||
/** Set for pages, which are listed in the llms.txt of their site space. */
|
||||
isPage?: boolean;
|
||||
} {
|
||||
let pathname = removeLeadingSlash(removeTrailingSlash(siteURLData.pathname));
|
||||
|
||||
@@ -901,6 +913,22 @@ function encodePathInSiteContent(
|
||||
};
|
||||
}
|
||||
|
||||
// The llms.txt of a page lists the pages under it (llms.txt at the space root is handled below).
|
||||
const pageLLMsMatch = pathname.match(PAGE_LLMS_PATH_REGEX);
|
||||
if (pageLLMsMatch) {
|
||||
return {
|
||||
pathname: `~gitbook/llms/${encodePagePath(pageLLMsMatch[1])}`,
|
||||
routeType: 'static',
|
||||
events: [
|
||||
{
|
||||
type: 'llms_request',
|
||||
llmsVariant: SiteInsightsLLMSVariant.Standard,
|
||||
location: eventLocation,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// If the pathname is an embedded page
|
||||
const embedPage = pathname.match(EMBED_PAGE_PATH_REGEX);
|
||||
if (embedPage) {
|
||||
@@ -983,6 +1011,7 @@ function encodePathInSiteContent(
|
||||
}`
|
||||
: `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`,
|
||||
routeType: 'static',
|
||||
isPage: typeof ask !== 'string',
|
||||
// Left undefined for non-agents to avoid splitting the static cache for them.
|
||||
isAiAgent: isAiAgent || undefined,
|
||||
askVisitorAgent:
|
||||
@@ -1011,7 +1040,7 @@ function encodePathInSiteContent(
|
||||
],
|
||||
};
|
||||
}
|
||||
return { pathname: encodePagePath(pathname) };
|
||||
return { pathname: encodePagePath(pathname), isPage: true };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+204
-115
@@ -1,56 +1,51 @@
|
||||
import assertNever from 'assert-never';
|
||||
import type { ListItem, Paragraph, Root, RootContent } from 'mdast';
|
||||
import { toMarkdown } from 'mdast-util-to-markdown';
|
||||
import type { ListItem, Paragraph, RootContent } from 'mdast';
|
||||
|
||||
import type { SiteSection, SiteSpace } from '@gitbook/api';
|
||||
import type { RevisionPageDocument, RevisionPageGroup, SiteSection, SiteSpace } from '@gitbook/api';
|
||||
|
||||
import { renderSiteAgentInstructions } from '@/lib/ask-prompt';
|
||||
import { type GitBookSiteContext, checkIsRootSiteContext } from '@/lib/context';
|
||||
import {
|
||||
type GitBookSiteContext,
|
||||
type SiteStructureNode,
|
||||
checkIsRootSiteContext,
|
||||
} from '@/lib/context';
|
||||
import { throwIfDataError } from '@/lib/data';
|
||||
import { type GitBookLinker, linkerWithMarkdownPages } from '@/lib/links';
|
||||
import { type FlatPageEntry, getIndexablePages } from '@/lib/sitemap';
|
||||
import { type LLMsIndexNode, renderLLMsIndex } from '@/lib/llms-index';
|
||||
import { joinPath } from '@/lib/paths';
|
||||
import { type FlatPageEntry, type PageTreeEntry, getIndexablePageTree } from '@/lib/sitemap';
|
||||
import {
|
||||
filterSiteSpacesByLocale,
|
||||
getFallbackSiteSpacePath,
|
||||
getLocalizedDescription,
|
||||
getLocalizedTitle,
|
||||
getSiteStructureSections,
|
||||
resolveSiteSpacePagePathDocumentOrGroup,
|
||||
} from '@/lib/sites';
|
||||
|
||||
/**
|
||||
* Generate a llms.txt file for the site.
|
||||
* Generate the llms.txt index for the part of the site it is served under: the whole site at
|
||||
* the root, a section, a site space, or the pages under `pagePath`.
|
||||
*/
|
||||
export async function serveLLMsTxt(baseContext: GitBookSiteContext) {
|
||||
const { site } = baseContext;
|
||||
|
||||
if (!checkIsRootSiteContext(baseContext)) {
|
||||
return new Response('llms.txt is only served from the root of the site', { status: 404 });
|
||||
export async function serveLLMsTxt(context: GitBookSiteContext, pagePath?: string) {
|
||||
const index = await getLLMsIndex(context, pagePath);
|
||||
if (!index) {
|
||||
return new Response('No llms.txt for this path', { status: 404 });
|
||||
}
|
||||
|
||||
const context = {
|
||||
...baseContext,
|
||||
linker: linkerWithMarkdownPages(baseContext.linker),
|
||||
};
|
||||
|
||||
const tree: Root = {
|
||||
type: 'root',
|
||||
children: [
|
||||
{
|
||||
type: 'heading',
|
||||
depth: 1,
|
||||
children: [{ type: 'text', value: site.title }],
|
||||
},
|
||||
...(await getNodesFromSiteStructure(context)),
|
||||
],
|
||||
};
|
||||
|
||||
let output = toMarkdown(tree, {
|
||||
bullet: '-',
|
||||
});
|
||||
|
||||
const rootURL = getLLMsTxtURLForPath(context, '');
|
||||
const agentInstructions = renderSiteAgentInstructions(context);
|
||||
if (agentInstructions) {
|
||||
output += `\n\n---\n\n${agentInstructions}`;
|
||||
}
|
||||
|
||||
const output = renderLLMsIndex({
|
||||
title: index.title,
|
||||
summary:
|
||||
index.url === rootURL
|
||||
? undefined
|
||||
: `This index lists part of the site. The index for the whole site is at ${rootURL}.`,
|
||||
url: index.url,
|
||||
nodes: index.nodes,
|
||||
footer: agentInstructions,
|
||||
});
|
||||
|
||||
return new Response(output, {
|
||||
headers: {
|
||||
@@ -59,113 +54,207 @@ export async function serveLLMsTxt(baseContext: GitBookSiteContext) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MDAST nodes from site structure.
|
||||
*/
|
||||
async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<RootContent[]> {
|
||||
async function getLLMsIndex(
|
||||
context: GitBookSiteContext,
|
||||
pagePath: string | undefined
|
||||
): Promise<{ title: string; url: string; nodes: LLMsIndexNode[] } | null> {
|
||||
const { site, siteSpace, linker } = context;
|
||||
|
||||
if (pagePath !== undefined) {
|
||||
const resolved = resolveSiteSpacePagePathDocumentOrGroup(
|
||||
siteSpace,
|
||||
context.revision.pages,
|
||||
pagePath
|
||||
);
|
||||
const entry = resolved
|
||||
? findPageTreeEntry(getIndexablePageTree(context.revision.pages), resolved.page.id)
|
||||
: null;
|
||||
if (!entry || entry.children.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const nodes = getNodesFromPageTree(entry.children, linker);
|
||||
return { title: entry.page.title, url: getPageLLMsTxtURL(linker, entry.page), nodes };
|
||||
}
|
||||
|
||||
if (checkIsRootSiteContext(context)) {
|
||||
return {
|
||||
title: site.title,
|
||||
url: getLLMsTxtURLForPath(context, ''),
|
||||
nodes: await getNodesFromSiteStructure(context),
|
||||
};
|
||||
}
|
||||
|
||||
const currentSection = context.sections?.current;
|
||||
// A section's default site space is served at the section's own path.
|
||||
if (currentSection && siteSpace.default) {
|
||||
const node = await getNodeFromSection(context, currentSection);
|
||||
return node?.indexURL
|
||||
? { title: node.title, url: node.indexURL, nodes: node.children }
|
||||
: null;
|
||||
}
|
||||
|
||||
const node = await getNodeFromSiteSpace(context, siteSpace);
|
||||
if (!node?.indexURL) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
title: currentSection
|
||||
? `${getLocalizedTitle(currentSection, context.locale)}: ${node.title}`
|
||||
: node.title,
|
||||
url: node.indexURL,
|
||||
nodes: node.children,
|
||||
};
|
||||
}
|
||||
|
||||
async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<LLMsIndexNode[]> {
|
||||
switch (context.structure.type) {
|
||||
case 'sections':
|
||||
return getNodesFromSections(
|
||||
return getNodesFromSectionNodes(
|
||||
context,
|
||||
getSiteStructureSections(context.structure, { ignoreGroups: true })
|
||||
getSiteStructureSections(context.structure, { ignoreGroups: false })
|
||||
);
|
||||
case 'siteSpaces':
|
||||
return getNodesFromSiteSpaces(context, context.structure.structure, {
|
||||
heading: true,
|
||||
});
|
||||
return getNodesFromSiteSpaces(context, context.structure.structure);
|
||||
default:
|
||||
assertNever(context.structure);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MDAST nodes from site sections.
|
||||
*/
|
||||
async function getNodesFromSections(
|
||||
async function getNodesFromSectionNodes(
|
||||
context: GitBookSiteContext,
|
||||
siteSections: SiteSection[]
|
||||
): Promise<RootContent[]> {
|
||||
const currentLanguage = context.locale;
|
||||
const all = await Promise.all(
|
||||
siteSections.map(async (siteSection): Promise<RootContent[]> => {
|
||||
const siteSpaceNodes = await getNodesFromSiteSpaces(context, siteSection.siteSpaces, {
|
||||
heading: false,
|
||||
});
|
||||
return [
|
||||
{
|
||||
type: 'heading',
|
||||
depth: 2,
|
||||
children: [
|
||||
{ type: 'text', value: getLocalizedTitle(siteSection, currentLanguage) },
|
||||
],
|
||||
},
|
||||
...siteSpaceNodes,
|
||||
];
|
||||
sectionNodes: SiteStructureNode[]
|
||||
): Promise<LLMsIndexNode[]> {
|
||||
const nodes = await Promise.all(
|
||||
sectionNodes.map(async (sectionNode): Promise<LLMsIndexNode | null> => {
|
||||
switch (sectionNode.object) {
|
||||
case 'site-section':
|
||||
return getNodeFromSection(context, sectionNode);
|
||||
case 'site-section-group': {
|
||||
const children = await getNodesFromSectionNodes(context, sectionNode.children);
|
||||
return children.length > 0
|
||||
? {
|
||||
kind: 'structure',
|
||||
title: getLocalizedTitle(sectionNode, context.locale),
|
||||
children,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
case 'site-external-link':
|
||||
return null;
|
||||
default:
|
||||
assertNever(sectionNode);
|
||||
}
|
||||
})
|
||||
);
|
||||
return all.flat();
|
||||
return nodes.filter((node) => node !== null);
|
||||
}
|
||||
|
||||
async function getNodeFromSection(
|
||||
context: GitBookSiteContext,
|
||||
section: SiteSection
|
||||
): Promise<LLMsIndexNode | null> {
|
||||
const defaultSiteSpace = section.siteSpaces.find((siteSpace) => siteSpace.default);
|
||||
const children = await getNodesFromSiteSpaces(context, section.siteSpaces);
|
||||
if (children.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'structure',
|
||||
title: getLocalizedTitle(section, context.locale),
|
||||
description: getLocalizedDescription(section, context.locale),
|
||||
indexURL: defaultSiteSpace
|
||||
? getLLMsTxtURLForPath(context, getFallbackSiteSpacePath(context, defaultSiteSpace))
|
||||
: undefined,
|
||||
children,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MDAST nodes from site spaces.
|
||||
* A single site space is listed directly, without a heading of its own.
|
||||
*/
|
||||
async function getNodesFromSiteSpaces(
|
||||
context: GitBookSiteContext,
|
||||
siteSpaces: SiteSpace[],
|
||||
options: {
|
||||
/**
|
||||
* Includes a heading for each site space.
|
||||
*/
|
||||
heading?: boolean;
|
||||
siteSpaces: SiteSpace[]
|
||||
): Promise<LLMsIndexNode[]> {
|
||||
const filtered = filterSiteSpacesByLocale(siteSpaces, context.locale);
|
||||
const nodes = (
|
||||
await Promise.all(filtered.map((siteSpace) => getNodeFromSiteSpace(context, siteSpace)))
|
||||
).filter((node) => node !== null);
|
||||
|
||||
return nodes.length === 1 ? nodes[0]!.children : nodes;
|
||||
}
|
||||
|
||||
async function getNodeFromSiteSpace(
|
||||
context: GitBookSiteContext,
|
||||
siteSpace: SiteSpace
|
||||
): Promise<LLMsIndexNode | null> {
|
||||
if (!siteSpace.urls.published) {
|
||||
return null;
|
||||
}
|
||||
): Promise<RootContent[]> {
|
||||
const { dataFetcher, linker } = context;
|
||||
|
||||
const filteredSiteSpaces = filterSiteSpacesByLocale(siteSpaces, context.locale);
|
||||
const revision =
|
||||
siteSpace.id === context.siteSpace.id
|
||||
? context.revision
|
||||
: await throwIfDataError(
|
||||
context.dataFetcher.getRevision({
|
||||
spaceId: siteSpace.space.id,
|
||||
revisionId: siteSpace.space.revision,
|
||||
})
|
||||
);
|
||||
const siteSpacePath = getFallbackSiteSpacePath(context, siteSpace);
|
||||
const linker = context.linker.withOtherSiteSpace({ spaceBasePath: siteSpacePath });
|
||||
|
||||
const all = await Promise.all(
|
||||
filteredSiteSpaces.map(async (siteSpace): Promise<RootContent[]> => {
|
||||
const siteSpaceUrl = siteSpace.urls.published;
|
||||
if (!siteSpaceUrl) {
|
||||
return [];
|
||||
}
|
||||
const revision = await throwIfDataError(
|
||||
dataFetcher.getRevision({
|
||||
spaceId: siteSpace.space.id,
|
||||
revisionId: siteSpace.space.revision,
|
||||
})
|
||||
);
|
||||
const pages = getIndexablePages(revision.pages);
|
||||
return {
|
||||
kind: 'structure',
|
||||
title: getLocalizedTitle(siteSpace, context.locale),
|
||||
indexURL: getLLMsTxtURLForPath(context, siteSpacePath),
|
||||
children: getNodesFromPageTree(getIndexablePageTree(revision.pages), linker),
|
||||
};
|
||||
}
|
||||
|
||||
const nodes: RootContent[] = [];
|
||||
function getNodesFromPageTree(entries: PageTreeEntry[], linker: GitBookLinker): LLMsIndexNode[] {
|
||||
const markdownLinker = linkerWithMarkdownPages(linker);
|
||||
const toNode = ({ page, children }: PageTreeEntry): LLMsIndexNode => ({
|
||||
kind: 'page',
|
||||
title: page.title,
|
||||
description: (page.type === 'document' && page.description) || undefined,
|
||||
url:
|
||||
page.type === 'document'
|
||||
? markdownLinker.toAbsoluteURL(
|
||||
markdownLinker.toPathForPagePath({ path: page.path })
|
||||
)
|
||||
: undefined,
|
||||
indexURL: children.length > 0 ? getPageLLMsTxtURL(linker, page) : undefined,
|
||||
children: children.map(toNode),
|
||||
});
|
||||
return entries.map(toNode);
|
||||
}
|
||||
|
||||
// Add the space title as a heading
|
||||
if (options.heading) {
|
||||
nodes.push({
|
||||
type: 'heading',
|
||||
depth: 2,
|
||||
children: [
|
||||
{
|
||||
type: 'text',
|
||||
value: getLocalizedTitle(siteSpace, context.locale),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
function findPageTreeEntry(entries: PageTreeEntry[], pageId: string): PageTreeEntry | null {
|
||||
for (const entry of entries) {
|
||||
if (entry.page.id === pageId) {
|
||||
return entry;
|
||||
}
|
||||
const found = findPageTreeEntry(entry.children, pageId);
|
||||
if (found) {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const siteSpaceLinker = linkerWithMarkdownPages(
|
||||
linker.withOtherSiteSpace({
|
||||
spaceBasePath: getFallbackSiteSpacePath(context, siteSpace),
|
||||
})
|
||||
);
|
||||
function getPageLLMsTxtURL(
|
||||
linker: GitBookLinker,
|
||||
page: RevisionPageDocument | RevisionPageGroup
|
||||
): string {
|
||||
return linker.toAbsoluteURL(linker.toPathInSpace(joinPath(page.path, 'llms.txt')));
|
||||
}
|
||||
|
||||
// Add the pages as a list
|
||||
nodes.push(...(await getMarkdownForPagesTree(pages, siteSpaceLinker)));
|
||||
|
||||
return nodes;
|
||||
})
|
||||
function getLLMsTxtURLForPath(context: GitBookSiteContext, pathInSite: string): string {
|
||||
return context.linker.toAbsoluteURL(
|
||||
context.linker.toPathInSite(joinPath(pathInSite, 'llms.txt'))
|
||||
);
|
||||
return all.flat();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user