Compare commits

...

2 Commits

Author SHA1 Message Date
Steven H eb6d3f85bb changeset
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 10:35:14 +00:00
Steven H 6d056cdd76 Split llms.txt into scoped indexes past 100k characters
Large sites produced llms.txt files that agents truncate or skip. Once the
index exceeds 100,000 characters, the parts that save the most space are
replaced by links to their own llms.txt, served at the section, site space
or page path as the llms.txt spec allows. Pages also point to their
llms.txt with a describedby link header.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 10:35:14 +00:00
7 changed files with 686 additions and 117 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"gitbook": patch
---
Split llms.txt into per-section and per-page indexes when it exceeds 100,000 characters, and point pages to their llms.txt with a `describedby` link header.
@@ -0,0 +1,13 @@
import type { NextRequest } from 'next/server';
import { type RouteParams, getPagePathFromParams, getStaticSiteContext } from '@/app/utils';
import { serveLLMsTxt } from '@/routes/llms';
export const dynamic = 'force-static';
export async function GET(_request: NextRequest, { params }: { params: Promise<RouteParams> }) {
const { context } = await getStaticSiteContext(await params);
const pathname = getPagePathFromParams(await params);
return serveLLMsTxt(context, pathname);
}
+115
View File
@@ -0,0 +1,115 @@
import { describe, expect, it } from 'bun:test';
import { type LLMsIndexFile, type LLMsIndexNode, renderLLMsIndex } from './llms-index';
const ROOT = 'https://docs.example.com/llms.txt';
function page(path: string, children: LLMsIndexNode[] = []): LLMsIndexNode {
return {
kind: 'page',
title: `Page ${path}`,
description: `About ${path}`,
url: `https://docs.example.com/${path}.md`,
indexURL: `https://docs.example.com/${path}/llms.txt`,
children,
};
}
function section(path: string, children: LLMsIndexNode[], indexURL?: string): LLMsIndexNode {
return {
kind: 'structure',
title: `Section ${path}`,
indexURL: indexURL ?? `https://docs.example.com/${path}/llms.txt`,
children,
};
}
function pages(prefix: string, count: number): LLMsIndexNode[] {
return Array.from({ length: count }, (_, index) => page(`${prefix}/page-${index}`));
}
function file(nodes: LLMsIndexNode[], footer?: string): LLMsIndexFile {
return { title: 'Docs', url: ROOT, nodes, footer };
}
describe('renderLLMsIndex', () => {
it('renders everything inline when it fits', () => {
const output = renderLLMsIndex(
file([section('guides', [page('guides/start', [page('guides/start/install')])])])
);
expect(output).toBe(
[
'# Docs',
'',
'## Section guides',
'',
'- [Page guides/start](https://docs.example.com/guides/start.md): About guides/start',
' - [Page guides/start/install](https://docs.example.com/guides/start/install.md): About guides/start/install',
'',
].join('\n')
);
});
it('splits out the largest sections first', () => {
const output = renderLLMsIndex(
file([section('small', pages('small', 2)), section('large', pages('large', 50))]),
3_000
);
expect(output.length).toBeLessThanOrEqual(3_000);
expect(output).toContain('## Section small');
expect(output).toContain('Page small/page-1');
expect(output).toContain(
'## Section large\n\n- [Section large](https://docs.example.com/large/llms.txt)'
);
expect(output).not.toContain('Page large/page-1');
});
it('never links a node to the file it is rendered in', () => {
const output = renderLLMsIndex(
file([section('default', [page('a', pages('a', 30)), page('b', pages('b', 5))], ROOT)]),
2_000
);
expect(output.length).toBeLessThanOrEqual(2_000);
expect(output).toContain('## Section default');
expect(output).toContain('- [Page a](https://docs.example.com/a.md): About a');
expect(output).toContain(' - [Pages under Page a](https://docs.example.com/a/llms.txt)');
expect(output).not.toContain(ROOT);
});
it('serves an oversized index when nothing can be split out', () => {
const output = renderLLMsIndex(file(pages('flat', 50)), 1_000);
expect(output.length).toBeGreaterThan(1_000);
expect(output).toContain('Page flat/page-49');
});
it('counts the footer in the size', () => {
const nodes = [section('a', pages('a', 10)), section('b', pages('b', 10))];
const withoutFooter = renderLLMsIndex(file(nodes), 2_000);
const withFooter = renderLLMsIndex(file(nodes, 'x'.repeat(1_000)), 2_000);
expect(withoutFooter).toContain('Page a/page-1');
expect(withFooter.length).toBeLessThanOrEqual(2_000);
expect(withFooter).toContain('- [Section a](https://docs.example.com/a/llms.txt)');
});
it('fits large sites within the default limit', () => {
const nodes = Array.from({ length: 20 }, (_, sectionIndex) =>
section(
`s${sectionIndex}`,
Array.from({ length: 20 }, (_, groupIndex) =>
page(
`s${sectionIndex}/g${groupIndex}`,
pages(`s${sectionIndex}/g${groupIndex}`, 10)
)
)
)
);
const output = renderLLMsIndex(file(nodes));
expect(output.length).toBeLessThanOrEqual(100_000);
});
});
+291
View File
@@ -0,0 +1,291 @@
import type { Heading, List, ListItem, Paragraph, Root, RootContent } from 'mdast';
import { toMarkdown } from 'mdast-util-to-markdown';
/**
* Agents truncate or skip llms.txt files past this size, so larger indexes are split into
* scoped child files (see https://llmstxt.org/ on files at subpaths).
*/
export const LLMS_TXT_MAX_CHARACTERS = 100_000;
export type LLMsIndexNode = {
/** A navigation level (section group, section, site space) rendered as a heading, or a page rendered as a list item. */
kind: 'structure' | 'page';
title: string;
description?: string;
/** Markdown URL of the page itself. */
url?: string;
/** URL of the llms.txt scoped to this node, when it can be split out. */
indexURL?: string;
children: LLMsIndexNode[];
};
export type LLMsIndexFile = {
title: string;
/** Rendered as the summary blockquote below the title. */
summary?: string;
/** URL this file is served at, so nodes sharing it are never collapsed into a link to itself. */
url: string;
nodes: LLMsIndexNode[];
/** Appended after the index, and counted in its size. */
footer?: string;
};
/**
* Render an llms.txt index, collapsing the nodes that save the most space into links to
* their own llms.txt until the file fits within the limit.
*/
export function renderLLMsIndex(
file: LLMsIndexFile,
limit: number = LLMS_TXT_MAX_CHARACTERS
): string {
const collapsed = new Set<LLMsIndexNode>();
const candidates: LLMsIndexNode[] = [];
const visit = (node: LLMsIndexNode) => {
// A node sharing this file's URL would collapse into a link to the file itself.
if (node.indexURL && node.indexURL !== file.url && node.children.length > 0) {
candidates.push(node);
}
node.children.forEach(visit);
};
file.nodes.forEach(visit);
const ownSizes = new Map<LLMsIndexNode, number>();
const collapsedSizes = new Map<LLMsIndexNode, number>();
const ownSize = (node: LLMsIndexNode) =>
getCached(ownSizes, node, () => measureNode(node, false));
const collapsedSize = (node: LLMsIndexNode) =>
getCached(collapsedSizes, node, () => measureNode(node, true));
const baseSize = renderFile(file, collapsed, []).length;
// Sizes are estimated per node to stay linear in the number of pages; the exact render
// only runs once the estimate fits.
while (true) {
const sizes = new Map<LLMsIndexNode, number>();
const sizeOf = (node: LLMsIndexNode): number => {
const size = collapsed.has(node)
? collapsedSize(node)
: node.children.reduce((total, child) => total + sizeOf(child), ownSize(node));
sizes.set(node, size);
return size;
};
const estimated = file.nodes.reduce((total, node) => total + sizeOf(node), baseSize);
if (estimated <= limit) {
const output = renderFile(file, collapsed, file.nodes);
if (output.length <= limit) {
return output;
}
}
let best: LLMsIndexNode | null = null;
let bestSaving = 0;
for (const node of candidates) {
const size = sizes.get(node);
// Nodes under a collapsed ancestor weren't measured in this pass.
if (size === undefined || collapsed.has(node)) {
continue;
}
const saving = size - collapsedSize(node);
if (saving > bestSaving) {
best = node;
bestSaving = saving;
}
}
if (!best) {
// Nothing left to split out: serve the smallest index we can build.
return renderFile(file, collapsed, file.nodes);
}
collapsed.add(best);
}
}
function getCached<K, V>(cache: Map<K, V>, key: K, compute: () => V): V {
let value = cache.get(key);
if (value === undefined) {
value = compute();
cache.set(key, value);
}
return value;
}
/**
* Approximate the characters a node adds on its own, excluding its children.
*/
function measureNode(node: LLMsIndexNode, asCollapsed: boolean): number {
const blocks =
node.kind === 'structure'
? asCollapsed
? toCollapsedStructureBlocks(node, 2)
: toStructureBlocks(node, 2, new Set(), false)
: [toList([asCollapsed ? toCollapsedItem(node) : toPageItem(node, new Set(), false)])];
// Nested list items are indented, which the standalone render doesn't account for.
return toMarkdown({ type: 'root', children: blocks }, { bullet: '-' }).length + 4;
}
function renderFile(
file: LLMsIndexFile,
collapsed: Set<LLMsIndexNode>,
nodes: LLMsIndexNode[]
): string {
const tree: Root = {
type: 'root',
children: [
{ type: 'heading', depth: 1, children: [{ type: 'text', value: file.title }] },
...(file.summary
? [
{
type: 'blockquote' as const,
children: [toParagraph(file.summary)],
},
]
: []),
...toBlocks(nodes, 2, collapsed),
],
};
let output = toMarkdown(tree, { bullet: '-' });
if (file.footer) {
output += `\n\n---\n\n${file.footer}`;
}
return output;
}
/**
* Render sibling nodes, grouping consecutive list items into a single list.
*/
function toBlocks(
nodes: LLMsIndexNode[],
depth: number,
collapsed: Set<LLMsIndexNode>
): RootContent[] {
const blocks: RootContent[] = [];
let items: ListItem[] = [];
const flush = () => {
if (items.length > 0) {
blocks.push(toList(items));
items = [];
}
};
for (const node of nodes) {
if (node.kind === 'structure') {
flush();
blocks.push(
...(collapsed.has(node)
? toCollapsedStructureBlocks(node, depth)
: toStructureBlocks(node, depth, collapsed, true))
);
} else if (collapsed.has(node)) {
items.push(toCollapsedItem(node));
} else {
items.push(toPageItem(node, collapsed, true));
}
}
flush();
return blocks;
}
function toStructureBlocks(
node: LLMsIndexNode,
depth: number,
collapsed: Set<LLMsIndexNode>,
withChildren: boolean
): RootContent[] {
return [
{
type: 'heading',
depth: Math.min(depth, 6) as Heading['depth'],
children: [{ type: 'text', value: node.title }],
},
...(node.description ? [toParagraph(node.description)] : []),
...(withChildren ? toBlocks(node.children, depth + 1, collapsed) : []),
];
}
/**
* Keep the heading so the link isn't read as part of the previous section's list.
*/
function toCollapsedStructureBlocks(node: LLMsIndexNode, depth: number): RootContent[] {
return [
{
type: 'heading',
depth: Math.min(depth, 6) as Heading['depth'],
children: [{ type: 'text', value: node.title }],
},
toList([toCollapsedItem(node)]),
];
}
function toPageItem(
node: LLMsIndexNode,
collapsed: Set<LLMsIndexNode>,
withChildren: boolean
): ListItem {
const children: ListItem['children'] = [toLinkParagraph(node, node.url)];
if (withChildren && node.children.length > 0) {
children.push(
toList(
node.children.map((child) =>
collapsed.has(child)
? toCollapsedItem(child)
: toPageItem(child, collapsed, true)
)
)
);
}
return { type: 'listItem', spread: false, children };
}
/**
* A page that is also a document keeps its own link, with its subtree moved to its index.
*/
function toCollapsedItem(node: LLMsIndexNode): ListItem {
if (node.kind === 'page' && node.url) {
return {
type: 'listItem',
spread: false,
children: [
toLinkParagraph(node, node.url),
toList([
{
type: 'listItem',
spread: false,
children: [
toLinkParagraph({ title: `Pages under ${node.title}` }, node.indexURL),
],
},
]),
],
};
}
return {
type: 'listItem',
spread: false,
children: [toLinkParagraph(node, node.indexURL)],
};
}
function toLinkParagraph(
node: Pick<LLMsIndexNode, 'title' | 'description'>,
url: string | undefined
): Paragraph {
const children: Paragraph['children'] = [
url
? { type: 'link', url, children: [{ type: 'text', value: node.title }] }
: { type: 'text', value: node.title },
];
if (node.description) {
children.push({ type: 'text', value: `: ${node.description}` });
}
return { type: 'paragraph', children };
}
function toParagraph(value: string): Paragraph {
return { type: 'paragraph', children: [{ type: 'text', value }] };
}
function toList(children: ListItem[]): List {
return { type: 'list', spread: false, children };
}
+27
View File
@@ -47,3 +47,30 @@ function flattenPages(
export function getIndexablePages(rootPages: RevisionPage[]) {
return flattenPages(rootPages, (page, ancestors) => isPageIndexable(ancestors, page));
}
export type PageTreeEntry = {
page: RevisionPageDocument | RevisionPageGroup;
children: PageTreeEntry[];
};
/**
* Get all indexable pages from a revision as a tree, keeping groups that contain indexable pages.
*/
export function getIndexablePageTree(
rootPages: RevisionPage[],
ancestors: (RevisionPageDocument | RevisionPageGroup)[] = []
): PageTreeEntry[] {
return rootPages.flatMap((page): PageTreeEntry[] => {
if (page.type !== 'document' && page.type !== 'group') {
return [];
}
if (!isPageIndexable(ancestors, page)) {
return [];
}
const children = getIndexablePageTree(page.pages, [...ancestors, page]);
if (page.type === 'group' && children.length === 0) {
return [];
}
return [{ page, children }];
});
}
+31 -2
View File
@@ -47,7 +47,7 @@ import {
isOAuthProtectedResourceMetadataRequestForAuthEndpoint,
isOAuthProtectedResourceRequest,
} from '@/lib/oauth-protected';
import { removeLeadingSlash, removeTrailingSlash } from '@/lib/paths';
import { removeLeadingSlash, removeTrailingSlash, withTrailingSlash } from '@/lib/paths';
import {
getPreviewCookieResponse,
getPreviewRequestIdentifier,
@@ -496,6 +496,7 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
events,
isAiAgent,
askVisitorAgent,
isPage,
} = encodePathInSiteContent(siteURLData, request);
routeType = routeTypeFromPathname ?? routeType;
// Only set for markdown routes, so it becomes part of their static cache key.
@@ -609,6 +610,14 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
}
// AI related headers
if (isPage) {
// Point agents to the most specific llms.txt that is always served for this page.
const llmsTxtURL = new URL(
'llms.txt',
`${siteCanonicalURL.origin}${withTrailingSlash(siteURLData.basePath)}`
);
response.headers.append('link', `<${llmsTxtURL.toString()}>; rel="describedby"`);
}
// This one is technically useless, but is used by a bunch of scoring systems
response.headers.set(
'vary',
@@ -802,6 +811,7 @@ function getSiteURLFromRequest(request: NextRequest): URLWithMode | null {
const RSS_PATH_REGEX = /^((\S+)\/)?rss\.xml$/;
const MARKDOWN_PATH_REGEX = /\.md$/;
const LLMS_FULL_PATH_REGEX = /^llms-full\.txt\/\d+$/;
const PAGE_LLMS_PATH_REGEX = /^(\S+)\/llms\.txt$/;
const EMBED_PAGE_PATH_REGEX = /^~gitbook\/embed\/page(\/(\S*))?$/;
const PATH_ALIASES: Record<string, string> = {
'sitemap.md': 'llms.txt',
@@ -847,6 +857,8 @@ function encodePathInSiteContent(
isAiAgent?: boolean;
/** Only set for the ask route, where the API classifies the asker from these headers. */
askVisitorAgent?: AskVisitorAgent;
/** Set for pages, which are listed in the llms.txt of their site space. */
isPage?: boolean;
} {
let pathname = removeLeadingSlash(removeTrailingSlash(siteURLData.pathname));
@@ -901,6 +913,22 @@ function encodePathInSiteContent(
};
}
// The llms.txt of a page lists the pages under it (llms.txt at the space root is handled below).
const pageLLMsMatch = pathname.match(PAGE_LLMS_PATH_REGEX);
if (pageLLMsMatch) {
return {
pathname: `~gitbook/llms/${encodePagePath(pageLLMsMatch[1])}`,
routeType: 'static',
events: [
{
type: 'llms_request',
llmsVariant: SiteInsightsLLMSVariant.Standard,
location: eventLocation,
},
],
};
}
// If the pathname is an embedded page
const embedPage = pathname.match(EMBED_PAGE_PATH_REGEX);
if (embedPage) {
@@ -983,6 +1011,7 @@ function encodePathInSiteContent(
}`
: `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`,
routeType: 'static',
isPage: typeof ask !== 'string',
// Left undefined for non-agents to avoid splitting the static cache for them.
isAiAgent: isAiAgent || undefined,
askVisitorAgent:
@@ -1011,7 +1040,7 @@ function encodePathInSiteContent(
],
};
}
return { pathname: encodePagePath(pathname) };
return { pathname: encodePagePath(pathname), isPage: true };
}
}
}
+204 -115
View File
@@ -1,56 +1,51 @@
import assertNever from 'assert-never';
import type { ListItem, Paragraph, Root, RootContent } from 'mdast';
import { toMarkdown } from 'mdast-util-to-markdown';
import type { ListItem, Paragraph, RootContent } from 'mdast';
import type { SiteSection, SiteSpace } from '@gitbook/api';
import type { RevisionPageDocument, RevisionPageGroup, SiteSection, SiteSpace } from '@gitbook/api';
import { renderSiteAgentInstructions } from '@/lib/ask-prompt';
import { type GitBookSiteContext, checkIsRootSiteContext } from '@/lib/context';
import {
type GitBookSiteContext,
type SiteStructureNode,
checkIsRootSiteContext,
} from '@/lib/context';
import { throwIfDataError } from '@/lib/data';
import { type GitBookLinker, linkerWithMarkdownPages } from '@/lib/links';
import { type FlatPageEntry, getIndexablePages } from '@/lib/sitemap';
import { type LLMsIndexNode, renderLLMsIndex } from '@/lib/llms-index';
import { joinPath } from '@/lib/paths';
import { type FlatPageEntry, type PageTreeEntry, getIndexablePageTree } from '@/lib/sitemap';
import {
filterSiteSpacesByLocale,
getFallbackSiteSpacePath,
getLocalizedDescription,
getLocalizedTitle,
getSiteStructureSections,
resolveSiteSpacePagePathDocumentOrGroup,
} from '@/lib/sites';
/**
* Generate a llms.txt file for the site.
* Generate the llms.txt index for the part of the site it is served under: the whole site at
* the root, a section, a site space, or the pages under `pagePath`.
*/
export async function serveLLMsTxt(baseContext: GitBookSiteContext) {
const { site } = baseContext;
if (!checkIsRootSiteContext(baseContext)) {
return new Response('llms.txt is only served from the root of the site', { status: 404 });
export async function serveLLMsTxt(context: GitBookSiteContext, pagePath?: string) {
const index = await getLLMsIndex(context, pagePath);
if (!index) {
return new Response('No llms.txt for this path', { status: 404 });
}
const context = {
...baseContext,
linker: linkerWithMarkdownPages(baseContext.linker),
};
const tree: Root = {
type: 'root',
children: [
{
type: 'heading',
depth: 1,
children: [{ type: 'text', value: site.title }],
},
...(await getNodesFromSiteStructure(context)),
],
};
let output = toMarkdown(tree, {
bullet: '-',
});
const rootURL = getLLMsTxtURLForPath(context, '');
const agentInstructions = renderSiteAgentInstructions(context);
if (agentInstructions) {
output += `\n\n---\n\n${agentInstructions}`;
}
const output = renderLLMsIndex({
title: index.title,
summary:
index.url === rootURL
? undefined
: `This index lists part of the site. The index for the whole site is at ${rootURL}.`,
url: index.url,
nodes: index.nodes,
footer: agentInstructions,
});
return new Response(output, {
headers: {
@@ -59,113 +54,207 @@ export async function serveLLMsTxt(baseContext: GitBookSiteContext) {
});
}
/**
* Get MDAST nodes from site structure.
*/
async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<RootContent[]> {
async function getLLMsIndex(
context: GitBookSiteContext,
pagePath: string | undefined
): Promise<{ title: string; url: string; nodes: LLMsIndexNode[] } | null> {
const { site, siteSpace, linker } = context;
if (pagePath !== undefined) {
const resolved = resolveSiteSpacePagePathDocumentOrGroup(
siteSpace,
context.revision.pages,
pagePath
);
const entry = resolved
? findPageTreeEntry(getIndexablePageTree(context.revision.pages), resolved.page.id)
: null;
if (!entry || entry.children.length === 0) {
return null;
}
const nodes = getNodesFromPageTree(entry.children, linker);
return { title: entry.page.title, url: getPageLLMsTxtURL(linker, entry.page), nodes };
}
if (checkIsRootSiteContext(context)) {
return {
title: site.title,
url: getLLMsTxtURLForPath(context, ''),
nodes: await getNodesFromSiteStructure(context),
};
}
const currentSection = context.sections?.current;
// A section's default site space is served at the section's own path.
if (currentSection && siteSpace.default) {
const node = await getNodeFromSection(context, currentSection);
return node?.indexURL
? { title: node.title, url: node.indexURL, nodes: node.children }
: null;
}
const node = await getNodeFromSiteSpace(context, siteSpace);
if (!node?.indexURL) {
return null;
}
return {
title: currentSection
? `${getLocalizedTitle(currentSection, context.locale)}: ${node.title}`
: node.title,
url: node.indexURL,
nodes: node.children,
};
}
async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<LLMsIndexNode[]> {
switch (context.structure.type) {
case 'sections':
return getNodesFromSections(
return getNodesFromSectionNodes(
context,
getSiteStructureSections(context.structure, { ignoreGroups: true })
getSiteStructureSections(context.structure, { ignoreGroups: false })
);
case 'siteSpaces':
return getNodesFromSiteSpaces(context, context.structure.structure, {
heading: true,
});
return getNodesFromSiteSpaces(context, context.structure.structure);
default:
assertNever(context.structure);
}
}
/**
* Get MDAST nodes from site sections.
*/
async function getNodesFromSections(
async function getNodesFromSectionNodes(
context: GitBookSiteContext,
siteSections: SiteSection[]
): Promise<RootContent[]> {
const currentLanguage = context.locale;
const all = await Promise.all(
siteSections.map(async (siteSection): Promise<RootContent[]> => {
const siteSpaceNodes = await getNodesFromSiteSpaces(context, siteSection.siteSpaces, {
heading: false,
});
return [
{
type: 'heading',
depth: 2,
children: [
{ type: 'text', value: getLocalizedTitle(siteSection, currentLanguage) },
],
},
...siteSpaceNodes,
];
sectionNodes: SiteStructureNode[]
): Promise<LLMsIndexNode[]> {
const nodes = await Promise.all(
sectionNodes.map(async (sectionNode): Promise<LLMsIndexNode | null> => {
switch (sectionNode.object) {
case 'site-section':
return getNodeFromSection(context, sectionNode);
case 'site-section-group': {
const children = await getNodesFromSectionNodes(context, sectionNode.children);
return children.length > 0
? {
kind: 'structure',
title: getLocalizedTitle(sectionNode, context.locale),
children,
}
: null;
}
case 'site-external-link':
return null;
default:
assertNever(sectionNode);
}
})
);
return all.flat();
return nodes.filter((node) => node !== null);
}
async function getNodeFromSection(
context: GitBookSiteContext,
section: SiteSection
): Promise<LLMsIndexNode | null> {
const defaultSiteSpace = section.siteSpaces.find((siteSpace) => siteSpace.default);
const children = await getNodesFromSiteSpaces(context, section.siteSpaces);
if (children.length === 0) {
return null;
}
return {
kind: 'structure',
title: getLocalizedTitle(section, context.locale),
description: getLocalizedDescription(section, context.locale),
indexURL: defaultSiteSpace
? getLLMsTxtURLForPath(context, getFallbackSiteSpacePath(context, defaultSiteSpace))
: undefined,
children,
};
}
/**
* Get MDAST nodes from site spaces.
* A single site space is listed directly, without a heading of its own.
*/
async function getNodesFromSiteSpaces(
context: GitBookSiteContext,
siteSpaces: SiteSpace[],
options: {
/**
* Includes a heading for each site space.
*/
heading?: boolean;
siteSpaces: SiteSpace[]
): Promise<LLMsIndexNode[]> {
const filtered = filterSiteSpacesByLocale(siteSpaces, context.locale);
const nodes = (
await Promise.all(filtered.map((siteSpace) => getNodeFromSiteSpace(context, siteSpace)))
).filter((node) => node !== null);
return nodes.length === 1 ? nodes[0]!.children : nodes;
}
async function getNodeFromSiteSpace(
context: GitBookSiteContext,
siteSpace: SiteSpace
): Promise<LLMsIndexNode | null> {
if (!siteSpace.urls.published) {
return null;
}
): Promise<RootContent[]> {
const { dataFetcher, linker } = context;
const filteredSiteSpaces = filterSiteSpacesByLocale(siteSpaces, context.locale);
const revision =
siteSpace.id === context.siteSpace.id
? context.revision
: await throwIfDataError(
context.dataFetcher.getRevision({
spaceId: siteSpace.space.id,
revisionId: siteSpace.space.revision,
})
);
const siteSpacePath = getFallbackSiteSpacePath(context, siteSpace);
const linker = context.linker.withOtherSiteSpace({ spaceBasePath: siteSpacePath });
const all = await Promise.all(
filteredSiteSpaces.map(async (siteSpace): Promise<RootContent[]> => {
const siteSpaceUrl = siteSpace.urls.published;
if (!siteSpaceUrl) {
return [];
}
const revision = await throwIfDataError(
dataFetcher.getRevision({
spaceId: siteSpace.space.id,
revisionId: siteSpace.space.revision,
})
);
const pages = getIndexablePages(revision.pages);
return {
kind: 'structure',
title: getLocalizedTitle(siteSpace, context.locale),
indexURL: getLLMsTxtURLForPath(context, siteSpacePath),
children: getNodesFromPageTree(getIndexablePageTree(revision.pages), linker),
};
}
const nodes: RootContent[] = [];
function getNodesFromPageTree(entries: PageTreeEntry[], linker: GitBookLinker): LLMsIndexNode[] {
const markdownLinker = linkerWithMarkdownPages(linker);
const toNode = ({ page, children }: PageTreeEntry): LLMsIndexNode => ({
kind: 'page',
title: page.title,
description: (page.type === 'document' && page.description) || undefined,
url:
page.type === 'document'
? markdownLinker.toAbsoluteURL(
markdownLinker.toPathForPagePath({ path: page.path })
)
: undefined,
indexURL: children.length > 0 ? getPageLLMsTxtURL(linker, page) : undefined,
children: children.map(toNode),
});
return entries.map(toNode);
}
// Add the space title as a heading
if (options.heading) {
nodes.push({
type: 'heading',
depth: 2,
children: [
{
type: 'text',
value: getLocalizedTitle(siteSpace, context.locale),
},
],
});
}
function findPageTreeEntry(entries: PageTreeEntry[], pageId: string): PageTreeEntry | null {
for (const entry of entries) {
if (entry.page.id === pageId) {
return entry;
}
const found = findPageTreeEntry(entry.children, pageId);
if (found) {
return found;
}
}
return null;
}
const siteSpaceLinker = linkerWithMarkdownPages(
linker.withOtherSiteSpace({
spaceBasePath: getFallbackSiteSpacePath(context, siteSpace),
})
);
function getPageLLMsTxtURL(
linker: GitBookLinker,
page: RevisionPageDocument | RevisionPageGroup
): string {
return linker.toAbsoluteURL(linker.toPathInSpace(joinPath(page.path, 'llms.txt')));
}
// Add the pages as a list
nodes.push(...(await getMarkdownForPagesTree(pages, siteSpaceLinker)));
return nodes;
})
function getLLMsTxtURLForPath(context: GitBookSiteContext, pathInSite: string): string {
return context.linker.toAbsoluteURL(
context.linker.toPathInSite(joinPath(pathInSite, 'llms.txt'))
);
return all.flat();
}
/**