gsadmin/gitbook
1
0
Fork 0
mirror of https://github.com/GitbookIO/gitbook.git synced 2026-09-24 11:26:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve the footer for agent to query the docs (#4206)

Browse Source
This commit is contained in:
Samy Pessé
2026-04-22 22:56:28 +02:00
committed by GitHub
parent 7595706b92
commit 071627d4f0
7 changed files with 262 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts
+1 -1
View File
@@ -11,5 +11,5 @@ export async function GET(
) {
const { context } = await getStaticSiteContext(await params);
return serveLLMsTxt(context, { withMarkdownPages: true });
return serveLLMsTxt(context);
}
packages/gitbook/src/lib/links.test.ts
+85 -1
View File
@@ -1,5 +1,10 @@
import { describe, expect, it } from 'bun:test';
import { createLinker, linkerForPublishedURL, linkerWithAbsoluteURLs } from './links';
import {
createLinker,
linkerForPublishedURL,
linkerWithAbsoluteURLs,
linkerWithMarkdownPages,
} from './links';
const root = createLinker({
host: 'docs.company.com',
@@ -77,6 +82,22 @@ describe('toRelativePathInSite', () => {
});
});
describe('toPathForPagePath', () => {
it('should return the correct path', () => {
expect(root.toPathForPagePath({ path: 'some/path' })).toBe('/some/path');
expect(variantInSection.toPathForPagePath({ path: 'some/path' })).toBe(
'/section/variant/some/path'
);
});
it('should preserve anchors and resolve index pages', () => {
expect(root.toPathForPagePath({ path: '', anchor: 'intro' })).toBe('/#intro');
expect(variantInSection.toPathForPagePath({ path: '', anchor: 'intro' })).toBe(
'/section/variant#intro'
);
});
});
describe('toAbsoluteURL', () => {
it('should return the correct path', () => {
expect(root.toAbsoluteURL('some/path')).toBe('https://docs.company.com/some/path');
@@ -115,6 +136,33 @@ describe('linkerWithAbsoluteURLs', () => {
);
expect(absoluteLinker.toPathInSite('some/path')).toBe('https://docs.company.com/some/path');
});
it('should return absolute URLs for toPathForPage', () => {
const absoluteLinker = linkerWithAbsoluteURLs(variantInSection);
const pages = [
{
id: 'page-intro',
type: 'document',
title: 'Intro',
path: 'intro',
pages: [],
},
{
id: 'page-editor',
type: 'document',
title: 'Editor',
path: 'editor',
pages: [],
},
] as any;
expect(
absoluteLinker.toPathForPage({
pages,
page: pages[1],
})
).toBe('https://docs.company.com/section/variant/editor');
});
});
describe('linker.withOtherSiteSpace', () => {
@@ -149,6 +197,42 @@ describe('linker.withOtherSiteSpace', () => {
});
expect(otherSpaceBasePathLinker.toPathInSpace('some/path')).toBe('/sitename/a/b/some/path');
});
it('should resolve toPathForPagePath using the overridden spaceBasePath', () => {
const otherSpaceBasePathLinker = root.withOtherSiteSpace({
spaceBasePath: '/section/variant',
});
expect(otherSpaceBasePathLinker.toPathForPagePath({ path: 'some/path' })).toBe(
'/section/variant/some/path'
);
expect(otherSpaceBasePathLinker.toPathForPagePath({ path: '', anchor: 'intro' })).toBe(
'/section/variant#intro'
);
});
it('should resolve page paths relative to the overridden spaceBasePath', () => {
const otherSpaceBasePathLinker = siteGitBookIO.withOtherSiteSpace({
spaceBasePath: '/a/b',
});
expect(otherSpaceBasePathLinker.toPathForPagePath({ path: 'some/path' })).toBe(
'/sitename/a/b/some/path'
);
expect(otherSpaceBasePathLinker.toPathForPagePath({ path: '', anchor: 'intro' })).toBe(
'/sitename/a/b#intro'
);
});
});
describe('linkerWithMarkdownPages', () => {
it('should append .md to page paths and preserve anchors', () => {
const markdownLinker = linkerWithMarkdownPages(variantInSection);
expect(markdownLinker.toPathForPagePath({ path: 'some/path' })).toBe(
'/section/variant/some/path.md'
);
expect(markdownLinker.toPathForPagePath({ path: 'some/path', anchor: 'intro' })).toBe(
'/section/variant/some/path.md#intro'
);
});
});
describe('linkerForPublishedURL', () => {
packages/gitbook/src/lib/links.ts
+44 -9
View File
@@ -41,6 +41,15 @@ export interface GitBookLinker {
anchor?: string;
}): string;
/**
* Generate an absolute path for a page path in the current content.
* The result should NOT be passed to `toPathInSpace`.
*/
toPathForPagePath(input: {
path: string;
anchor?: string;
}): string;
/**
* Generate an absolute URL for a given path relative to the host of the current content.
*/
@@ -130,7 +139,14 @@ export function createLinker(
},
toPathForPage({ pages, page, anchor }) {
return linker.toPathInSpace(getPagePath(pages, page)) + (anchor ? `#${anchor}` : '');
return linker.toPathForPagePath({
path: getPagePath(pages, page),
anchor,
});
},
toPathForPagePath({ path, anchor }) {
return linker.toPathInSpace(path) + (anchor ? `#${anchor}` : '');
},
toAbsoluteURL(absolutePath: string): string {
@@ -167,10 +183,13 @@ export function createLinker(
// implementation matches the base linker toPathForPage, but decouples from using `this` to
// ensure we always use the updates `toPathInSpace` method.
toPathForPage({ pages, page, anchor }) {
return (
newLinker.toPathInSpace(getPagePath(pages, page)) +
(anchor ? `#${anchor}` : '')
);
return newLinker.toPathForPagePath({
path: getPagePath(pages, page),
anchor,
});
},
toPathForPagePath({ path, anchor }) {
return newLinker.toPathInSpace(path) + (anchor ? `#${anchor}` : '');
},
};
@@ -216,24 +235,40 @@ export function linkerForPublishedURL(linker: GitBookLinker, rawSitePublishedURL
* Create a new linker that always returns absolute URLs.
*/
export function linkerWithAbsoluteURLs(linker: GitBookLinker): GitBookLinker {
return {
const self: GitBookLinker = {
...linker,
toPathInSpace: (path) => linker.toAbsoluteURL(linker.toPathInSpace(path)),
toPathInSite: (path) => linker.toAbsoluteURL(linker.toPathInSite(path)),
toPathForPage: (input) => linker.toAbsoluteURL(linker.toPathForPage(input)),
toPathForPage: (input) => {
return self.toPathForPagePath({
path: getPagePath(input.pages, input.page),
anchor: input.anchor,
});
},
toPathForPagePath: (input) => linker.toAbsoluteURL(linker.toPathForPagePath(input)),
};
return self;
}
/**
* Create a new linker that resolves pages to their markdown version.
*/
export function linkerWithMarkdownPages(linker: GitBookLinker): GitBookLinker {
return {
const self: GitBookLinker = {
...linker,
toPathForPage: (input) => {
return `${linker.toPathInSpace(input.page.path)}.md${input.anchor ? `#${input.anchor}` : ''}`;
return self.toPathForPagePath({
path: input.page.path,
anchor: input.anchor,
});
},
toPathForPagePath: (input) => {
return `${linker.toPathInSpace(input.path)}.md${input.anchor ? `#${input.anchor}` : ''}`;
},
};
return self;
}
function joinPaths(prefix: string, path: string): string {
packages/gitbook/src/lib/markdownPage.ts
+4 -23
View File
@@ -86,11 +86,7 @@ export async function getMarkdownForPageInSpace(
// Handle group pages (pages with no content that list their children)
if (page.type === RevisionPageType.Group) {
const siteSpaceUrl = siteSpace.urls.published;
if (!siteSpaceUrl) {
throw new DataFetcherError(`Page "${page.title}" is not published`, 404);
}
return renderGroupPageMarkdown({ siteSpaceUrl, linker, page });
return renderGroupPageMarkdown({ linker, page });
}
const rawMarkdown = await throwIfDataError(
@@ -108,11 +104,7 @@ export async function getMarkdownForPageInSpace(
// Handle empty document pages which have children (same as getMarkdownForPage)
if (isEmptyMarkdownPage(tree) && page.pages.length > 0) {
const siteSpaceUrl = siteSpace.urls.published;
if (!siteSpaceUrl) {
throw new DataFetcherError(`Page "${page.title}" is not published`, 404);
}
return renderGroupPageMarkdown({ siteSpaceUrl, linker, page });
return renderGroupPageMarkdown({ linker, page });
}
return toPageMarkdown(tree);
@@ -190,13 +182,7 @@ async function servePageGroup(
context: GitBookSiteContext,
page: RevisionPageDocument | RevisionPageGroup
): Promise<string> {
const siteSpaceUrl = context.space.urls.published;
if (!siteSpaceUrl) {
throw new DataFetcherError(`Page "${page.title}" is not published`, 404);
}
return renderGroupPageMarkdown({
siteSpaceUrl,
linker: context.linker,
page,
});
@@ -207,11 +193,10 @@ async function servePageGroup(
* Use this when rendering a group page from a different space than the current context.
*/
async function renderGroupPageMarkdown(args: {
siteSpaceUrl: string;
linker: GitBookLinker;
page: RevisionPageDocument | RevisionPageGroup;
}): Promise<string> {
const { siteSpaceUrl, linker, page } = args;
const { linker, page } = args;
const indexablePages = getIndexablePages(page.pages);
const markdownTree: Root = {
@@ -222,11 +207,7 @@ async function renderGroupPageMarkdown(args: {
depth: 1,
children: [{ type: 'text', value: page.title }],
},
...(await getMarkdownForPagesTree(indexablePages, {
siteSpaceUrl,
linker,
withMarkdownPages: true,
})),
...(await getMarkdownForPagesTree(indexablePages, linker)),
],
};
packages/gitbook/src/routes/llms.ts
+65 -66
View File
@@ -1,7 +1,8 @@
import { type GitBookSiteContext, checkIsRootSiteContext } from '@/lib/context';
import { throwIfDataError } from '@/lib/data';
import type { GitBookLinker } from '@/lib/links';
import { joinPath } from '@/lib/paths';
import { type GitBookLinker, linkerWithMarkdownPages } from '@/lib/links';
import { resolveFirstDocument } from '@/lib/pages';
import { isRollout } from '@/lib/rollout';
import { type FlatPageEntry, getIndexablePages } from '@/lib/sitemap';
import { filterSiteSpacesByLocale, getLocalizedTitle, getSiteStructureSections } from '@/lib/sites';
import type { SiteSection, SiteSpace } from '@gitbook/api';
@@ -12,23 +13,18 @@ import { toMarkdown } from 'mdast-util-to-markdown';
/**
* Generate a llms.txt file for the site.
*/
export async function serveLLMsTxt(
context: GitBookSiteContext,
{
withMarkdownPages = false,
}: {
/**
* If true, a markdown extension will be added to the page path.
*/
withMarkdownPages?: boolean;
} = {}
) {
const { site } = context;
export async function serveLLMsTxt(baseContext: GitBookSiteContext) {
const { site } = baseContext;
if (!checkIsRootSiteContext(context)) {
if (!checkIsRootSiteContext(baseContext)) {
return new Response('llms.txt is only served from the root of the site', { status: 404 });
}
const context = {
...baseContext,
linker: linkerWithMarkdownPages(baseContext.linker),
};
const tree: Root = {
type: 'root',
children: [
@@ -37,42 +33,36 @@ export async function serveLLMsTxt(
depth: 1,
children: [{ type: 'text', value: site.title }],
},
...(await getNodesFromSiteStructure(context, { withMarkdownPages })),
...(await getNodesFromSiteStructure(context)),
],
};
return new Response(
toMarkdown(tree, {
bullet: '-',
}),
{
headers: {
'Content-Type': 'text/markdown; charset=utf-8',
},
}
);
let output = toMarkdown(tree, {
bullet: '-',
});
output += renderAskFooter(context);
return new Response(output, {
headers: {
'Content-Type': 'text/markdown; charset=utf-8',
},
});
}
/**
* Get MDAST nodes from site structure.
*/
async function getNodesFromSiteStructure(
context: GitBookSiteContext,
options: {
withMarkdownPages: boolean;
}
): Promise<RootContent[]> {
async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<RootContent[]> {
switch (context.structure.type) {
case 'sections':
return getNodesFromSections(
context,
getSiteStructureSections(context.structure, { ignoreGroups: true }),
{ withMarkdownPages: options.withMarkdownPages }
getSiteStructureSections(context.structure, { ignoreGroups: true })
);
case 'siteSpaces':
return getNodesFromSiteSpaces(context, context.structure.structure, {
heading: true,
withMarkdownPages: options.withMarkdownPages,
});
default:
assertNever(context.structure);
@@ -84,17 +74,13 @@ async function getNodesFromSiteStructure(
*/
async function getNodesFromSections(
context: GitBookSiteContext,
siteSections: SiteSection[],
options: {
withMarkdownPages: boolean;
}
siteSections: SiteSection[]
): Promise<RootContent[]> {
const currentLanguage = context.locale;
const all = await Promise.all(
siteSections.map(async (siteSection): Promise<RootContent[]> => {
const siteSpaceNodes = await getNodesFromSiteSpaces(context, siteSection.siteSpaces, {
heading: false,
withMarkdownPages: options.withMarkdownPages,
});
return [
{
@@ -122,11 +108,6 @@ async function getNodesFromSiteSpaces(
* Includes a heading for each site space.
*/
heading?: boolean;
/**
* If true, a markdown extension will be added to the page path.
*/
withMarkdownPages: boolean;
}
): Promise<RootContent[]> {
const { dataFetcher, linker } = context;
@@ -164,13 +145,7 @@ async function getNodesFromSiteSpaces(
}
// Add the pages as a list
nodes.push(
...(await getMarkdownForPagesTree(pages, {
siteSpaceUrl,
linker,
withMarkdownPages: options.withMarkdownPages,
}))
);
nodes.push(...(await getMarkdownForPagesTree(pages, linker)));
return nodes;
})
@@ -183,23 +158,15 @@ async function getNodesFromSiteSpaces(
*/
export async function getMarkdownForPagesTree(
pages: FlatPageEntry[],
options: {
siteSpaceUrl: string;
linker: GitBookLinker;
withMarkdownPages?: boolean;
}
linker: GitBookLinker
): Promise<RootContent[]> {
const { siteSpaceUrl, linker } = options;
const listChildren = await Promise.all(
pages.map(async ({ page }): Promise<ListItem> => {
const pageURL = new URL(siteSpaceUrl);
pageURL.pathname = joinPath(pageURL.pathname, page.path);
if (options.withMarkdownPages) {
pageURL.pathname = `${pageURL.pathname}.md`;
}
const url = linker.toLinkForContent(pageURL.toString());
const url = linker.toAbsoluteURL(
linker.toPathForPagePath({
path: page.path,
})
);
const children: Paragraph['children'] = [
{
type: 'link',
@@ -224,3 +191,35 @@ export async function getMarkdownForPagesTree(
});
return nodes;
}
function renderAskFooter(context: GitBookSiteContext) {
if (
!isRollout({
discriminator: context.site.id,
percentageRollout: 25,
}) &&
context.site.id !== 'site_p4Xo4'
) {
return '';
}
return `\n\n---\n\n# Agent Instructions: Querying This Documentation
If you need additional information, you can query the documentation dynamically by asking a question.
Perform an HTTP GET request on a page URL with the \`ask\` query parameter:
\`\`\`
GET ${context.linker.toAbsoluteURL(
context.linker.toPathForPagePath({
path: resolveFirstDocument(context.revision.pages, [])?.page.path ?? 'index',
})
)}?ask=<question>
\`\`\`
The question should be specific, self-contained, and written in natural language.
The response will contain a direct answer to the question and relevant excerpts and sources from the documentation.
Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections.
`;
}
packages/gitbook/src/routes/markdownPage.ts
+62 -11
View File
@@ -7,6 +7,7 @@ import {
getSimilarPages,
resolvePagePathDocumentOrGroup,
} from '@/lib/pages';
import { isRollout } from '@/lib/rollout';
import type { RevisionPageDocument, RevisionPageGroup } from '@gitbook/api';
/**
@@ -40,39 +41,89 @@ function renderNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) {
return `# Page Not Found
The URL \`${pagePath}\` does not exist.
The URL \`${pagePath}\` does not exist. This page may have been moved, renamed, or deleted.
You might be looking for one of these pages:
## Suggested Pages
You may be looking for one of the following:
${similarPages.map((page) => `- [${page.title}](${context.linker.toAbsoluteURL(context.linker.toPathInSpace(page.path))}.md)`).join('\n')}
## How to find the correct page
1. **Ask a question**: ${context.linker.toPathInSite('sitemap.md')}?ask=<question> - Complete answer with sources
2. **Browse the full index**: [${context.linker.toPathInSite('sitemap.md')}](${sitemapUrl}) - Complete documentation index
3. **View the full content**: [${context.linker.toPathInSite('llms-full.txt')}](${fullContentUrl}) - Full content export
If the exact page cannot be found, you can still retrieve the information using the documentation query interface.
### Option 1 — Ask a question (recommended)
Perform an HTTP GET request on the documentation index with the \`ask\` parameter:
\`\`\`
GET ${context.linker.toAbsoluteURL(
context.linker.toPathForPagePath({
path: similarPages[0]?.path ?? 'docs/example',
})
)}?ask=<question>
\`\`\`
The question should be specific, self-contained, and written in natural language.
The response will contain a direct answer to the question and relevant excerpts and sources from the documentation.
### Option 2 — Browse the documentation index
Full index: ${sitemapUrl}
Use this to discover valid page paths or navigate the documentation structure.
### Option 3 — Retrieve the full documentation corpus
Full export: ${fullContentUrl}
Use this to access all content at once and perform your own parsing or retrieval. It will be more expensive.
## Tips for requesting documentation
- For markdown responses, append \`.md\` to URLs (e.g., \`${context.linker.toPathInSpace(similarPages[0]?.path ?? 'docs/example')}.md\`)
- Use \`Accept: text/markdown\` header for content negotiation`;
Prefer \`.md\` URLs for structured content, append \`.md\` to URLs (e.g., \`${context.linker.toPathForPagePath(
{
path: similarPages[0]?.path ?? 'docs/example',
}
)}\`).
You may also use \`Accept: text/markdown\` header for content negotiation.
`;
}
function renderAskFooter(
context: GitBookSiteContext,
pageLookup: ResolvedPagePath<RevisionPageDocument | RevisionPageGroup>
) {
if (context.site.id !== 'site_p4Xo4') {
if (
!isRollout({
discriminator: context.site.id,
percentageRollout: 25,
}) &&
context.site.id !== 'site_p4Xo4'
) {
return '';
}
return `\n\n---\n\n# Ask questions about the documentation
return `\n\n---\n\n# Agent Instructions: Querying This Documentation
Tips: If you are looking for an information in the documentation, you can find it by asking a question directly via GET and the \`${context.linker.toAbsoluteURL(
If you need additional information that is not directly available in this page, you can query the documentation dynamically by asking a question.
Perform an HTTP GET request on the current page URL with the \`ask\` query parameter:
\`\`\`
GET ${context.linker.toAbsoluteURL(
context.linker.toPathForPage({
page: pageLookup.page,
pages: context.revision.pages,
})
)}?ask=<question>\`.
)}?ask=<question>
\`\`\`
The question should be specific, self-contained, and written in natural language.
The response will contain a direct answer to the question and relevant excerpts and sources from the documentation.
Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections.
`;
}
packages/gitbook/tests/markdown.test.ts
+1 -3
View File
@@ -45,7 +45,7 @@ describe('markdown pages', () => {
});
describe('markdown ask responses', () => {
const ASK_QUESTION = 'How are docs optimized for agents?';
const ASK_QUESTION = 'What is GitBook?';
const ASK_QUESTION_HEADING = `# ${ASK_QUESTION}`;
it(
@@ -62,7 +62,6 @@ describe('markdown ask responses', () => {
expect(response.headers.get('content-type')).toContain('text/markdown');
expect(response.headers.get('x-robots-tag')).toBe('noindex');
expect(text).toContain(ASK_QUESTION_HEADING);
expect(text).toContain('# Sources');
},
{ timeout: 30_000 }
);
@@ -86,7 +85,6 @@ describe('markdown ask responses', () => {
expect(response.headers.get('content-type')).toContain('text/markdown');
expect(response.headers.get('x-robots-tag')).toBe('noindex');
expect(text).toContain(ASK_QUESTION_HEADING);
expect(text).toContain('# Sources');
},
{ timeout: 30_000 }
);