diff --git a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts index 475297000..754b2d038 100644 --- a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts +++ b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts @@ -11,5 +11,5 @@ export async function GET( ) { const { context } = await getStaticSiteContext(await params); - return serveLLMsTxt(context, { withMarkdownPages: true }); + return serveLLMsTxt(context); } diff --git a/packages/gitbook/src/lib/links.test.ts b/packages/gitbook/src/lib/links.test.ts index 342fbae85..047dcf7c8 100644 --- a/packages/gitbook/src/lib/links.test.ts +++ b/packages/gitbook/src/lib/links.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'bun:test'; -import { createLinker, linkerForPublishedURL, linkerWithAbsoluteURLs } from './links'; +import { + createLinker, + linkerForPublishedURL, + linkerWithAbsoluteURLs, + linkerWithMarkdownPages, +} from './links'; const root = createLinker({ host: 'docs.company.com', @@ -77,6 +82,22 @@ describe('toRelativePathInSite', () => { }); }); +describe('toPathForPagePath', () => { + it('should return the correct path', () => { + expect(root.toPathForPagePath({ path: 'some/path' })).toBe('/some/path'); + expect(variantInSection.toPathForPagePath({ path: 'some/path' })).toBe( + '/section/variant/some/path' + ); + }); + + it('should preserve anchors and resolve index pages', () => { + expect(root.toPathForPagePath({ path: '', anchor: 'intro' })).toBe('/#intro'); + expect(variantInSection.toPathForPagePath({ path: '', anchor: 'intro' })).toBe( + '/section/variant#intro' + ); + }); +}); + describe('toAbsoluteURL', () => { it('should return the correct path', () => { expect(root.toAbsoluteURL('some/path')).toBe('https://docs.company.com/some/path'); @@ -115,6 +136,33 @@ describe('linkerWithAbsoluteURLs', () => { ); expect(absoluteLinker.toPathInSite('some/path')).toBe('https://docs.company.com/some/path'); }); + + it('should return absolute URLs for toPathForPage', () => { + const absoluteLinker = linkerWithAbsoluteURLs(variantInSection); + const pages = [ + { + id: 'page-intro', + type: 'document', + title: 'Intro', + path: 'intro', + pages: [], + }, + { + id: 'page-editor', + type: 'document', + title: 'Editor', + path: 'editor', + pages: [], + }, + ] as any; + + expect( + absoluteLinker.toPathForPage({ + pages, + page: pages[1], + }) + ).toBe('https://docs.company.com/section/variant/editor'); + }); }); describe('linker.withOtherSiteSpace', () => { @@ -149,6 +197,42 @@ describe('linker.withOtherSiteSpace', () => { }); expect(otherSpaceBasePathLinker.toPathInSpace('some/path')).toBe('/sitename/a/b/some/path'); }); + + it('should resolve toPathForPagePath using the overridden spaceBasePath', () => { + const otherSpaceBasePathLinker = root.withOtherSiteSpace({ + spaceBasePath: '/section/variant', + }); + expect(otherSpaceBasePathLinker.toPathForPagePath({ path: 'some/path' })).toBe( + '/section/variant/some/path' + ); + expect(otherSpaceBasePathLinker.toPathForPagePath({ path: '', anchor: 'intro' })).toBe( + '/section/variant#intro' + ); + }); + + it('should resolve page paths relative to the overridden spaceBasePath', () => { + const otherSpaceBasePathLinker = siteGitBookIO.withOtherSiteSpace({ + spaceBasePath: '/a/b', + }); + expect(otherSpaceBasePathLinker.toPathForPagePath({ path: 'some/path' })).toBe( + '/sitename/a/b/some/path' + ); + expect(otherSpaceBasePathLinker.toPathForPagePath({ path: '', anchor: 'intro' })).toBe( + '/sitename/a/b#intro' + ); + }); +}); + +describe('linkerWithMarkdownPages', () => { + it('should append .md to page paths and preserve anchors', () => { + const markdownLinker = linkerWithMarkdownPages(variantInSection); + expect(markdownLinker.toPathForPagePath({ path: 'some/path' })).toBe( + '/section/variant/some/path.md' + ); + expect(markdownLinker.toPathForPagePath({ path: 'some/path', anchor: 'intro' })).toBe( + '/section/variant/some/path.md#intro' + ); + }); }); describe('linkerForPublishedURL', () => { diff --git a/packages/gitbook/src/lib/links.ts b/packages/gitbook/src/lib/links.ts index 81d9b691c..87c047f0e 100644 --- a/packages/gitbook/src/lib/links.ts +++ b/packages/gitbook/src/lib/links.ts @@ -41,6 +41,15 @@ export interface GitBookLinker { anchor?: string; }): string; + /** + * Generate an absolute path for a page path in the current content. + * The result should NOT be passed to `toPathInSpace`. + */ + toPathForPagePath(input: { + path: string; + anchor?: string; + }): string; + /** * Generate an absolute URL for a given path relative to the host of the current content. */ @@ -130,7 +139,14 @@ export function createLinker( }, toPathForPage({ pages, page, anchor }) { - return linker.toPathInSpace(getPagePath(pages, page)) + (anchor ? `#${anchor}` : ''); + return linker.toPathForPagePath({ + path: getPagePath(pages, page), + anchor, + }); + }, + + toPathForPagePath({ path, anchor }) { + return linker.toPathInSpace(path) + (anchor ? `#${anchor}` : ''); }, toAbsoluteURL(absolutePath: string): string { @@ -167,10 +183,13 @@ export function createLinker( // implementation matches the base linker toPathForPage, but decouples from using `this` to // ensure we always use the updates `toPathInSpace` method. toPathForPage({ pages, page, anchor }) { - return ( - newLinker.toPathInSpace(getPagePath(pages, page)) + - (anchor ? `#${anchor}` : '') - ); + return newLinker.toPathForPagePath({ + path: getPagePath(pages, page), + anchor, + }); + }, + toPathForPagePath({ path, anchor }) { + return newLinker.toPathInSpace(path) + (anchor ? `#${anchor}` : ''); }, }; @@ -216,24 +235,40 @@ export function linkerForPublishedURL(linker: GitBookLinker, rawSitePublishedURL * Create a new linker that always returns absolute URLs. */ export function linkerWithAbsoluteURLs(linker: GitBookLinker): GitBookLinker { - return { + const self: GitBookLinker = { ...linker, toPathInSpace: (path) => linker.toAbsoluteURL(linker.toPathInSpace(path)), toPathInSite: (path) => linker.toAbsoluteURL(linker.toPathInSite(path)), - toPathForPage: (input) => linker.toAbsoluteURL(linker.toPathForPage(input)), + toPathForPage: (input) => { + return self.toPathForPagePath({ + path: getPagePath(input.pages, input.page), + anchor: input.anchor, + }); + }, + toPathForPagePath: (input) => linker.toAbsoluteURL(linker.toPathForPagePath(input)), }; + + return self; } /** * Create a new linker that resolves pages to their markdown version. */ export function linkerWithMarkdownPages(linker: GitBookLinker): GitBookLinker { - return { + const self: GitBookLinker = { ...linker, toPathForPage: (input) => { - return `${linker.toPathInSpace(input.page.path)}.md${input.anchor ? `#${input.anchor}` : ''}`; + return self.toPathForPagePath({ + path: input.page.path, + anchor: input.anchor, + }); + }, + toPathForPagePath: (input) => { + return `${linker.toPathInSpace(input.path)}.md${input.anchor ? `#${input.anchor}` : ''}`; }, }; + + return self; } function joinPaths(prefix: string, path: string): string { diff --git a/packages/gitbook/src/lib/markdownPage.ts b/packages/gitbook/src/lib/markdownPage.ts index 035d05b55..5e11b1a1c 100644 --- a/packages/gitbook/src/lib/markdownPage.ts +++ b/packages/gitbook/src/lib/markdownPage.ts @@ -86,11 +86,7 @@ export async function getMarkdownForPageInSpace( // Handle group pages (pages with no content that list their children) if (page.type === RevisionPageType.Group) { - const siteSpaceUrl = siteSpace.urls.published; - if (!siteSpaceUrl) { - throw new DataFetcherError(`Page "${page.title}" is not published`, 404); - } - return renderGroupPageMarkdown({ siteSpaceUrl, linker, page }); + return renderGroupPageMarkdown({ linker, page }); } const rawMarkdown = await throwIfDataError( @@ -108,11 +104,7 @@ export async function getMarkdownForPageInSpace( // Handle empty document pages which have children (same as getMarkdownForPage) if (isEmptyMarkdownPage(tree) && page.pages.length > 0) { - const siteSpaceUrl = siteSpace.urls.published; - if (!siteSpaceUrl) { - throw new DataFetcherError(`Page "${page.title}" is not published`, 404); - } - return renderGroupPageMarkdown({ siteSpaceUrl, linker, page }); + return renderGroupPageMarkdown({ linker, page }); } return toPageMarkdown(tree); @@ -190,13 +182,7 @@ async function servePageGroup( context: GitBookSiteContext, page: RevisionPageDocument | RevisionPageGroup ): Promise { - const siteSpaceUrl = context.space.urls.published; - if (!siteSpaceUrl) { - throw new DataFetcherError(`Page "${page.title}" is not published`, 404); - } - return renderGroupPageMarkdown({ - siteSpaceUrl, linker: context.linker, page, }); @@ -207,11 +193,10 @@ async function servePageGroup( * Use this when rendering a group page from a different space than the current context. */ async function renderGroupPageMarkdown(args: { - siteSpaceUrl: string; linker: GitBookLinker; page: RevisionPageDocument | RevisionPageGroup; }): Promise { - const { siteSpaceUrl, linker, page } = args; + const { linker, page } = args; const indexablePages = getIndexablePages(page.pages); const markdownTree: Root = { @@ -222,11 +207,7 @@ async function renderGroupPageMarkdown(args: { depth: 1, children: [{ type: 'text', value: page.title }], }, - ...(await getMarkdownForPagesTree(indexablePages, { - siteSpaceUrl, - linker, - withMarkdownPages: true, - })), + ...(await getMarkdownForPagesTree(indexablePages, linker)), ], }; diff --git a/packages/gitbook/src/routes/llms.ts b/packages/gitbook/src/routes/llms.ts index a75a3f33b..385852cff 100644 --- a/packages/gitbook/src/routes/llms.ts +++ b/packages/gitbook/src/routes/llms.ts @@ -1,7 +1,8 @@ import { type GitBookSiteContext, checkIsRootSiteContext } from '@/lib/context'; import { throwIfDataError } from '@/lib/data'; -import type { GitBookLinker } from '@/lib/links'; -import { joinPath } from '@/lib/paths'; +import { type GitBookLinker, linkerWithMarkdownPages } from '@/lib/links'; +import { resolveFirstDocument } from '@/lib/pages'; +import { isRollout } from '@/lib/rollout'; import { type FlatPageEntry, getIndexablePages } from '@/lib/sitemap'; import { filterSiteSpacesByLocale, getLocalizedTitle, getSiteStructureSections } from '@/lib/sites'; import type { SiteSection, SiteSpace } from '@gitbook/api'; @@ -12,23 +13,18 @@ import { toMarkdown } from 'mdast-util-to-markdown'; /** * Generate a llms.txt file for the site. */ -export async function serveLLMsTxt( - context: GitBookSiteContext, - { - withMarkdownPages = false, - }: { - /** - * If true, a markdown extension will be added to the page path. - */ - withMarkdownPages?: boolean; - } = {} -) { - const { site } = context; +export async function serveLLMsTxt(baseContext: GitBookSiteContext) { + const { site } = baseContext; - if (!checkIsRootSiteContext(context)) { + if (!checkIsRootSiteContext(baseContext)) { return new Response('llms.txt is only served from the root of the site', { status: 404 }); } + const context = { + ...baseContext, + linker: linkerWithMarkdownPages(baseContext.linker), + }; + const tree: Root = { type: 'root', children: [ @@ -37,42 +33,36 @@ export async function serveLLMsTxt( depth: 1, children: [{ type: 'text', value: site.title }], }, - ...(await getNodesFromSiteStructure(context, { withMarkdownPages })), + ...(await getNodesFromSiteStructure(context)), ], }; - return new Response( - toMarkdown(tree, { - bullet: '-', - }), - { - headers: { - 'Content-Type': 'text/markdown; charset=utf-8', - }, - } - ); + let output = toMarkdown(tree, { + bullet: '-', + }); + + output += renderAskFooter(context); + + return new Response(output, { + headers: { + 'Content-Type': 'text/markdown; charset=utf-8', + }, + }); } /** * Get MDAST nodes from site structure. */ -async function getNodesFromSiteStructure( - context: GitBookSiteContext, - options: { - withMarkdownPages: boolean; - } -): Promise { +async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise { switch (context.structure.type) { case 'sections': return getNodesFromSections( context, - getSiteStructureSections(context.structure, { ignoreGroups: true }), - { withMarkdownPages: options.withMarkdownPages } + getSiteStructureSections(context.structure, { ignoreGroups: true }) ); case 'siteSpaces': return getNodesFromSiteSpaces(context, context.structure.structure, { heading: true, - withMarkdownPages: options.withMarkdownPages, }); default: assertNever(context.structure); @@ -84,17 +74,13 @@ async function getNodesFromSiteStructure( */ async function getNodesFromSections( context: GitBookSiteContext, - siteSections: SiteSection[], - options: { - withMarkdownPages: boolean; - } + siteSections: SiteSection[] ): Promise { const currentLanguage = context.locale; const all = await Promise.all( siteSections.map(async (siteSection): Promise => { const siteSpaceNodes = await getNodesFromSiteSpaces(context, siteSection.siteSpaces, { heading: false, - withMarkdownPages: options.withMarkdownPages, }); return [ { @@ -122,11 +108,6 @@ async function getNodesFromSiteSpaces( * Includes a heading for each site space. */ heading?: boolean; - - /** - * If true, a markdown extension will be added to the page path. - */ - withMarkdownPages: boolean; } ): Promise { const { dataFetcher, linker } = context; @@ -164,13 +145,7 @@ async function getNodesFromSiteSpaces( } // Add the pages as a list - nodes.push( - ...(await getMarkdownForPagesTree(pages, { - siteSpaceUrl, - linker, - withMarkdownPages: options.withMarkdownPages, - })) - ); + nodes.push(...(await getMarkdownForPagesTree(pages, linker))); return nodes; }) @@ -183,23 +158,15 @@ async function getNodesFromSiteSpaces( */ export async function getMarkdownForPagesTree( pages: FlatPageEntry[], - options: { - siteSpaceUrl: string; - linker: GitBookLinker; - withMarkdownPages?: boolean; - } + linker: GitBookLinker ): Promise { - const { siteSpaceUrl, linker } = options; - const listChildren = await Promise.all( pages.map(async ({ page }): Promise => { - const pageURL = new URL(siteSpaceUrl); - pageURL.pathname = joinPath(pageURL.pathname, page.path); - if (options.withMarkdownPages) { - pageURL.pathname = `${pageURL.pathname}.md`; - } - - const url = linker.toLinkForContent(pageURL.toString()); + const url = linker.toAbsoluteURL( + linker.toPathForPagePath({ + path: page.path, + }) + ); const children: Paragraph['children'] = [ { type: 'link', @@ -224,3 +191,35 @@ export async function getMarkdownForPagesTree( }); return nodes; } + +function renderAskFooter(context: GitBookSiteContext) { + if ( + !isRollout({ + discriminator: context.site.id, + percentageRollout: 25, + }) && + context.site.id !== 'site_p4Xo4' + ) { + return ''; + } + + return `\n\n---\n\n# Agent Instructions: Querying This Documentation + +If you need additional information, you can query the documentation dynamically by asking a question. + +Perform an HTTP GET request on a page URL with the \`ask\` query parameter: + +\`\`\` +GET ${context.linker.toAbsoluteURL( + context.linker.toPathForPagePath({ + path: resolveFirstDocument(context.revision.pages, [])?.page.path ?? 'index', + }) + )}?ask= +\`\`\` + +The question should be specific, self-contained, and written in natural language. +The response will contain a direct answer to the question and relevant excerpts and sources from the documentation. + +Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections. +`; +} diff --git a/packages/gitbook/src/routes/markdownPage.ts b/packages/gitbook/src/routes/markdownPage.ts index 9eec2e99a..fdf00ffab 100644 --- a/packages/gitbook/src/routes/markdownPage.ts +++ b/packages/gitbook/src/routes/markdownPage.ts @@ -7,6 +7,7 @@ import { getSimilarPages, resolvePagePathDocumentOrGroup, } from '@/lib/pages'; +import { isRollout } from '@/lib/rollout'; import type { RevisionPageDocument, RevisionPageGroup } from '@gitbook/api'; /** @@ -40,39 +41,89 @@ function renderNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) { return `# Page Not Found -The URL \`${pagePath}\` does not exist. +The URL \`${pagePath}\` does not exist. This page may have been moved, renamed, or deleted. -You might be looking for one of these pages: +## Suggested Pages + +You may be looking for one of the following: ${similarPages.map((page) => `- [${page.title}](${context.linker.toAbsoluteURL(context.linker.toPathInSpace(page.path))}.md)`).join('\n')} ## How to find the correct page -1. **Ask a question**: ${context.linker.toPathInSite('sitemap.md')}?ask= - Complete answer with sources -2. **Browse the full index**: [${context.linker.toPathInSite('sitemap.md')}](${sitemapUrl}) - Complete documentation index -3. **View the full content**: [${context.linker.toPathInSite('llms-full.txt')}](${fullContentUrl}) - Full content export +If the exact page cannot be found, you can still retrieve the information using the documentation query interface. + +### Option 1 — Ask a question (recommended) + +Perform an HTTP GET request on the documentation index with the \`ask\` parameter: + +\`\`\` +GET ${context.linker.toAbsoluteURL( + context.linker.toPathForPagePath({ + path: similarPages[0]?.path ?? 'docs/example', + }) + )}?ask= +\`\`\` + +The question should be specific, self-contained, and written in natural language. +The response will contain a direct answer to the question and relevant excerpts and sources from the documentation. + +### Option 2 — Browse the documentation index + +Full index: ${sitemapUrl} + +Use this to discover valid page paths or navigate the documentation structure. + +### Option 3 — Retrieve the full documentation corpus + +Full export: ${fullContentUrl} + +Use this to access all content at once and perform your own parsing or retrieval. It will be more expensive. ## Tips for requesting documentation -- For markdown responses, append \`.md\` to URLs (e.g., \`${context.linker.toPathInSpace(similarPages[0]?.path ?? 'docs/example')}.md\`) -- Use \`Accept: text/markdown\` header for content negotiation`; +Prefer \`.md\` URLs for structured content, append \`.md\` to URLs (e.g., \`${context.linker.toPathForPagePath( + { + path: similarPages[0]?.path ?? 'docs/example', + } + )}\`). + +You may also use \`Accept: text/markdown\` header for content negotiation. +`; } function renderAskFooter( context: GitBookSiteContext, pageLookup: ResolvedPagePath ) { - if (context.site.id !== 'site_p4Xo4') { + if ( + !isRollout({ + discriminator: context.site.id, + percentageRollout: 25, + }) && + context.site.id !== 'site_p4Xo4' + ) { return ''; } - return `\n\n---\n\n# Ask questions about the documentation + return `\n\n---\n\n# Agent Instructions: Querying This Documentation -Tips: If you are looking for an information in the documentation, you can find it by asking a question directly via GET and the \`${context.linker.toAbsoluteURL( +If you need additional information that is not directly available in this page, you can query the documentation dynamically by asking a question. + +Perform an HTTP GET request on the current page URL with the \`ask\` query parameter: + +\`\`\` +GET ${context.linker.toAbsoluteURL( context.linker.toPathForPage({ page: pageLookup.page, pages: context.revision.pages, }) - )}?ask=\`. + )}?ask= +\`\`\` + +The question should be specific, self-contained, and written in natural language. +The response will contain a direct answer to the question and relevant excerpts and sources from the documentation. + +Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections. `; } diff --git a/packages/gitbook/tests/markdown.test.ts b/packages/gitbook/tests/markdown.test.ts index ef45f19d8..975c9c572 100644 --- a/packages/gitbook/tests/markdown.test.ts +++ b/packages/gitbook/tests/markdown.test.ts @@ -45,7 +45,7 @@ describe('markdown pages', () => { }); describe('markdown ask responses', () => { - const ASK_QUESTION = 'How are docs optimized for agents?'; + const ASK_QUESTION = 'What is GitBook?'; const ASK_QUESTION_HEADING = `# ${ASK_QUESTION}`; it( @@ -62,7 +62,6 @@ describe('markdown ask responses', () => { expect(response.headers.get('content-type')).toContain('text/markdown'); expect(response.headers.get('x-robots-tag')).toBe('noindex'); expect(text).toContain(ASK_QUESTION_HEADING); - expect(text).toContain('# Sources'); }, { timeout: 30_000 } ); @@ -86,7 +85,6 @@ describe('markdown ask responses', () => { expect(response.headers.get('content-type')).toContain('text/markdown'); expect(response.headers.get('x-robots-tag')).toBe('noindex'); expect(text).toContain(ASK_QUESTION_HEADING); - expect(text).toContain('# Sources'); }, { timeout: 30_000 } );