diff --git a/.changeset/llms-full-pagination-header.md b/.changeset/llms-full-pagination-header.md new file mode 100644 index 000000000..f32bb78e2 --- /dev/null +++ b/.changeset/llms-full-pagination-header.md @@ -0,0 +1,5 @@ +--- +"gitbook": patch +--- + +Announce pagination at the top of every part of a multi-part llms-full.txt, with absolute links to the previous and next parts. diff --git a/packages/gitbook/src/routes/llms-full.ts b/packages/gitbook/src/routes/llms-full.ts index 64fbfbf08..b89c341e7 100644 --- a/packages/gitbook/src/routes/llms-full.ts +++ b/packages/gitbook/src/routes/llms-full.ts @@ -151,6 +151,26 @@ async function streamMarkdownPageEntries( const pagesToProcess = allPages.slice(offset, offset + DEFAULT_PAGE_LIMIT); let totalPagesProcessed = offset; + // Agents often grep the file or only read its start, so announce the other parts up front. + const part = Math.floor(offset / DEFAULT_PAGE_LIMIT); + const partCount = Math.ceil(allPages.length / DEFAULT_PAGE_LIMIT); + if (partCount > 1) { + const header = [ + `> This is part ${part + 1} of ${partCount} of the full documentation (pages ${offset + 1}–${offset + pagesToProcess.length} of ${allPages.length}).`, + '> The content is paginated: fetch every part to see all of it.', + ]; + if (part > 0) { + header.push(`> Previous part: ${getPartURL(context, part - 1)}`); + } + if (part < partCount - 1) { + header.push(`> Next part: ${getPartURL(context, part + 1)}`); + } + header.push( + `> Page index: ${context.linker.toAbsoluteURL(context.linker.toPathInSite('llms.txt'))}` + ); + stream.enqueue(new TextEncoder().encode(`${header.join('\n')}\n\n`)); + } + // Process the pages for await (const markdown of pMapIterable( pagesToProcess, @@ -168,8 +188,7 @@ async function streamMarkdownPageEntries( // Check if there are more pages and add next page link if needed const hasMorePages = allPages.length > offset + DEFAULT_PAGE_LIMIT; if (hasMorePages) { - const nextPage = Math.floor(offset / DEFAULT_PAGE_LIMIT) + 1; - const nextPageUrl = context.linker.toPathInSite(`llms-full.txt/${nextPage}`); + const nextPageUrl = getPartURL(context, part + 1); const nextPageLink = `\n\n---\n\n[Next Page](${nextPageUrl})\n\n`; stream.enqueue(new TextEncoder().encode(nextPageLink)); } @@ -177,6 +196,14 @@ async function streamMarkdownPageEntries( return { currentPageIndex: totalPagesProcessed, reachedLimit: hasMorePages }; } +/** + * Get the absolute URL of a part of llms-full.txt. + */ +function getPartURL(context: GitBookSiteContext, part: number) { + const path = part === 0 ? 'llms-full.txt' : `llms-full.txt/${part}`; + return context.linker.toAbsoluteURL(context.linker.toPathInSite(path)); +} + /** * Get markdown from a page. */ diff --git a/packages/gitbook/tests/llms.test.ts b/packages/gitbook/tests/llms.test.ts index e8b6f1564..91ad1777c 100644 --- a/packages/gitbook/tests/llms.test.ts +++ b/packages/gitbook/tests/llms.test.ts @@ -152,6 +152,21 @@ describe('llms-full.txt', () => { { timeout: 30_000 } ); + it( + 'should not add a pagination header to a single-part llms-full.txt', + async () => { + const response = await fetch( + getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms-full.txt') + ); + const text = await response.text(); + + expect(response.status).toBe(200); + expect(text).not.toContain('The content is paginated'); + expect(text).not.toContain('[Next Page]'); + }, + { timeout: 30_000 } + ); + it( 'should return 404 when a llms-full.txt page has no content', async () => {