diff --git a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms-full.txt/[page]/route.ts b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms-full.txt/[page]/route.ts new file mode 100644 index 000000000..f9ef65cde --- /dev/null +++ b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms-full.txt/[page]/route.ts @@ -0,0 +1,20 @@ +import type { NextRequest } from 'next/server'; + +import { type RouteLayoutParams, getStaticSiteContext } from '@/app/utils'; +import { serveLLMsFullTxt } from '@/routes/llms-full'; + +export const dynamic = 'force-static'; + +export async function GET( + _request: NextRequest, + { params }: { params: Promise } +) { + const awaitedParams = await params; + const page = Number(awaitedParams.page); + // If page is not a number, not an integer, or less than 0, return an error + if (Number.isNaN(page) || !Number.isInteger(page) || page < 0) { + return new Response('Invalid page', { status: 400 }); + } + const { context } = await getStaticSiteContext(awaitedParams); + return serveLLMsFullTxt(context, page); +} diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index 372d865fc..d25364007 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -540,6 +540,11 @@ function encodePathInSiteContent(rawPathname: string): { }; } + // We skip encoding for paginated llms-full.txt pages (i.e. llms-full.txt/100) + if (pathname.match(/^llms-full\.txt\/\d+$/)) { + return { pathname, routeType: 'static' }; + } + // If the pathname is an embedded page const embedPage = pathname.match(/^~gitbook\/embed\/page\/(\S+)$/); if (embedPage) { diff --git a/packages/gitbook/src/routes/llms-full.test.ts b/packages/gitbook/src/routes/llms-full.test.ts new file mode 100644 index 000000000..e8bc95416 --- /dev/null +++ b/packages/gitbook/src/routes/llms-full.test.ts @@ -0,0 +1,444 @@ +import { describe, expect, it, mock } from 'bun:test'; +import type { GitBookSiteContext } from '@/lib/context'; +import type { SiteSpace } from '@gitbook/api'; + +import { streamMarkdownFromSiteSpaces } from './llms-full'; + +describe('streamMarkdownFromSiteSpaces', () => { + // Test with real mocks of the dependencies + it('processes pages correctly with pagination', async () => { + // Mock the dependencies by replacing them in the module + const mockDataFetcher = { + getRevision: mock(() => + Promise.resolve({ + data: { + id: 'revision-1', + pages: [ + { + id: 'page-1', + type: 'document', + title: 'Page 1', + path: 'page-1', + pages: [], + hidden: false, + }, + { + id: 'page-2', + type: 'document', + title: 'Page 2', + path: 'page-2', + pages: [], + hidden: false, + }, + { + id: 'page-3', + type: 'document', + title: 'Page 3', + path: 'page-3', + pages: [], + hidden: false, + }, + { + id: 'page-4', + type: 'document', + title: 'Page 4', + path: 'page-4', + pages: [], + hidden: false, + }, + { + id: 'page-5', + type: 'document', + title: 'Page 5', + path: 'page-5', + pages: [], + hidden: false, + }, + ], + }, + }) + ), + getRevisionPageMarkdown: mock(() => + Promise.resolve({ + data: '# Test Page\n\nSome content\n', + }) + ), + }; + + const mockLinker = { + toPathInSite: mock((path: string) => `/test/${path}`), + }; + + const mockContext: GitBookSiteContext = { + dataFetcher: mockDataFetcher, + linker: mockLinker, + } as unknown as GitBookSiteContext; + + const mockSiteSpace: SiteSpace = { + id: 'space-1', + space: { + id: 'space-1', + revision: 'rev-1', + }, + urls: { + published: 'https://example.com', + }, + path: 'test-space', + } as SiteSpace; + + // Capture stream output + const chunks: string[] = []; + const mockController = { + enqueue: mock((chunk: Uint8Array) => { + chunks.push(new TextDecoder().decode(chunk)); + }), + } as unknown as ReadableStreamDefaultController; + + const result = await streamMarkdownFromSiteSpaces( + mockContext, + mockController, + [mockSiteSpace], + 'base-path', + 0, + 0 + ); + + // Verify results + expect(result.currentPageIndex).toBe(5); // Should process 5 pages + expect(result.reachedLimit).toBe(false); // Under limit + expect(chunks.length).toBe(5); // Should have 5 markdown chunks + expect(mockDataFetcher.getRevision).toHaveBeenCalledTimes(1); + expect(mockDataFetcher.getRevisionPageMarkdown).toHaveBeenCalledTimes(5); + }); + + it('applies offset correctly', async () => { + const mockDataFetcher = { + getRevision: mock(() => + Promise.resolve({ + data: { + pages: Array.from({ length: 10 }, (_, i) => ({ + id: `page-${i + 1}`, + type: 'document', + title: `Page ${i + 1}`, + path: `page-${i + 1}`, + pages: [], + hidden: false, + })), + }, + }) + ), + getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })), + }; + + const mockContext: GitBookSiteContext = { + dataFetcher: mockDataFetcher, + linker: { toPathInSite: mock((path: string) => `/${path}`) }, + } as unknown as GitBookSiteContext; + + const mockSiteSpace: SiteSpace = { + space: { id: 'space-1', revision: 'rev-1' }, + urls: { published: 'https://example.com' }, + path: 'test-space', + } as SiteSpace; + + const chunks: string[] = []; + const mockController = { + enqueue: mock((chunk: Uint8Array) => { + chunks.push(new TextDecoder().decode(chunk)); + }), + } as unknown as ReadableStreamDefaultController; + + const result = await streamMarkdownFromSiteSpaces( + mockContext, + mockController, + [mockSiteSpace], + 'base-path', + 3, // offset = 3 + 0 + ); + + // Should process pages from index 3 onwards (7 pages) + expect(result.currentPageIndex).toBe(10); + expect(chunks.length).toBe(7); // 10 total - 3 offset = 7 processed + }); + + it('handles pagination when there are more than 100 pages', async () => { + const mockDataFetcher = { + getRevision: mock(() => + Promise.resolve({ + data: { + pages: Array.from({ length: 150 }, (_, i) => ({ + id: `page-${i + 1}`, + type: 'document', + title: `Page ${i + 1}`, + path: `page-${i + 1}`, + pages: [], + hidden: false, + })), + }, + }) + ), + getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })), + }; + + const mockLinker = { + toPathInSite: mock((path: string) => `/site/${path}`), + }; + + const mockContext: GitBookSiteContext = { + dataFetcher: mockDataFetcher, + linker: mockLinker, + } as unknown as GitBookSiteContext; + + const mockSiteSpace: SiteSpace = { + space: { id: 'space-1', revision: 'rev-1' }, + urls: { published: 'https://example.com' }, + path: 'test-space', + } as SiteSpace; + + const chunks: string[] = []; + const mockController = { + enqueue: mock((chunk: Uint8Array) => { + chunks.push(new TextDecoder().decode(chunk)); + }), + } as unknown as ReadableStreamDefaultController; + + const result = await streamMarkdownFromSiteSpaces( + mockContext, + mockController, + [mockSiteSpace], + 'base-path', + 0, + 0 + ); + + // Should only process 100 pages (default limit) + expect(result.currentPageIndex).toBe(100); + expect(result.reachedLimit).toBe(true); + expect(chunks.length).toBe(101); // 100 pages + 1 next page link + + // Check that next page link is included + const fullContent = chunks.join(''); + expect(fullContent).toContain('[Next Page]'); + expect(fullContent).toContain('/site/llms-full.txt/1'); + }); + + it('handles multiple site spaces', async () => { + const mockDataFetcher = { + getRevision: mock() + .mockReturnValueOnce( + Promise.resolve({ + data: { + pages: [ + { + id: 'page-1', + type: 'document', + title: 'Space 1 Page 1', + path: 'page-1', + pages: [], + hidden: false, + }, + { + id: 'page-2', + type: 'document', + title: 'Space 1 Page 2', + path: 'page-2', + pages: [], + hidden: false, + }, + ], + }, + }) + ) + .mockReturnValueOnce( + Promise.resolve({ + data: { + pages: [ + { + id: 'page-3', + type: 'document', + title: 'Space 2 Page 1', + path: 'page-3', + pages: [], + hidden: false, + }, + { + id: 'page-4', + type: 'document', + title: 'Space 2 Page 2', + path: 'page-4', + pages: [], + hidden: false, + }, + { + id: 'page-5', + type: 'document', + title: 'Space 2 Page 3', + path: 'page-5', + pages: [], + hidden: false, + }, + ], + }, + }) + ), + getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })), + }; + + const mockContext: GitBookSiteContext = { + dataFetcher: mockDataFetcher, + linker: { toPathInSite: mock((path: string) => `/${path}`) }, + } as unknown as GitBookSiteContext; + + const mockSiteSpaces: SiteSpace[] = [ + { + space: { id: 'space-1', revision: 'rev-1' }, + urls: { published: 'https://example1.com' }, + path: 'space-1', + }, + { + space: { id: 'space-2', revision: 'rev-2' }, + urls: { published: 'https://example2.com' }, + path: 'space-2', + }, + ] as SiteSpace[]; + + const chunks: string[] = []; + const mockController = { + enqueue: mock((chunk: Uint8Array) => { + chunks.push(new TextDecoder().decode(chunk)); + }), + } as unknown as ReadableStreamDefaultController; + + const { streamMarkdownFromSiteSpaces } = await import('./llms-full'); + + const result = await streamMarkdownFromSiteSpaces( + mockContext, + mockController, + mockSiteSpaces, + 'base-path', + 0, + 0 + ); + + // Should process all pages from both spaces (2 + 3 = 5) + expect(result.currentPageIndex).toBe(5); + expect(result.reachedLimit).toBe(false); + expect(chunks.length).toBe(5); + expect(mockDataFetcher.getRevision).toHaveBeenCalledTimes(2); + }); + + it('skips site spaces without published URLs', async () => { + const mockDataFetcher = { + getRevision: mock(), + getRevisionPageMarkdown: mock(), + }; + + const mockContext: GitBookSiteContext = { + dataFetcher: mockDataFetcher, + linker: { toPathInSite: mock((path: string) => `/${path}`) }, + } as unknown as GitBookSiteContext; + + const mockSiteSpace: SiteSpace = { + space: { id: 'space-1', revision: 'rev-1' }, + urls: { published: undefined }, // No published URL + path: 'test-space', + } as SiteSpace; + + const mockController = { + enqueue: mock(), + } as unknown as ReadableStreamDefaultController; + + const result = await streamMarkdownFromSiteSpaces( + mockContext, + mockController, + [mockSiteSpace], + 'base-path', + 0, + 0 + ); + + // Should not process any pages + expect(result.currentPageIndex).toBe(0); + expect(result.reachedLimit).toBe(false); + expect(mockDataFetcher.getRevision).not.toHaveBeenCalled(); + }); + + it('filters only document type pages', async () => { + const mockDataFetcher = { + getRevision: mock(() => + Promise.resolve({ + data: { + pages: [ + { + id: 'doc-1', + type: 'document', + title: 'Document 1', + path: 'doc-1', + pages: [], + hidden: false, + }, + { + id: 'group-1', + type: 'group', + title: 'Group 1', + path: 'group-1', + pages: [], + hidden: false, + }, + { + id: 'doc-2', + type: 'document', + title: 'Document 2', + path: 'doc-2', + pages: [], + hidden: false, + }, + { + id: 'link-1', + type: 'link', + title: 'Link 1', + path: 'link-1', + pages: [], + hidden: false, + }, + ], + }, + }) + ), + getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })), + }; + + const mockContext: GitBookSiteContext = { + dataFetcher: mockDataFetcher, + linker: { toPathInSite: mock((path: string) => `/${path}`) }, + } as unknown as GitBookSiteContext; + + const mockSiteSpace: SiteSpace = { + space: { id: 'space-1', revision: 'rev-1' }, + urls: { published: 'https://example.com' }, + path: 'test-space', + } as SiteSpace; + + const chunks: string[] = []; + const mockController = { + enqueue: mock((chunk: Uint8Array) => { + chunks.push(new TextDecoder().decode(chunk)); + }), + } as unknown as ReadableStreamDefaultController; + + const result = await streamMarkdownFromSiteSpaces( + mockContext, + mockController, + [mockSiteSpace], + 'base-path', + 0, + 0 + ); + + // Should only process the 2 document pages + expect(result.currentPageIndex).toBe(2); + expect(chunks.length).toBe(2); + expect(mockDataFetcher.getRevisionPageMarkdown).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/gitbook/src/routes/llms-full.ts b/packages/gitbook/src/routes/llms-full.ts index fac0f0daf..cc626de8a 100644 --- a/packages/gitbook/src/routes/llms-full.ts +++ b/packages/gitbook/src/routes/llms-full.ts @@ -22,19 +22,24 @@ import { visit } from 'unist-util-visit'; // or file descriptor limits. const MAX_CONCURRENCY = 100; +// Default limit for pages per batch +const DEFAULT_PAGE_LIMIT = 100; + /** * Generate a llms-full.txt file for the site. * As the result can be large, we stream it as we generate it. */ -export async function serveLLMsFullTxt(context: GitBookSiteContext) { +export async function serveLLMsFullTxt(context: GitBookSiteContext, page = 0) { if (!checkIsRootSiteContext(context)) { return new Response('llms.txt is only served from the root of the site', { status: 404 }); } + const offset = page * DEFAULT_PAGE_LIMIT; + return new Response( new ReadableStream({ async pull(controller) { - await streamMarkdownFromSiteStructure(context, controller); + await streamMarkdownFromSiteStructure(context, controller, offset); controller.close(); }, }), @@ -51,17 +56,26 @@ export async function serveLLMsFullTxt(context: GitBookSiteContext) { */ async function streamMarkdownFromSiteStructure( context: GitBookSiteContext, - stream: ReadableStreamDefaultController + stream: ReadableStreamDefaultController, + offset: number ): Promise { switch (context.structure.type) { case 'sections': return streamMarkdownFromSections( context, stream, - getSiteStructureSections(context.structure, { ignoreGroups: true }) + getSiteStructureSections(context.structure, { ignoreGroups: true }), + offset ); case 'siteSpaces': - return streamMarkdownFromSiteSpaces(context, stream, context.structure.structure, ''); + await streamMarkdownFromSiteSpaces( + context, + stream, + context.structure.structure, + '', + offset + ); + return; default: assertNever(context.structure); } @@ -73,28 +87,45 @@ async function streamMarkdownFromSiteStructure( async function streamMarkdownFromSections( context: GitBookSiteContext, stream: ReadableStreamDefaultController, - siteSections: SiteSection[] + siteSections: SiteSection[], + offset: number ): Promise { + let currentPageIndex = 0; + for (const siteSection of siteSections) { - await streamMarkdownFromSiteSpaces( + const result = await streamMarkdownFromSiteSpaces( context, stream, siteSection.siteSpaces, - siteSection.path + siteSection.path, + offset, + currentPageIndex ); + currentPageIndex = result.currentPageIndex; + + if (result.reachedLimit) { + break; + } } } /** * Stream markdown from site spaces. */ -async function streamMarkdownFromSiteSpaces( +export async function streamMarkdownFromSiteSpaces( context: GitBookSiteContext, stream: ReadableStreamDefaultController, siteSpaces: SiteSpace[], - basePath: string -): Promise { + basePath: string, + offset = 0, + initialPageIndex = 0 +): Promise<{ currentPageIndex: number; reachedLimit: boolean }> { const { dataFetcher } = context; + let totalPagesProcessed = initialPageIndex; + + // Collect all pages first + const allPages: Array<{ page: RevisionPageDocument; siteSpace: SiteSpace; basePath: string }> = + []; for (const siteSpace of siteSpaces) { const siteSpaceUrl = siteSpace.urls.published; @@ -109,27 +140,46 @@ async function streamMarkdownFromSiteSpaces( ); const pages = getIndexablePages(revision.pages); - for await (const markdown of pMapIterable( - pages, - async ({ page }) => { - if (page.type !== 'document') { - return ''; - } - - return getMarkdownForPage( - context, - siteSpace, + // Add document pages to our collection + for (const { page } of pages) { + if (page.type === 'document') { + allPages.push({ page, - joinPath(basePath, siteSpace.path) - ); - }, - { - concurrency: MAX_CONCURRENCY, + siteSpace, + basePath: joinPath(basePath, siteSpace.path), + }); } - )) { - stream.enqueue(new TextEncoder().encode(markdown)); } } + + // Apply pagination - skip pages before offset + const pagesToProcess = allPages.slice(offset, offset + DEFAULT_PAGE_LIMIT); + totalPagesProcessed = offset; + + // Process the pages + for await (const markdown of pMapIterable( + pagesToProcess, + async ({ page, siteSpace, basePath }) => { + return getMarkdownForPage(context, siteSpace, page, basePath); + }, + { + concurrency: MAX_CONCURRENCY, + } + )) { + stream.enqueue(new TextEncoder().encode(markdown)); + totalPagesProcessed++; + } + + // Check if there are more pages and add next page link if needed + const hasMorePages = allPages.length > offset + DEFAULT_PAGE_LIMIT; + if (hasMorePages) { + const nextPage = Math.floor(offset / DEFAULT_PAGE_LIMIT) + 1; + const nextPageUrl = context.linker.toPathInSite(`llms-full.txt/${nextPage}`); + const nextPageLink = `\n\n---\n\n[Next Page](${nextPageUrl})\n\n`; + stream.enqueue(new TextEncoder().encode(nextPageLink)); + } + + return { currentPageIndex: totalPagesProcessed, reachedLimit: hasMorePages }; } /**