mirror of
https://github.com/GitbookIO/gitbook.git
synced 2026-09-21 10:03:31 +00:00
Add pagination for llms-full.txt file (#3619)
Co-authored-by: Nicolas Dorseuil <nicolas@gitbook.io>
This commit is contained in:
conico974
+20
@@ -0,0 +1,20 @@
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
import { type RouteLayoutParams, getStaticSiteContext } from '@/app/utils';
|
||||
import { serveLLMsFullTxt } from '@/routes/llms-full';
|
||||
|
||||
export const dynamic = 'force-static';
|
||||
|
||||
export async function GET(
|
||||
_request: NextRequest,
|
||||
{ params }: { params: Promise<RouteLayoutParams & { page: string }> }
|
||||
) {
|
||||
const awaitedParams = await params;
|
||||
const page = Number(awaitedParams.page);
|
||||
// If page is not a number, not an integer, or less than 0, return an error
|
||||
if (Number.isNaN(page) || !Number.isInteger(page) || page < 0) {
|
||||
return new Response('Invalid page', { status: 400 });
|
||||
}
|
||||
const { context } = await getStaticSiteContext(awaitedParams);
|
||||
return serveLLMsFullTxt(context, page);
|
||||
}
|
||||
@@ -540,6 +540,11 @@ function encodePathInSiteContent(rawPathname: string): {
|
||||
};
|
||||
}
|
||||
|
||||
// We skip encoding for paginated llms-full.txt pages (i.e. llms-full.txt/100)
|
||||
if (pathname.match(/^llms-full\.txt\/\d+$/)) {
|
||||
return { pathname, routeType: 'static' };
|
||||
}
|
||||
|
||||
// If the pathname is an embedded page
|
||||
const embedPage = pathname.match(/^~gitbook\/embed\/page\/(\S+)$/);
|
||||
if (embedPage) {
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
import { describe, expect, it, mock } from 'bun:test';
|
||||
import type { GitBookSiteContext } from '@/lib/context';
|
||||
import type { SiteSpace } from '@gitbook/api';
|
||||
|
||||
import { streamMarkdownFromSiteSpaces } from './llms-full';
|
||||
|
||||
describe('streamMarkdownFromSiteSpaces', () => {
|
||||
// Test with real mocks of the dependencies
|
||||
it('processes pages correctly with pagination', async () => {
|
||||
// Mock the dependencies by replacing them in the module
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
id: 'revision-1',
|
||||
pages: [
|
||||
{
|
||||
id: 'page-1',
|
||||
type: 'document',
|
||||
title: 'Page 1',
|
||||
path: 'page-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-2',
|
||||
type: 'document',
|
||||
title: 'Page 2',
|
||||
path: 'page-2',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-3',
|
||||
type: 'document',
|
||||
title: 'Page 3',
|
||||
path: 'page-3',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-4',
|
||||
type: 'document',
|
||||
title: 'Page 4',
|
||||
path: 'page-4',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-5',
|
||||
type: 'document',
|
||||
title: 'Page 5',
|
||||
path: 'page-5',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() =>
|
||||
Promise.resolve({
|
||||
data: '# Test Page\n\nSome content\n',
|
||||
})
|
||||
),
|
||||
};
|
||||
|
||||
const mockLinker = {
|
||||
toPathInSite: mock((path: string) => `/test/${path}`),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: mockLinker,
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
id: 'space-1',
|
||||
space: {
|
||||
id: 'space-1',
|
||||
revision: 'rev-1',
|
||||
},
|
||||
urls: {
|
||||
published: 'https://example.com',
|
||||
},
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
// Capture stream output
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Verify results
|
||||
expect(result.currentPageIndex).toBe(5); // Should process 5 pages
|
||||
expect(result.reachedLimit).toBe(false); // Under limit
|
||||
expect(chunks.length).toBe(5); // Should have 5 markdown chunks
|
||||
expect(mockDataFetcher.getRevision).toHaveBeenCalledTimes(1);
|
||||
expect(mockDataFetcher.getRevisionPageMarkdown).toHaveBeenCalledTimes(5);
|
||||
});
|
||||
|
||||
it('applies offset correctly', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: Array.from({ length: 10 }, (_, i) => ({
|
||||
id: `page-${i + 1}`,
|
||||
type: 'document',
|
||||
title: `Page ${i + 1}`,
|
||||
path: `page-${i + 1}`,
|
||||
pages: [],
|
||||
hidden: false,
|
||||
})),
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: { toPathInSite: mock((path: string) => `/${path}`) },
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example.com' },
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
3, // offset = 3
|
||||
0
|
||||
);
|
||||
|
||||
// Should process pages from index 3 onwards (7 pages)
|
||||
expect(result.currentPageIndex).toBe(10);
|
||||
expect(chunks.length).toBe(7); // 10 total - 3 offset = 7 processed
|
||||
});
|
||||
|
||||
it('handles pagination when there are more than 100 pages', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: Array.from({ length: 150 }, (_, i) => ({
|
||||
id: `page-${i + 1}`,
|
||||
type: 'document',
|
||||
title: `Page ${i + 1}`,
|
||||
path: `page-${i + 1}`,
|
||||
pages: [],
|
||||
hidden: false,
|
||||
})),
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockLinker = {
|
||||
toPathInSite: mock((path: string) => `/site/${path}`),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: mockLinker,
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example.com' },
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should only process 100 pages (default limit)
|
||||
expect(result.currentPageIndex).toBe(100);
|
||||
expect(result.reachedLimit).toBe(true);
|
||||
expect(chunks.length).toBe(101); // 100 pages + 1 next page link
|
||||
|
||||
// Check that next page link is included
|
||||
const fullContent = chunks.join('');
|
||||
expect(fullContent).toContain('[Next Page]');
|
||||
expect(fullContent).toContain('/site/llms-full.txt/1');
|
||||
});
|
||||
|
||||
it('handles multiple site spaces', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock()
|
||||
.mockReturnValueOnce(
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
id: 'page-1',
|
||||
type: 'document',
|
||||
title: 'Space 1 Page 1',
|
||||
path: 'page-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-2',
|
||||
type: 'document',
|
||||
title: 'Space 1 Page 2',
|
||||
path: 'page-2',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
)
|
||||
.mockReturnValueOnce(
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
id: 'page-3',
|
||||
type: 'document',
|
||||
title: 'Space 2 Page 1',
|
||||
path: 'page-3',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-4',
|
||||
type: 'document',
|
||||
title: 'Space 2 Page 2',
|
||||
path: 'page-4',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-5',
|
||||
type: 'document',
|
||||
title: 'Space 2 Page 3',
|
||||
path: 'page-5',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: { toPathInSite: mock((path: string) => `/${path}`) },
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpaces: SiteSpace[] = [
|
||||
{
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example1.com' },
|
||||
path: 'space-1',
|
||||
},
|
||||
{
|
||||
space: { id: 'space-2', revision: 'rev-2' },
|
||||
urls: { published: 'https://example2.com' },
|
||||
path: 'space-2',
|
||||
},
|
||||
] as SiteSpace[];
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const { streamMarkdownFromSiteSpaces } = await import('./llms-full');
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
mockSiteSpaces,
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should process all pages from both spaces (2 + 3 = 5)
|
||||
expect(result.currentPageIndex).toBe(5);
|
||||
expect(result.reachedLimit).toBe(false);
|
||||
expect(chunks.length).toBe(5);
|
||||
expect(mockDataFetcher.getRevision).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('skips site spaces without published URLs', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(),
|
||||
getRevisionPageMarkdown: mock(),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: { toPathInSite: mock((path: string) => `/${path}`) },
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: undefined }, // No published URL
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const mockController = {
|
||||
enqueue: mock(),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should not process any pages
|
||||
expect(result.currentPageIndex).toBe(0);
|
||||
expect(result.reachedLimit).toBe(false);
|
||||
expect(mockDataFetcher.getRevision).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('filters only document type pages', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
id: 'doc-1',
|
||||
type: 'document',
|
||||
title: 'Document 1',
|
||||
path: 'doc-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'group-1',
|
||||
type: 'group',
|
||||
title: 'Group 1',
|
||||
path: 'group-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'doc-2',
|
||||
type: 'document',
|
||||
title: 'Document 2',
|
||||
path: 'doc-2',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'link-1',
|
||||
type: 'link',
|
||||
title: 'Link 1',
|
||||
path: 'link-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: { toPathInSite: mock((path: string) => `/${path}`) },
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example.com' },
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should only process the 2 document pages
|
||||
expect(result.currentPageIndex).toBe(2);
|
||||
expect(chunks.length).toBe(2);
|
||||
expect(mockDataFetcher.getRevisionPageMarkdown).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -22,19 +22,24 @@ import { visit } from 'unist-util-visit';
|
||||
// or file descriptor limits.
|
||||
const MAX_CONCURRENCY = 100;
|
||||
|
||||
// Default limit for pages per batch
|
||||
const DEFAULT_PAGE_LIMIT = 100;
|
||||
|
||||
/**
|
||||
* Generate a llms-full.txt file for the site.
|
||||
* As the result can be large, we stream it as we generate it.
|
||||
*/
|
||||
export async function serveLLMsFullTxt(context: GitBookSiteContext) {
|
||||
export async function serveLLMsFullTxt(context: GitBookSiteContext, page = 0) {
|
||||
if (!checkIsRootSiteContext(context)) {
|
||||
return new Response('llms.txt is only served from the root of the site', { status: 404 });
|
||||
}
|
||||
|
||||
const offset = page * DEFAULT_PAGE_LIMIT;
|
||||
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
await streamMarkdownFromSiteStructure(context, controller);
|
||||
await streamMarkdownFromSiteStructure(context, controller, offset);
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
@@ -51,17 +56,26 @@ export async function serveLLMsFullTxt(context: GitBookSiteContext) {
|
||||
*/
|
||||
async function streamMarkdownFromSiteStructure(
|
||||
context: GitBookSiteContext,
|
||||
stream: ReadableStreamDefaultController<Uint8Array>
|
||||
stream: ReadableStreamDefaultController<Uint8Array>,
|
||||
offset: number
|
||||
): Promise<void> {
|
||||
switch (context.structure.type) {
|
||||
case 'sections':
|
||||
return streamMarkdownFromSections(
|
||||
context,
|
||||
stream,
|
||||
getSiteStructureSections(context.structure, { ignoreGroups: true })
|
||||
getSiteStructureSections(context.structure, { ignoreGroups: true }),
|
||||
offset
|
||||
);
|
||||
case 'siteSpaces':
|
||||
return streamMarkdownFromSiteSpaces(context, stream, context.structure.structure, '');
|
||||
await streamMarkdownFromSiteSpaces(
|
||||
context,
|
||||
stream,
|
||||
context.structure.structure,
|
||||
'',
|
||||
offset
|
||||
);
|
||||
return;
|
||||
default:
|
||||
assertNever(context.structure);
|
||||
}
|
||||
@@ -73,28 +87,45 @@ async function streamMarkdownFromSiteStructure(
|
||||
async function streamMarkdownFromSections(
|
||||
context: GitBookSiteContext,
|
||||
stream: ReadableStreamDefaultController<Uint8Array>,
|
||||
siteSections: SiteSection[]
|
||||
siteSections: SiteSection[],
|
||||
offset: number
|
||||
): Promise<void> {
|
||||
let currentPageIndex = 0;
|
||||
|
||||
for (const siteSection of siteSections) {
|
||||
await streamMarkdownFromSiteSpaces(
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
context,
|
||||
stream,
|
||||
siteSection.siteSpaces,
|
||||
siteSection.path
|
||||
siteSection.path,
|
||||
offset,
|
||||
currentPageIndex
|
||||
);
|
||||
currentPageIndex = result.currentPageIndex;
|
||||
|
||||
if (result.reachedLimit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream markdown from site spaces.
|
||||
*/
|
||||
async function streamMarkdownFromSiteSpaces(
|
||||
export async function streamMarkdownFromSiteSpaces(
|
||||
context: GitBookSiteContext,
|
||||
stream: ReadableStreamDefaultController<Uint8Array>,
|
||||
siteSpaces: SiteSpace[],
|
||||
basePath: string
|
||||
): Promise<void> {
|
||||
basePath: string,
|
||||
offset = 0,
|
||||
initialPageIndex = 0
|
||||
): Promise<{ currentPageIndex: number; reachedLimit: boolean }> {
|
||||
const { dataFetcher } = context;
|
||||
let totalPagesProcessed = initialPageIndex;
|
||||
|
||||
// Collect all pages first
|
||||
const allPages: Array<{ page: RevisionPageDocument; siteSpace: SiteSpace; basePath: string }> =
|
||||
[];
|
||||
|
||||
for (const siteSpace of siteSpaces) {
|
||||
const siteSpaceUrl = siteSpace.urls.published;
|
||||
@@ -109,27 +140,46 @@ async function streamMarkdownFromSiteSpaces(
|
||||
);
|
||||
const pages = getIndexablePages(revision.pages);
|
||||
|
||||
for await (const markdown of pMapIterable(
|
||||
pages,
|
||||
async ({ page }) => {
|
||||
if (page.type !== 'document') {
|
||||
return '';
|
||||
}
|
||||
|
||||
return getMarkdownForPage(
|
||||
context,
|
||||
siteSpace,
|
||||
// Add document pages to our collection
|
||||
for (const { page } of pages) {
|
||||
if (page.type === 'document') {
|
||||
allPages.push({
|
||||
page,
|
||||
joinPath(basePath, siteSpace.path)
|
||||
);
|
||||
},
|
||||
{
|
||||
concurrency: MAX_CONCURRENCY,
|
||||
siteSpace,
|
||||
basePath: joinPath(basePath, siteSpace.path),
|
||||
});
|
||||
}
|
||||
)) {
|
||||
stream.enqueue(new TextEncoder().encode(markdown));
|
||||
}
|
||||
}
|
||||
|
||||
// Apply pagination - skip pages before offset
|
||||
const pagesToProcess = allPages.slice(offset, offset + DEFAULT_PAGE_LIMIT);
|
||||
totalPagesProcessed = offset;
|
||||
|
||||
// Process the pages
|
||||
for await (const markdown of pMapIterable(
|
||||
pagesToProcess,
|
||||
async ({ page, siteSpace, basePath }) => {
|
||||
return getMarkdownForPage(context, siteSpace, page, basePath);
|
||||
},
|
||||
{
|
||||
concurrency: MAX_CONCURRENCY,
|
||||
}
|
||||
)) {
|
||||
stream.enqueue(new TextEncoder().encode(markdown));
|
||||
totalPagesProcessed++;
|
||||
}
|
||||
|
||||
// Check if there are more pages and add next page link if needed
|
||||
const hasMorePages = allPages.length > offset + DEFAULT_PAGE_LIMIT;
|
||||
if (hasMorePages) {
|
||||
const nextPage = Math.floor(offset / DEFAULT_PAGE_LIMIT) + 1;
|
||||
const nextPageUrl = context.linker.toPathInSite(`llms-full.txt/${nextPage}`);
|
||||
const nextPageLink = `\n\n---\n\n[Next Page](${nextPageUrl})\n\n`;
|
||||
stream.enqueue(new TextEncoder().encode(nextPageLink));
|
||||
}
|
||||
|
||||
return { currentPageIndex: totalPagesProcessed, reachedLimit: hasMorePages };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user