mirror of
https://github.com/GitbookIO/gitbook.git
synced 2026-09-27 12:39:08 +00:00
Compare commits
29 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1ae09b1ec5 | |||
| f71ab8426c | |||
| a3d1558f99 | |||
| ca53869e2d | |||
| 33598e665b | |||
| 9045ba5248 | |||
| 6cd9f2aaaa | |||
| 6d4a1653a2 | |||
| 9ce2759e1b | |||
| 5663064b46 | |||
| 6246b106af | |||
| 9ad85cb30c | |||
| 25bcff6eff | |||
| f232b8fff5 | |||
| e7f9bc2565 | |||
| 7061fc2223 | |||
| 21053f56fb | |||
| 2746ffc000 | |||
| 568ee3a660 | |||
| 420bf4a6f6 | |||
| 72cffdd14b | |||
| 0c9e3952b7 | |||
| 07be0e224c | |||
| c765394914 | |||
| ab96ee806b | |||
| 7cc6090e03 | |||
| 4b78672135 | |||
| 10289e4881 | |||
| 8b6a6df1d6 |
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@gitbook/icons": minor
|
||||
"gitbook": patch
|
||||
---
|
||||
|
||||
Update icon usage to render svg symbols rather than file.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"gitbook": patch
|
||||
---
|
||||
|
||||
Fix links to other spaces/sections in the llms.txt.
|
||||
@@ -356,7 +356,7 @@
|
||||
"react-dom": "catalog:",
|
||||
},
|
||||
"catalog": {
|
||||
"@gitbook/api": "0.179.0",
|
||||
"@gitbook/api": "0.180.0",
|
||||
"@scalar/api-client-react": "^1.3.46",
|
||||
"@tsconfig/node20": "^20.1.6",
|
||||
"@tsconfig/strictest": "^2.0.6",
|
||||
@@ -752,7 +752,7 @@
|
||||
|
||||
"@fortawesome/fontawesome-svg-core": ["@fortawesome/fontawesome-svg-core@7.2.0", "", { "dependencies": { "@fortawesome/fontawesome-common-types": "7.2.0" } }, "sha512-6639htZMjEkwskf3J+e6/iar+4cTNM9qhoWuRfj9F3eJD6r7iCzV1SWnQr2Mdv0QT0suuqU8BoJCZUyCtP9R4Q=="],
|
||||
|
||||
"@gitbook/api": ["@gitbook/api@0.179.0", "", { "dependencies": { "event-iterator": "^2.0.0", "eventsource-parser": "^3.0.0" } }, "sha512-DrD/Pdfkdv6WWQb1nE0o91O4+gzIhRh+v8l9fhL5Wq57yt0Tk0kTCUycVpI1+H9qUt39bopN4O+KTns4mR2eLQ=="],
|
||||
"@gitbook/api": ["@gitbook/api@0.180.0", "", { "dependencies": { "event-iterator": "^2.0.0", "eventsource-parser": "^3.0.0" } }, "sha512-jGPP9cVGDLqVv1YjuZMpoVNfD0Yj30vVbkYwnVc4B2GRhYENJm/27x+nAzG7379YNilWOneqE245NyIWNgTu1w=="],
|
||||
|
||||
"@gitbook/browser-types": ["@gitbook/browser-types@workspace:packages/browser-types"],
|
||||
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@
|
||||
"catalog": {
|
||||
"@tsconfig/strictest": "^2.0.6",
|
||||
"@tsconfig/node20": "^20.1.6",
|
||||
"@gitbook/api": "0.179.0",
|
||||
"@gitbook/api": "0.180.0",
|
||||
"@scalar/api-client-react": "^1.3.46",
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
|
||||
@@ -36,4 +36,4 @@ screenshots/
|
||||
# cloudflare
|
||||
.open-next
|
||||
.wrangler
|
||||
worker-configuration.d.ts
|
||||
worker-configuration.d.ts
|
||||
|
||||
@@ -146,6 +146,11 @@ export const headerLinks: CustomizationHeaderItem[] = [
|
||||
},
|
||||
];
|
||||
|
||||
type IconURLState = { state: 'pending'; uri: null } | { state: 'loaded'; uri: string };
|
||||
type IconStateWindow = Window & {
|
||||
__ICONS_STATES__?: Record<string, IconURLState>;
|
||||
};
|
||||
|
||||
export async function waitForCookiesDialog(page: Page) {
|
||||
const dialog = page.getByTestId('cookies-dialog');
|
||||
await expect(dialog).toBeVisible({
|
||||
@@ -396,11 +401,9 @@ export function getCustomizationURL(partial: DeepPartial<SiteCustomizationSettin
|
||||
*/
|
||||
export async function waitForIcons(page: Page) {
|
||||
await page.waitForFunction(() => {
|
||||
const urlStates: Record<
|
||||
string,
|
||||
{ state: 'pending'; uri: null } | { state: 'loaded'; uri: string }
|
||||
> = (window as any).__ICONS_STATES__ || {};
|
||||
(window as any).__ICONS_STATES__ = urlStates;
|
||||
const iconWindow = window as IconStateWindow;
|
||||
const urlStates: Record<string, IconURLState> = iconWindow.__ICONS_STATES__ || {};
|
||||
iconWindow.__ICONS_STATES__ = urlStates;
|
||||
|
||||
const fetchSvgAsDataUri = async (url: string): Promise<string> => {
|
||||
const response = await fetch(url);
|
||||
@@ -433,6 +436,20 @@ export async function waitForIcons(page: Page) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const svgSymbol = icon.querySelector('[data-testid="symbol-use"]');
|
||||
if (svgSymbol) {
|
||||
if (icon.dataset.gbIconSymbolState === 'loaded') {
|
||||
return true;
|
||||
}
|
||||
|
||||
const href = svgSymbol.getAttribute('href') ?? svgSymbol.getAttribute('xlink:href');
|
||||
if (!href?.startsWith('#')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return document.getElementById(href.slice(1)) instanceof SVGElement;
|
||||
}
|
||||
|
||||
const state = icon.getAttribute('data-argos-state');
|
||||
|
||||
if (state === 'pending') {
|
||||
|
||||
@@ -92,7 +92,8 @@
|
||||
},
|
||||
"staging": {
|
||||
"vars": {
|
||||
"OPEN_NEXT_REQUEST_ID_HEADER": "true"
|
||||
"OPEN_NEXT_REQUEST_ID_HEADER": "true",
|
||||
"NEXT_PRIVATE_DEBUG_CACHE": "true"
|
||||
},
|
||||
"r2_buckets": [
|
||||
{
|
||||
@@ -136,7 +137,8 @@
|
||||
// This is a bit misleading, but it means that we can have 500 concurrent revalidations
|
||||
// This means that we'll have up to 100 durable objects instance running at the same time
|
||||
"MAX_REVALIDATE_CONCURRENCY": "100",
|
||||
"OPEN_NEXT_REQUEST_ID_HEADER": "true"
|
||||
"OPEN_NEXT_REQUEST_ID_HEADER": "true",
|
||||
"NEXT_PRIVATE_DEBUG_CACHE": "true"
|
||||
},
|
||||
"r2_buckets": [
|
||||
{
|
||||
|
||||
@@ -29,6 +29,7 @@ export class GitbookIncrementalCache implements IncrementalCache {
|
||||
cacheType?: CacheType
|
||||
): Promise<WithLastModified<CacheValue<CacheType>> | null> {
|
||||
const cacheKey = this.getR2Key(key, cacheType);
|
||||
console.log(`[GitbookIncrementalCache] Getting cache for key: ${cacheKey}`, key, cacheType);
|
||||
|
||||
const r2 = getCloudflareContext().env[BINDING_NAME];
|
||||
if (!r2) throw new Error('No R2 bucket');
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { withRegionalCache } from '@opennextjs/cloudflare/overrides/incremental-cache/regional-cache';
|
||||
import { GitbookIncrementalCache } from './incrementalCache';
|
||||
|
||||
export default withRegionalCache(new GitbookIncrementalCache(), {
|
||||
mode: 'long-lived',
|
||||
// Because of a race condition, the middleware may have populated the cache entry before `cache.match` had time to run on the server.
|
||||
// TODO: We should bypass the incremental cache entirely when the interceptor has caught the request. Should be done in OpenNext.
|
||||
bypassTagCacheOnCacheHit: false,
|
||||
defaultLongLivedTtlSec: 60 * 60 * 24 /* 24 hours */,
|
||||
// We don't want to update the cache entry on every cache hit
|
||||
shouldLazilyUpdateOnCacheHit: false,
|
||||
});
|
||||
// export default withRegionalCache(new GitbookIncrementalCache(), {
|
||||
// mode: 'long-lived',
|
||||
// // Because of a race condition, the middleware may have populated the cache entry before `cache.match` had time to run on the server.
|
||||
// // TODO: We should bypass the incremental cache entirely when the interceptor has caught the request. Should be done in OpenNext.
|
||||
// bypassTagCacheOnCacheHit: false,
|
||||
// //TODO: remove, reducing cache ttl of regional cache to help debugging
|
||||
// defaultLongLivedTtlSec: 5 * 60 /* 5 minutes */,
|
||||
// // We don't want to update the cache entry on every cache hit
|
||||
// shouldLazilyUpdateOnCacheHit: false,
|
||||
// });
|
||||
|
||||
//TODO: reenable regional cache once we know what's going on
|
||||
export default new GitbookIncrementalCache();
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server';
|
||||
|
||||
import { getIconSymbol } from '@/lib/icons/symbols';
|
||||
import { getIconSymbolId } from '@gitbook/icons';
|
||||
|
||||
export async function GET(
|
||||
_request: NextRequest,
|
||||
{ params }: { params: Promise<{ style: string; icon: string }> }
|
||||
) {
|
||||
const { style, icon } = await params;
|
||||
const symbol = await getIconSymbol(style, icon, getIconSymbolId(style, icon));
|
||||
|
||||
if (!symbol) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Symbol not found',
|
||||
},
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
return new NextResponse(symbol.document, {
|
||||
headers: {
|
||||
'content-type': 'image/svg+xml; charset=utf-8',
|
||||
'cache-control': 'public, max-age=31536000, immutable',
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import { getContentLocale, getSpaceLanguage } from '@/intl/server';
|
||||
import { getAssetURL } from '@/lib/assets';
|
||||
import { tcls } from '@/lib/tailwind';
|
||||
|
||||
import { IconSpriteDefinitions } from './IconSpriteDefinitions';
|
||||
import { RootLayoutClientContexts } from './RootLayoutClientContexts';
|
||||
|
||||
import './globals.css';
|
||||
@@ -76,6 +77,9 @@ export async function CustomizationRootLayout(props: {
|
||||
const sidebarStyles = getSidebarStyles(customization);
|
||||
const { infoColor, successColor, warningColor, dangerColor } = getSemanticColors(customization);
|
||||
const fontData = getFontData(customization.styling.font, 'content');
|
||||
const iconStyle =
|
||||
('icons' in customization.styling ? apiToIconsStyles[customization.styling.icons] : null) ||
|
||||
IconStyle.Regular;
|
||||
// Temporarily add a if here while the cache is being warmed up.
|
||||
// We can remove the condition after 14-07-2025.
|
||||
const monospaceFontData = customization.styling.monospaceFont
|
||||
@@ -194,15 +198,14 @@ export async function CustomizationRootLayout(props: {
|
||||
assetsURL: getAssetURL('icons'),
|
||||
},
|
||||
}}
|
||||
iconStyle={
|
||||
('icons' in customization.styling
|
||||
? apiToIconsStyles[customization.styling.icons]
|
||||
: null) || IconStyle.Regular
|
||||
}
|
||||
renderMode="symbol"
|
||||
symbolLoaderURL="/~gitbook/icons/symbol"
|
||||
iconStyle={iconStyle}
|
||||
>
|
||||
<RootLayoutClientContexts language={language}>
|
||||
{children}
|
||||
</RootLayoutClientContexts>
|
||||
<IconSpriteDefinitions />
|
||||
</IconsProvider>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { getIconSymbol } from '@/lib/icons/symbols';
|
||||
import {
|
||||
type RegisteredIconSymbol,
|
||||
clearRegisteredServerIconSymbols,
|
||||
getRegisteredServerIconSymbols,
|
||||
} from '@gitbook/icons';
|
||||
|
||||
/**
|
||||
* Emits the subset of icon symbols that were rendered during the current request.
|
||||
*/
|
||||
export async function IconSpriteDefinitions() {
|
||||
const registered: Map<string, RegisteredIconSymbol> = new Map(
|
||||
getRegisteredServerIconSymbols().map((symbol: RegisteredIconSymbol) => [
|
||||
`${symbol.style}/${symbol.icon}`,
|
||||
symbol,
|
||||
])
|
||||
);
|
||||
|
||||
if (registered.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const definitions = (
|
||||
await Promise.all(
|
||||
[...registered.values()].map((symbol) => {
|
||||
return getIconSymbol(symbol.style, symbol.icon, symbol.symbolId);
|
||||
})
|
||||
)
|
||||
).filter((symbol): symbol is NonNullable<typeof symbol> => !!symbol);
|
||||
|
||||
clearRegisteredServerIconSymbols();
|
||||
|
||||
if (!definitions.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<svg
|
||||
id="gb-icon-sprite-root"
|
||||
data-testid="icon-sprite-root"
|
||||
aria-hidden="true"
|
||||
focusable="false"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
style={{
|
||||
position: 'absolute',
|
||||
width: 0,
|
||||
height: 0,
|
||||
overflow: 'hidden',
|
||||
pointerEvents: 'none',
|
||||
}}
|
||||
dangerouslySetInnerHTML={{
|
||||
__html: definitions.map((symbol) => symbol.symbol).join(''),
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -77,12 +77,17 @@ function buildLangIndex(pages: RawIndexPage[]): Document<IndexPage> {
|
||||
});
|
||||
|
||||
for (const page of pages) {
|
||||
index.add({
|
||||
id: page.id,
|
||||
title: page.title,
|
||||
description: page.description ?? null,
|
||||
siteSpaceId: page.siteSpaceId,
|
||||
});
|
||||
index
|
||||
.addAsync({
|
||||
id: page.id,
|
||||
title: page.title,
|
||||
description: page.description ?? null,
|
||||
siteSpaceId: page.siteSpaceId,
|
||||
})
|
||||
.catch(() => {
|
||||
// We just ignore these errors, it's not worth failing the whole index for a single bad record
|
||||
// And we still have remote search as a fallback for these cases
|
||||
});
|
||||
|
||||
cachedPageData.set(`${page.siteSpaceId}:${page.id}`, {
|
||||
pathname: page.pathname,
|
||||
|
||||
@@ -10,7 +10,7 @@ import { LoadIntegrations } from '@/components/Integrations';
|
||||
import { SpaceLayout } from '@/components/SpaceLayout';
|
||||
import type { VisitorAuthClaims } from '@/lib/adaptive';
|
||||
import { buildVersion } from '@/lib/build';
|
||||
import { GITBOOK_API_PUBLIC_URL, GITBOOK_ASSETS_URL, GITBOOK_ICONS_URL } from '@/lib/env';
|
||||
import { GITBOOK_API_PUBLIC_URL, GITBOOK_ASSETS_URL } from '@/lib/env';
|
||||
import { getResizedImageURL } from '@/lib/images';
|
||||
import { isSiteIndexable } from '@/lib/seo';
|
||||
import { AIContextProvider } from '../AI';
|
||||
@@ -34,7 +34,6 @@ export async function SiteLayout(props: {
|
||||
const scripts = withTracking ? context.scripts : [];
|
||||
|
||||
ReactDOM.preconnect(GITBOOK_API_PUBLIC_URL);
|
||||
ReactDOM.preconnect(GITBOOK_ICONS_URL);
|
||||
if (GITBOOK_ASSETS_URL) {
|
||||
ReactDOM.preconnect(GITBOOK_ASSETS_URL);
|
||||
}
|
||||
|
||||
@@ -5,7 +5,12 @@ import {
|
||||
throwIfDataError,
|
||||
} from '@/lib/data';
|
||||
import { getLogger } from '@/lib/logger';
|
||||
import { getLocalizedTitle, getSiteStructureSections } from '@/lib/sites';
|
||||
import {
|
||||
findSiteSpaceBy,
|
||||
getFallbackSiteSpacePath,
|
||||
getLocalizedTitle,
|
||||
getSiteStructureSections,
|
||||
} from '@/lib/sites';
|
||||
import type {
|
||||
ChangeRequest,
|
||||
PublishedSiteContent,
|
||||
@@ -386,6 +391,54 @@ export async function fetchSiteContextByIds(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a site context scoped to a specific site space.
|
||||
* This keeps the site structure from the current context while resolving content
|
||||
* against the target space revision.
|
||||
*/
|
||||
export async function fetchSiteContextForSiteSpace(
|
||||
baseContext: GitBookSiteContext,
|
||||
siteSpace: SiteSpace
|
||||
): Promise<GitBookSiteContext> {
|
||||
const found = findSiteSpaceBy(baseContext.structure, (entry) => entry.id === siteSpace.id);
|
||||
|
||||
if (!found) {
|
||||
throw new Error(`Site space "${siteSpace.id}" not found in site structure`);
|
||||
}
|
||||
|
||||
const spaceContext = await fetchSpaceContextByIds(baseContext, {
|
||||
space: siteSpace.space.id,
|
||||
shareKey: baseContext.shareKey,
|
||||
changeRequest: undefined,
|
||||
revision: siteSpace.space.revision,
|
||||
});
|
||||
|
||||
const siteSpaces =
|
||||
baseContext.structure.type === 'siteSpaces'
|
||||
? baseContext.structure.structure
|
||||
: (found.siteSection?.siteSpaces ?? baseContext.siteSpaces);
|
||||
|
||||
return {
|
||||
...baseContext,
|
||||
...spaceContext,
|
||||
locale: siteSpace.space.language ?? spaceContext.locale,
|
||||
linker: baseContext.linker.withOtherSiteSpace({
|
||||
spaceBasePath: getFallbackSiteSpacePath(baseContext, siteSpace),
|
||||
}),
|
||||
siteSpace,
|
||||
siteSpaces,
|
||||
visibleSiteSpaces: filterHiddenSiteSpaces(siteSpaces),
|
||||
sections:
|
||||
baseContext.sections && found.siteSection
|
||||
? { ...baseContext.sections, current: found.siteSection }
|
||||
: baseContext.sections,
|
||||
visibleSections:
|
||||
baseContext.visibleSections && found.siteSection
|
||||
? { ...baseContext.visibleSections, current: found.siteSection }
|
||||
: baseContext.visibleSections,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a space context by IDs.
|
||||
*/
|
||||
|
||||
@@ -94,11 +94,18 @@ export function createDataFetcher(
|
||||
});
|
||||
},
|
||||
getRevisionPageDocument(params) {
|
||||
return getRevisionPageDocument(input, {
|
||||
spaceId: params.spaceId,
|
||||
revisionId: params.revisionId,
|
||||
pageId: params.pageId,
|
||||
});
|
||||
return getRevisionPageDocument(
|
||||
input,
|
||||
{
|
||||
spaceId: params.spaceId,
|
||||
revisionId: params.revisionId,
|
||||
pageId: params.pageId,
|
||||
},
|
||||
// We pass the name of the function to distinguish between cache entries
|
||||
// By default Next only uses the arguments, the filename and the position of the function inside the file
|
||||
// By adding a dummy argument with the function name, we can change the order without breaking anything
|
||||
'getRevisionPageDocument'
|
||||
);
|
||||
},
|
||||
getRevisionReusableContentDocument(params) {
|
||||
return getRevisionReusableContentDocument(input, {
|
||||
@@ -325,6 +332,7 @@ const getRevisionPageMarkdown = cache(
|
||||
params.pageId,
|
||||
{
|
||||
format: 'markdown',
|
||||
'format.markdown.refs': 'stable',
|
||||
},
|
||||
{
|
||||
...noCacheFetchOptions,
|
||||
@@ -350,7 +358,9 @@ const getRevisionPageMarkdown = cache(
|
||||
const getRevisionPageDocument = cache(
|
||||
async (
|
||||
input: DataFetcherInput,
|
||||
params: { spaceId: string; revisionId: string; pageId: string }
|
||||
params: { spaceId: string; revisionId: string; pageId: string },
|
||||
// used only to bust the cache
|
||||
_functionName: string
|
||||
) => {
|
||||
'use cache: remote';
|
||||
return wrapCacheDataFetcherError(async () => {
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
import 'server-only';
|
||||
|
||||
import { getAssetURL } from '@/lib/assets';
|
||||
import { GITBOOK_ICONS_TOKEN, GITBOOK_ICONS_URL } from '@/lib/env';
|
||||
import { joinPath, joinPathWithBaseURL } from '@/lib/paths';
|
||||
|
||||
const ICON_ASSET_VERSION = '2';
|
||||
const rawSvgPromises = new Map<string, Promise<string | null>>();
|
||||
const styleSpritePromises = new Map<string, Promise<Map<string, IconSymbolSource> | null>>();
|
||||
const svgPattern = /<svg\b([^>]*)>([\s\S]*?)<\/svg>\s*$/i;
|
||||
const symbolPattern = /<symbol\b([^>]*)>([\s\S]*?)<\/symbol>/gi;
|
||||
const viewBoxPattern = /\bviewBox="([^"]+)"/i;
|
||||
const idPattern = /\bid="([^"]+)"/i;
|
||||
const commentPattern = /<!--[\s\S]*?-->/g;
|
||||
|
||||
interface IconSymbolSource {
|
||||
viewBox: string;
|
||||
markup: string;
|
||||
}
|
||||
|
||||
function escapeAttribute(value: string): string {
|
||||
return value
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>');
|
||||
}
|
||||
|
||||
function getIconAssetBaseURL(style: string): string {
|
||||
if (style === 'custom-icons') {
|
||||
return getAssetURL('icons');
|
||||
}
|
||||
|
||||
return GITBOOK_ICONS_URL;
|
||||
}
|
||||
|
||||
function getIconAssetURL(style: string, icon: string): string {
|
||||
const url = new URL(
|
||||
joinPathWithBaseURL(getIconAssetBaseURL(style), joinPath('svgs', style, `${icon}.svg`))
|
||||
);
|
||||
url.searchParams.set('v', ICON_ASSET_VERSION);
|
||||
|
||||
if (style !== 'custom-icons' && GITBOOK_ICONS_TOKEN) {
|
||||
url.searchParams.set('token', GITBOOK_ICONS_TOKEN);
|
||||
}
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function getStyleSpriteAssetURL(style: string): string {
|
||||
const url = new URL(
|
||||
joinPathWithBaseURL(getIconAssetBaseURL(style), joinPath('sprites', `${style}.svg`))
|
||||
);
|
||||
url.searchParams.set('v', ICON_ASSET_VERSION);
|
||||
|
||||
if (style !== 'custom-icons' && GITBOOK_ICONS_TOKEN) {
|
||||
url.searchParams.set('token', GITBOOK_ICONS_TOKEN);
|
||||
}
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function parseRawSVG(document: string): IconSymbolSource | null {
|
||||
const svgMatch = document.match(svgPattern);
|
||||
if (!svgMatch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const svgAttributes = svgMatch[1];
|
||||
const rawMarkup = svgMatch[2];
|
||||
if (!svgAttributes || rawMarkup === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const viewBoxMatch = svgAttributes.match(viewBoxPattern);
|
||||
const viewBox = viewBoxMatch?.[1];
|
||||
if (!viewBox) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
viewBox,
|
||||
markup: rawMarkup.replace(commentPattern, '').trim(),
|
||||
};
|
||||
}
|
||||
|
||||
function parseStyleSprite(document: string): Map<string, IconSymbolSource> | null {
|
||||
const symbols = new Map<string, IconSymbolSource>();
|
||||
|
||||
for (const match of document.matchAll(symbolPattern)) {
|
||||
const symbolAttributes = match[1];
|
||||
const rawMarkup = match[2];
|
||||
|
||||
if (!symbolAttributes || rawMarkup === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const idMatch = symbolAttributes.match(idPattern);
|
||||
const viewBoxMatch = symbolAttributes.match(viewBoxPattern);
|
||||
const icon = idMatch?.[1];
|
||||
const viewBox = viewBoxMatch?.[1];
|
||||
|
||||
if (!icon || !viewBox) {
|
||||
continue;
|
||||
}
|
||||
|
||||
symbols.set(icon, {
|
||||
viewBox,
|
||||
markup: rawMarkup.replace(commentPattern, '').trim(),
|
||||
});
|
||||
}
|
||||
|
||||
return symbols.size > 0 ? symbols : null;
|
||||
}
|
||||
|
||||
function buildSymbolMarkup(symbolId: string, viewBox: string, markup: string) {
|
||||
return `<symbol id="${escapeAttribute(symbolId)}" viewBox="${escapeAttribute(viewBox)}" overflow="visible">${markup}</symbol>`;
|
||||
}
|
||||
|
||||
function buildSymbolDocument(symbolId: string, symbol: string) {
|
||||
return `<svg xmlns="http://www.w3.org/2000/svg"><defs>${symbol}</defs><use href="#${escapeAttribute(symbolId)}"/></svg>`;
|
||||
}
|
||||
|
||||
async function fetchRawSVG(style: string, icon: string): Promise<string | null> {
|
||||
const cacheKey = `${style}/${icon}`;
|
||||
const existing = rawSvgPromises.get(cacheKey);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const request = fetch(getIconAssetURL(style, icon), {
|
||||
cache: 'force-cache',
|
||||
})
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return response.text();
|
||||
})
|
||||
.catch(() => null);
|
||||
|
||||
rawSvgPromises.set(cacheKey, request);
|
||||
return request;
|
||||
}
|
||||
|
||||
async function fetchStyleSprite(style: string): Promise<Map<string, IconSymbolSource> | null> {
|
||||
const existing = styleSpritePromises.get(style);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const request = fetch(getStyleSpriteAssetURL(style), {
|
||||
cache: 'force-cache',
|
||||
})
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return parseStyleSprite(await response.text());
|
||||
})
|
||||
.catch(() => null);
|
||||
|
||||
styleSpritePromises.set(style, request);
|
||||
return request;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve one icon entry from the raw SVG source and serialize it for sprite injection or
|
||||
* same-origin lazy loading.
|
||||
*/
|
||||
export async function getIconSymbol(style: string, icon: string, symbolId: string) {
|
||||
const spriteSymbols = await fetchStyleSprite(style);
|
||||
let source = spriteSymbols?.get(icon) ?? null;
|
||||
|
||||
if (!source) {
|
||||
const rawSVG = await fetchRawSVG(style, icon);
|
||||
if (!rawSVG) {
|
||||
return null;
|
||||
}
|
||||
|
||||
source = parseRawSVG(rawSVG);
|
||||
}
|
||||
|
||||
if (!source) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const symbol = buildSymbolMarkup(symbolId, source.viewBox, source.markup);
|
||||
|
||||
return {
|
||||
style,
|
||||
icon,
|
||||
symbolId,
|
||||
viewBox: source.viewBox,
|
||||
markup: source.markup,
|
||||
symbol,
|
||||
document: buildSymbolDocument(symbolId, symbol),
|
||||
};
|
||||
}
|
||||
@@ -257,6 +257,9 @@ export function linkerWithAbsoluteURLs(linker: GitBookLinker): GitBookLinker {
|
||||
export function linkerWithMarkdownPages(linker: GitBookLinker): GitBookLinker {
|
||||
const self: GitBookLinker = {
|
||||
...linker,
|
||||
fork: (override) => linkerWithMarkdownPages(linker.fork(override)),
|
||||
withOtherSiteSpace: (override) =>
|
||||
linkerWithMarkdownPages(linker.withOtherSiteSpace(override)),
|
||||
toPathForPage: (input) => {
|
||||
return self.toPathForPagePath({
|
||||
path: input.page.path,
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import path from 'node:path';
|
||||
import type { GitBookAnyContext, GitBookSiteContext } from '@/lib/context';
|
||||
import {
|
||||
type GitBookAnyContext,
|
||||
type GitBookSiteContext,
|
||||
fetchSiteContextForSiteSpace,
|
||||
} from '@/lib/context';
|
||||
import { DataFetcherError, throwIfDataError } from '@/lib/data';
|
||||
import type { ResolvedPagePath } from '@/lib/pages';
|
||||
import { getIndexablePages } from '@/lib/sitemap';
|
||||
import { getFallbackSiteSpacePath } from '@/lib/sites';
|
||||
import { getMarkdownForPagesTree } from '@/routes/llms';
|
||||
import {
|
||||
type RevisionPageDocument,
|
||||
@@ -78,33 +81,29 @@ export async function getMarkdownForPageInSpace(
|
||||
siteSpace: SiteSpace,
|
||||
page: RevisionPageDocument | RevisionPageGroup
|
||||
): Promise<string> {
|
||||
const { dataFetcher } = context;
|
||||
const spaceBasePath = getFallbackSiteSpacePath(context, siteSpace);
|
||||
const linker = context.linker.withOtherSiteSpace({
|
||||
spaceBasePath,
|
||||
});
|
||||
const siteSpaceContext = await fetchSiteContextForSiteSpace(context, siteSpace);
|
||||
|
||||
// Handle group pages (pages with no content that list their children)
|
||||
if (page.type === RevisionPageType.Group) {
|
||||
return renderGroupPageMarkdown({ linker, page });
|
||||
return renderGroupPageMarkdown({ linker: siteSpaceContext.linker, page });
|
||||
}
|
||||
|
||||
const rawMarkdown = await throwIfDataError(
|
||||
dataFetcher.getRevisionPageMarkdown({
|
||||
spaceId: siteSpace.space.id,
|
||||
revisionId: siteSpace.space.revision,
|
||||
siteSpaceContext.dataFetcher.getRevisionPageMarkdown({
|
||||
spaceId: siteSpaceContext.space.id,
|
||||
revisionId: siteSpaceContext.revisionId,
|
||||
pageId: page.id,
|
||||
})
|
||||
);
|
||||
|
||||
const tree = await fromPageMarkdown(context, {
|
||||
const tree = await fromPageMarkdown(siteSpaceContext, {
|
||||
markdown: rawMarkdown,
|
||||
pagePath: page.path,
|
||||
});
|
||||
|
||||
// Handle empty document pages which have children (same as getMarkdownForPage)
|
||||
if (isEmptyMarkdownPage(tree) && page.pages.length > 0) {
|
||||
return renderGroupPageMarkdown({ linker, page });
|
||||
return renderGroupPageMarkdown({ linker: siteSpaceContext.linker, page });
|
||||
}
|
||||
|
||||
return toPageMarkdown(tree);
|
||||
@@ -231,6 +230,7 @@ async function rewriteMarkdownLinks(
|
||||
const pending: Array<Promise<void>> = [];
|
||||
|
||||
visit(tree, 'link', (node: Link) => {
|
||||
const isMention = isMentionLike(node);
|
||||
const original = node.url;
|
||||
|
||||
// Skip anchors, mailto:, http(s):, protocol-like
|
||||
@@ -246,10 +246,35 @@ async function rewriteMarkdownLinks(
|
||||
const resolved = await resolveContentRef(contentRef, context);
|
||||
if (resolved?.href) {
|
||||
node.url = resolved.href;
|
||||
} else {
|
||||
// We use an absolute URL so that crawler don't follow it.
|
||||
node.url = `broken://${original.startsWith('/') ? original.slice(1) : original}`;
|
||||
}
|
||||
|
||||
if (isMention) {
|
||||
// Replace the text for mentions as otherwise it contains the raw ref
|
||||
if (resolved) {
|
||||
node.children = [
|
||||
{
|
||||
type: 'text',
|
||||
value: resolved.text,
|
||||
},
|
||||
];
|
||||
} else {
|
||||
node.children = [
|
||||
{
|
||||
type: 'text',
|
||||
value: 'Broken mention',
|
||||
},
|
||||
];
|
||||
}
|
||||
node.title = undefined;
|
||||
}
|
||||
})()
|
||||
);
|
||||
} else {
|
||||
// DEPRECATED: to be removed once rollout for getRevisionPageMarkdown is done
|
||||
//
|
||||
// Resolve against the current page’s directory and strip any leading “/” or "../"
|
||||
// Sometimes the path can be "../" if we are on the default section
|
||||
// but it means we are just at the root of the site.
|
||||
@@ -267,3 +292,16 @@ async function rewriteMarkdownLinks(
|
||||
|
||||
return tree;
|
||||
}
|
||||
|
||||
function isMentionLike(node: Link) {
|
||||
if (node.title === 'mention') {
|
||||
return true;
|
||||
}
|
||||
|
||||
const singleText =
|
||||
node.children.length === 1 && node.children[0]?.type === 'text' ? node.children[0] : null;
|
||||
if (!singleText) {
|
||||
return false;
|
||||
}
|
||||
return singleText?.value === node.url;
|
||||
}
|
||||
|
||||
@@ -586,7 +586,7 @@ async function createContextForSpace(
|
||||
}
|
||||
|
||||
/**
|
||||
* When the API outputs markdown, it can sometimes format the content-ref into a strings that can be parsed back.
|
||||
* When the API outputs markdown with `format.markdown.refs: stable`, the content refs are formatted this way.
|
||||
*/
|
||||
export function resolveStringContentRef(src: string): ContentRef | null {
|
||||
for (const resolver of Object.values(RESOLVERS)) {
|
||||
|
||||
@@ -8,7 +8,7 @@ export function isRollout({
|
||||
discriminator: string;
|
||||
percentageRollout: number;
|
||||
}): boolean {
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
if (process.env.NODE_ENV === 'development' || process.env.VERCEL_ENV === 'preview') {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import type { RevisionPage } from '@gitbook/api';
|
||||
|
||||
import { getIndexablePages } from './sitemap';
|
||||
|
||||
describe('getIndexablePages', () => {
|
||||
it('includes hidden pages when they remain indexable', () => {
|
||||
const pages: RevisionPage[] = [
|
||||
{
|
||||
id: 'visible-page',
|
||||
type: 'document',
|
||||
title: 'Visible page',
|
||||
path: 'visible-page',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'hidden-page',
|
||||
type: 'document',
|
||||
title: 'Hidden page',
|
||||
path: 'hidden-page',
|
||||
pages: [],
|
||||
hidden: true,
|
||||
},
|
||||
] as RevisionPage[];
|
||||
|
||||
expect(getIndexablePages(pages).map(({ page }) => page.id)).toEqual([
|
||||
'visible-page',
|
||||
'hidden-page',
|
||||
]);
|
||||
});
|
||||
|
||||
it('excludes descendants of pages blocked from indexing', () => {
|
||||
const pages: RevisionPage[] = [
|
||||
{
|
||||
id: 'parent',
|
||||
type: 'group',
|
||||
title: 'Parent',
|
||||
path: 'parent',
|
||||
hidden: false,
|
||||
noRobotsIndex: true,
|
||||
pages: [
|
||||
{
|
||||
id: 'child',
|
||||
type: 'document',
|
||||
title: 'Child',
|
||||
path: 'parent/child',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
] as RevisionPage[];
|
||||
|
||||
expect(getIndexablePages(pages)).toEqual([]);
|
||||
});
|
||||
|
||||
it('includes documents nested under groups', () => {
|
||||
const pages: RevisionPage[] = [
|
||||
{
|
||||
id: 'outer-group',
|
||||
type: 'group',
|
||||
title: 'Outer group',
|
||||
path: 'outer-group',
|
||||
hidden: false,
|
||||
pages: [
|
||||
{
|
||||
id: 'inner-group',
|
||||
type: 'group',
|
||||
title: 'Inner group',
|
||||
path: 'outer-group/inner-group',
|
||||
hidden: false,
|
||||
pages: [
|
||||
{
|
||||
id: 'nested-page',
|
||||
type: 'document',
|
||||
title: 'Nested page',
|
||||
path: 'outer-group/inner-group/nested-page',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
] as RevisionPage[];
|
||||
|
||||
expect(getIndexablePages(pages).map(({ page }) => page.id)).toEqual(['nested-page']);
|
||||
});
|
||||
});
|
||||
@@ -9,27 +9,35 @@ export type FlatPageEntry = { page: RevisionPageDocument; depth: number };
|
||||
*/
|
||||
function flattenPages(
|
||||
rootPages: RevisionPage[],
|
||||
filter: (page: RevisionPageDocument | RevisionPageGroup) => boolean
|
||||
filter: (
|
||||
page: RevisionPageDocument | RevisionPageGroup,
|
||||
ancestors: Array<RevisionPageDocument | RevisionPageGroup>
|
||||
) => boolean
|
||||
): FlatPageEntry[] {
|
||||
const flattenPage = (
|
||||
page: RevisionPageDocument | RevisionPageGroup,
|
||||
depth: number
|
||||
depth: number,
|
||||
ancestors: Array<RevisionPageDocument | RevisionPageGroup>
|
||||
): FlatPageEntry[] => {
|
||||
const allowed = filter(page);
|
||||
const allowed = filter(page, ancestors);
|
||||
if (!allowed) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
...(page.type === 'document' ? [{ page, depth }] : []),
|
||||
...page.pages.flatMap((child) =>
|
||||
child.type === 'document' ? flattenPage(child, depth + 1) : []
|
||||
),
|
||||
];
|
||||
const children: FlatPageEntry[] = [];
|
||||
for (const child of page.pages) {
|
||||
if (child.type === 'link' || child.type === 'computed') {
|
||||
continue;
|
||||
}
|
||||
|
||||
children.push(...flattenPage(child, depth + 1, [...ancestors, page]));
|
||||
}
|
||||
|
||||
return [...(page.type === 'document' ? [{ page, depth }] : []), ...children];
|
||||
};
|
||||
|
||||
return rootPages.flatMap((page) =>
|
||||
page.type === 'group' || page.type === 'document' ? flattenPage(page, 0) : []
|
||||
page.type === 'group' || page.type === 'document' ? flattenPage(page, 0, []) : []
|
||||
);
|
||||
}
|
||||
|
||||
@@ -37,5 +45,5 @@ function flattenPages(
|
||||
* Get all indexable pages from a revision in a flat list.
|
||||
*/
|
||||
export function getIndexablePages(rootPages: RevisionPage[]) {
|
||||
return flattenPages(rootPages, (page) => !page.hidden && isPageIndexable([], page));
|
||||
return flattenPages(rootPages, (page, ancestors) => isPageIndexable(ancestors, page));
|
||||
}
|
||||
|
||||
@@ -1,444 +0,0 @@
|
||||
import { describe, expect, it, mock } from 'bun:test';
|
||||
import type { GitBookSiteContext } from '@/lib/context';
|
||||
import type { SiteSpace } from '@gitbook/api';
|
||||
|
||||
import { streamMarkdownFromSiteSpaces } from './llms-full';
|
||||
|
||||
function createMockLinker(args?: { spaceBasePath?: string }) {
|
||||
return {
|
||||
toAbsoluteURL: mock((path: string) => `https://example.com${path}`),
|
||||
toPathInSite: mock((path: string) => `/site/${args?.spaceBasePath ?? ''}${path}`),
|
||||
fork: (args: { spaceBasePath: string }) => createMockLinker(args),
|
||||
};
|
||||
}
|
||||
|
||||
describe('streamMarkdownFromSiteSpaces', () => {
|
||||
// Test with real mocks of the dependencies
|
||||
it('processes pages correctly with pagination', async () => {
|
||||
// Mock the dependencies by replacing them in the module
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
id: 'revision-1',
|
||||
pages: [
|
||||
{
|
||||
id: 'page-1',
|
||||
type: 'document',
|
||||
title: 'Page 1',
|
||||
path: 'page-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-2',
|
||||
type: 'document',
|
||||
title: 'Page 2',
|
||||
path: 'page-2',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-3',
|
||||
type: 'document',
|
||||
title: 'Page 3',
|
||||
path: 'page-3',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-4',
|
||||
type: 'document',
|
||||
title: 'Page 4',
|
||||
path: 'page-4',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-5',
|
||||
type: 'document',
|
||||
title: 'Page 5',
|
||||
path: 'page-5',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() =>
|
||||
Promise.resolve({
|
||||
data: '# Test Page\n\nSome content\n',
|
||||
})
|
||||
),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: createMockLinker(),
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
id: 'space-1',
|
||||
space: {
|
||||
id: 'space-1',
|
||||
revision: 'rev-1',
|
||||
},
|
||||
urls: {
|
||||
published: 'https://example.com',
|
||||
},
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
// Capture stream output
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Verify results
|
||||
expect(result.currentPageIndex).toBe(5); // Should process 5 pages
|
||||
expect(result.reachedLimit).toBe(false); // Under limit
|
||||
expect(chunks.length).toBe(5); // Should have 5 markdown chunks
|
||||
expect(mockDataFetcher.getRevision).toHaveBeenCalledTimes(1);
|
||||
expect(mockDataFetcher.getRevisionPageMarkdown).toHaveBeenCalledTimes(5);
|
||||
});
|
||||
|
||||
it('applies offset correctly', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: Array.from({ length: 10 }, (_, i) => ({
|
||||
id: `page-${i + 1}`,
|
||||
type: 'document',
|
||||
title: `Page ${i + 1}`,
|
||||
path: `page-${i + 1}`,
|
||||
pages: [],
|
||||
hidden: false,
|
||||
})),
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: createMockLinker(),
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example.com' },
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
3, // offset = 3
|
||||
0
|
||||
);
|
||||
|
||||
// Should process pages from index 3 onwards (7 pages)
|
||||
expect(result.currentPageIndex).toBe(10);
|
||||
expect(chunks.length).toBe(7); // 10 total - 3 offset = 7 processed
|
||||
});
|
||||
|
||||
it('handles pagination when there are more than 100 pages', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: Array.from({ length: 150 }, (_, i) => ({
|
||||
id: `page-${i + 1}`,
|
||||
type: 'document',
|
||||
title: `Page ${i + 1}`,
|
||||
path: `page-${i + 1}`,
|
||||
pages: [],
|
||||
hidden: false,
|
||||
})),
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: createMockLinker(),
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example.com' },
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should only process 100 pages (default limit)
|
||||
expect(result.currentPageIndex).toBe(100);
|
||||
expect(result.reachedLimit).toBe(true);
|
||||
expect(chunks.length).toBe(101); // 100 pages + 1 next page link
|
||||
|
||||
// Check that next page link is included
|
||||
const fullContent = chunks.join('');
|
||||
expect(fullContent).toContain('[Next Page]');
|
||||
expect(fullContent).toContain('/site/llms-full.txt/1');
|
||||
});
|
||||
|
||||
it('handles multiple site spaces', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock()
|
||||
.mockReturnValueOnce(
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
id: 'page-1',
|
||||
type: 'document',
|
||||
title: 'Space 1 Page 1',
|
||||
path: 'page-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-2',
|
||||
type: 'document',
|
||||
title: 'Space 1 Page 2',
|
||||
path: 'page-2',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
)
|
||||
.mockReturnValueOnce(
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
id: 'page-3',
|
||||
type: 'document',
|
||||
title: 'Space 2 Page 1',
|
||||
path: 'page-3',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-4',
|
||||
type: 'document',
|
||||
title: 'Space 2 Page 2',
|
||||
path: 'page-4',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'page-5',
|
||||
type: 'document',
|
||||
title: 'Space 2 Page 3',
|
||||
path: 'page-5',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: createMockLinker(),
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpaces: SiteSpace[] = [
|
||||
{
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example1.com' },
|
||||
path: 'space-1',
|
||||
},
|
||||
{
|
||||
space: { id: 'space-2', revision: 'rev-2' },
|
||||
urls: { published: 'https://example2.com' },
|
||||
path: 'space-2',
|
||||
},
|
||||
] as SiteSpace[];
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const { streamMarkdownFromSiteSpaces } = await import('./llms-full');
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
mockSiteSpaces,
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should process all pages from both spaces (2 + 3 = 5)
|
||||
expect(result.currentPageIndex).toBe(5);
|
||||
expect(result.reachedLimit).toBe(false);
|
||||
expect(chunks.length).toBe(5);
|
||||
expect(mockDataFetcher.getRevision).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('skips site spaces without published URLs', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(),
|
||||
getRevisionPageMarkdown: mock(),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: createMockLinker(),
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: undefined }, // No published URL
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const mockController = {
|
||||
enqueue: mock(),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should not process any pages
|
||||
expect(result.currentPageIndex).toBe(0);
|
||||
expect(result.reachedLimit).toBe(false);
|
||||
expect(mockDataFetcher.getRevision).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('filters only document type pages', async () => {
|
||||
const mockDataFetcher = {
|
||||
getRevision: mock(() =>
|
||||
Promise.resolve({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
id: 'doc-1',
|
||||
type: 'document',
|
||||
title: 'Document 1',
|
||||
path: 'doc-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'group-1',
|
||||
type: 'group',
|
||||
title: 'Group 1',
|
||||
path: 'group-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'doc-2',
|
||||
type: 'document',
|
||||
title: 'Document 2',
|
||||
path: 'doc-2',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
{
|
||||
id: 'link-1',
|
||||
type: 'link',
|
||||
title: 'Link 1',
|
||||
path: 'link-1',
|
||||
pages: [],
|
||||
hidden: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
),
|
||||
getRevisionPageMarkdown: mock(() => Promise.resolve({ data: 'content\n' })),
|
||||
};
|
||||
|
||||
const mockContext: GitBookSiteContext = {
|
||||
dataFetcher: mockDataFetcher,
|
||||
linker: createMockLinker(),
|
||||
} as unknown as GitBookSiteContext;
|
||||
|
||||
const mockSiteSpace: SiteSpace = {
|
||||
space: { id: 'space-1', revision: 'rev-1' },
|
||||
urls: { published: 'https://example.com' },
|
||||
path: 'test-space',
|
||||
} as SiteSpace;
|
||||
|
||||
const chunks: string[] = [];
|
||||
const mockController = {
|
||||
enqueue: mock((chunk: Uint8Array) => {
|
||||
chunks.push(new TextDecoder().decode(chunk));
|
||||
}),
|
||||
} as unknown as ReadableStreamDefaultController<Uint8Array>;
|
||||
|
||||
const result = await streamMarkdownFromSiteSpaces(
|
||||
mockContext,
|
||||
mockController,
|
||||
[mockSiteSpace],
|
||||
'base-path',
|
||||
0,
|
||||
0
|
||||
);
|
||||
|
||||
// Should only process the 2 document pages
|
||||
expect(result.currentPageIndex).toBe(2);
|
||||
expect(chunks.length).toBe(2);
|
||||
expect(mockDataFetcher.getRevisionPageMarkdown).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,10 @@
|
||||
import { type GitBookSiteContext, checkIsRootSiteContext } from '@/lib/context';
|
||||
import {
|
||||
type GitBookSiteContext,
|
||||
checkIsRootSiteContext,
|
||||
fetchSiteContextForSiteSpace,
|
||||
} from '@/lib/context';
|
||||
import { throwIfDataError } from '@/lib/data';
|
||||
import { fromPageMarkdown, toPageMarkdown } from '@/lib/markdownPage';
|
||||
import { joinPath } from '@/lib/paths';
|
||||
import { getIndexablePages } from '@/lib/sitemap';
|
||||
import { filterSiteSpacesByLocale, getSiteStructureSections } from '@/lib/sites';
|
||||
import type { RevisionPageDocument, SiteSection, SiteSpace } from '@gitbook/api';
|
||||
@@ -63,7 +66,6 @@ async function streamMarkdownFromSiteStructure(
|
||||
context,
|
||||
stream,
|
||||
context.structure.structure,
|
||||
'',
|
||||
offset
|
||||
);
|
||||
return;
|
||||
@@ -88,7 +90,6 @@ async function streamMarkdownFromSections(
|
||||
context,
|
||||
stream,
|
||||
siteSection.siteSpaces,
|
||||
siteSection.path,
|
||||
offset,
|
||||
currentPageIndex
|
||||
);
|
||||
@@ -107,16 +108,13 @@ export async function streamMarkdownFromSiteSpaces(
|
||||
context: GitBookSiteContext,
|
||||
stream: ReadableStreamDefaultController<Uint8Array>,
|
||||
siteSpaces: SiteSpace[],
|
||||
basePath: string,
|
||||
offset = 0,
|
||||
initialPageIndex = 0
|
||||
): Promise<{ currentPageIndex: number; reachedLimit: boolean }> {
|
||||
const { dataFetcher } = context;
|
||||
let totalPagesProcessed = initialPageIndex;
|
||||
|
||||
// Collect all pages first
|
||||
const allPages: Array<{ page: RevisionPageDocument; siteSpace: SiteSpace; basePath: string }> =
|
||||
[];
|
||||
const allPages: Array<{ context: GitBookSiteContext; page: RevisionPageDocument }> = [];
|
||||
|
||||
const filteredSiteSpaces = filterSiteSpacesByLocale(siteSpaces, context.locale);
|
||||
|
||||
@@ -125,21 +123,15 @@ export async function streamMarkdownFromSiteSpaces(
|
||||
if (!siteSpaceUrl) {
|
||||
continue;
|
||||
}
|
||||
const revision = await throwIfDataError(
|
||||
dataFetcher.getRevision({
|
||||
spaceId: siteSpace.space.id,
|
||||
revisionId: siteSpace.space.revision,
|
||||
})
|
||||
);
|
||||
const pages = getIndexablePages(revision.pages);
|
||||
const siteSpaceContext = await fetchSiteContextForSiteSpace(context, siteSpace);
|
||||
const pages = getIndexablePages(siteSpaceContext.revision.pages);
|
||||
|
||||
// Add document pages to our collection
|
||||
for (const { page } of pages) {
|
||||
if (page.type === 'document') {
|
||||
allPages.push({
|
||||
context: siteSpaceContext,
|
||||
page,
|
||||
siteSpace,
|
||||
basePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -152,8 +144,8 @@ export async function streamMarkdownFromSiteSpaces(
|
||||
// Process the pages
|
||||
for await (const markdown of pMapIterable(
|
||||
pagesToProcess,
|
||||
async ({ page, siteSpace, basePath }) => {
|
||||
return getMarkdownForPage(context, siteSpace, page, basePath);
|
||||
async ({ context: siteSpaceContext, page }) => {
|
||||
return getMarkdownForPage(siteSpaceContext, page);
|
||||
},
|
||||
{
|
||||
concurrency: MAX_CONCURRENCY,
|
||||
@@ -180,32 +172,22 @@ export async function streamMarkdownFromSiteSpaces(
|
||||
*/
|
||||
async function getMarkdownForPage(
|
||||
context: GitBookSiteContext,
|
||||
siteSpace: SiteSpace,
|
||||
page: RevisionPageDocument,
|
||||
basePath: string
|
||||
page: RevisionPageDocument
|
||||
): Promise<string> {
|
||||
const { dataFetcher } = context;
|
||||
|
||||
const pageMarkdown = await throwIfDataError(
|
||||
dataFetcher.getRevisionPageMarkdown({
|
||||
spaceId: siteSpace.space.id,
|
||||
revisionId: siteSpace.space.revision,
|
||||
spaceId: context.space.id,
|
||||
revisionId: context.revisionId,
|
||||
pageId: page.id,
|
||||
})
|
||||
);
|
||||
|
||||
const tree = await fromPageMarkdown(
|
||||
{
|
||||
...context,
|
||||
linker: context.linker.fork({
|
||||
spaceBasePath: joinPath(context.linker.siteBasePath, basePath),
|
||||
}),
|
||||
},
|
||||
{
|
||||
markdown: pageMarkdown,
|
||||
pagePath: page.path,
|
||||
}
|
||||
);
|
||||
const tree = await fromPageMarkdown(context, {
|
||||
markdown: pageMarkdown,
|
||||
pagePath: page.path,
|
||||
});
|
||||
|
||||
if (page.description) {
|
||||
// The first node is the page title as a H1, we insert the description as a paragraph
|
||||
|
||||
@@ -3,7 +3,12 @@ import { throwIfDataError } from '@/lib/data';
|
||||
import { type GitBookLinker, linkerWithMarkdownPages } from '@/lib/links';
|
||||
import { resolveFirstDocument } from '@/lib/pages';
|
||||
import { type FlatPageEntry, getIndexablePages } from '@/lib/sitemap';
|
||||
import { filterSiteSpacesByLocale, getLocalizedTitle, getSiteStructureSections } from '@/lib/sites';
|
||||
import {
|
||||
filterSiteSpacesByLocale,
|
||||
getFallbackSiteSpacePath,
|
||||
getLocalizedTitle,
|
||||
getSiteStructureSections,
|
||||
} from '@/lib/sites';
|
||||
import type { SiteSection, SiteSpace } from '@gitbook/api';
|
||||
import assertNever from 'assert-never';
|
||||
import type { ListItem, Paragraph, Root, RootContent } from 'mdast';
|
||||
@@ -143,8 +148,14 @@ async function getNodesFromSiteSpaces(
|
||||
});
|
||||
}
|
||||
|
||||
const siteSpaceLinker = linkerWithMarkdownPages(
|
||||
linker.withOtherSiteSpace({
|
||||
spaceBasePath: getFallbackSiteSpacePath(context, siteSpace),
|
||||
})
|
||||
);
|
||||
|
||||
// Add the pages as a list
|
||||
nodes.push(...(await getMarkdownForPagesTree(pages, linker)));
|
||||
nodes.push(...(await getMarkdownForPagesTree(pages, siteSpaceLinker)));
|
||||
|
||||
return nodes;
|
||||
})
|
||||
|
||||
@@ -12,6 +12,18 @@ describe('llms.txt', () => {
|
||||
expect(await response.text()).toContain('# E2E Tests GitBook Open');
|
||||
});
|
||||
|
||||
it('should properly format links', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook-open-e2e-sites.gitbook.io/sections/llms.txt')
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
const content = await response.text();
|
||||
expect(content).toContain('/sections/sections-3/readme.md');
|
||||
expect(content).toContain('/sections/sections-4/getting-started/quickstart.md');
|
||||
});
|
||||
|
||||
it('should expose a llms.txt file with the accept header', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms.txt'),
|
||||
@@ -49,28 +61,59 @@ describe('llms.txt', () => {
|
||||
});
|
||||
|
||||
describe('llms-full.txt', () => {
|
||||
it('should expose a llms-full.txt file', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms-full.txt')
|
||||
);
|
||||
it(
|
||||
'should expose a llms-full.txt file',
|
||||
async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms-full.txt')
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# Welcome');
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# Welcome');
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
);
|
||||
|
||||
it('should expose a llms-full.txt file with the accept header', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms-full.txt'),
|
||||
{
|
||||
headers: {
|
||||
Accept: 'text/markdown',
|
||||
},
|
||||
}
|
||||
);
|
||||
it(
|
||||
'should expose cross-space pages from a multi-version site',
|
||||
async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL(
|
||||
'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/llms-full.txt'
|
||||
)
|
||||
);
|
||||
const text = await response.text();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# Welcome');
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(text).toContain(
|
||||
'gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/2.0/quick-start'
|
||||
);
|
||||
expect(text).toContain(
|
||||
'gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/3.0/other-page'
|
||||
);
|
||||
expect(text).not.toContain('broken://');
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
);
|
||||
|
||||
it(
|
||||
'should expose a llms-full.txt file with the accept header',
|
||||
async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms-full.txt'),
|
||||
{
|
||||
headers: {
|
||||
Accept: 'text/markdown',
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(await response.text()).toContain('# Welcome');
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
);
|
||||
});
|
||||
|
||||
@@ -42,6 +42,18 @@ describe('markdown pages', () => {
|
||||
expect(response.headers.get('x-robots-tag')).toBe('noindex');
|
||||
expect(text).toContain('# Page Not Found');
|
||||
});
|
||||
|
||||
it('should rewrite links to markdown URLs', async () => {
|
||||
const response = await fetch(
|
||||
getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/blocks/links.md')
|
||||
);
|
||||
const text = await response.text();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('content-type')).toContain('text/markdown');
|
||||
expect(response.headers.get('x-robots-tag')).toBe('noindex');
|
||||
expect(text).toContain('gitbook.gitbook.io/test-gitbook-open/text-page.md');
|
||||
});
|
||||
});
|
||||
|
||||
describe('markdown ask responses', () => {
|
||||
|
||||
@@ -53,3 +53,34 @@ it(
|
||||
},
|
||||
{ timeout: 10_000 }
|
||||
);
|
||||
|
||||
it(
|
||||
'should get a page from another site space through MCP',
|
||||
async () => {
|
||||
const client = new Client({
|
||||
name: 'test',
|
||||
version: '1.0.0',
|
||||
});
|
||||
|
||||
await client.connect(
|
||||
new StreamableHTTPClientTransport(
|
||||
new URL(
|
||||
getContentTestURL(
|
||||
'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/~gitbook/mcp/auth'
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
const response = await client.callTool({
|
||||
name: 'getPage',
|
||||
arguments: {
|
||||
url: 'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/3.0/other-page',
|
||||
},
|
||||
});
|
||||
|
||||
// @ts-expect-error - response.content is of type unknown
|
||||
expect(response.content[0]?.text).toContain('# Other Page');
|
||||
},
|
||||
{ timeout: 15_000 }
|
||||
);
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { afterEach, describe, expect, it } from 'bun:test';
|
||||
import { renderToStaticMarkup } from 'react-dom/server';
|
||||
|
||||
import { Icon } from './Icon';
|
||||
import { IconsProvider } from './IconsProvider';
|
||||
import { clearRegisteredServerIconSymbols, getRegisteredServerIconSymbols } from './symbols';
|
||||
import { IconStyle } from './types';
|
||||
|
||||
afterEach(() => {
|
||||
clearRegisteredServerIconSymbols();
|
||||
});
|
||||
|
||||
function RegisteredSymbolsSummary() {
|
||||
const symbols = getRegisteredServerIconSymbols();
|
||||
return (
|
||||
<output
|
||||
data-testid="registered-symbols"
|
||||
data-count={symbols.length}
|
||||
data-symbol-ids={symbols.map((symbol) => symbol.symbolId).join(',')}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
describe('Icon', () => {
|
||||
it('renders the mask-image path when symbol mode is disabled', () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<IconsProvider assetsURL="https://icons.example.test" iconStyle={IconStyle.Regular}>
|
||||
<Icon icon="github" className="size-4" />
|
||||
</IconsProvider>
|
||||
);
|
||||
|
||||
expect(html).toContain('data-testid="mask-image"');
|
||||
expect(html).toContain('https://icons.example.test/svgs/brands/github.svg?v=2');
|
||||
expect(html).not.toContain('data-testid="symbol-use"');
|
||||
});
|
||||
|
||||
it('renders inline symbol references and deduplicates SSR registrations', () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<IconsProvider
|
||||
assetsURL="https://icons.example.test"
|
||||
iconStyle={IconStyle.Regular}
|
||||
renderMode="symbol"
|
||||
symbolLoaderURL="/~gitbook/icons/symbol"
|
||||
>
|
||||
<>
|
||||
<Icon icon="github" />
|
||||
<Icon icon="github" />
|
||||
<Icon icon="gitbook" />
|
||||
<RegisteredSymbolsSummary />
|
||||
</>
|
||||
</IconsProvider>
|
||||
);
|
||||
|
||||
expect(html).toContain('data-testid="symbol-use"');
|
||||
expect(html).toContain('href="#gb-icon-brands-github"');
|
||||
expect(html).toContain('href="#gb-icon-custom-icons-gitbook"');
|
||||
expect(html).toContain('data-testid="registered-symbols"');
|
||||
expect(html).toContain('data-count="2"');
|
||||
expect(html).toContain(
|
||||
'data-symbol-ids="gb-icon-brands-github,gb-icon-custom-icons-gitbook"'
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { getIconAssetURL, useIcons } from './IconsProvider';
|
||||
import { IconSymbolLoader } from './IconSymbolLoader';
|
||||
import { getIconAssetURL, getIconSpriteAssetURL, useIcons } from './IconsProvider';
|
||||
import { getIconStyle } from './getIconStyle';
|
||||
import { getIconSymbolId, prefetchServerIconAsset, registerServerIconSymbol } from './symbols';
|
||||
import type { IconName, IconStyle } from './types';
|
||||
|
||||
/**
|
||||
@@ -50,8 +52,47 @@ export const Icon = React.forwardRef(function Icon(
|
||||
} = props;
|
||||
|
||||
const [iconStyle, icon] = getIconStyle(propIconStyle, propIcon);
|
||||
const url = getIconAssetURL(context, iconStyle, icon);
|
||||
const maskId = React.useId();
|
||||
const iconInstanceId = React.useId();
|
||||
const symbolId = getIconSymbolId(iconStyle, icon);
|
||||
const iconAssetURL = getIconAssetURL(context, iconStyle, icon);
|
||||
const iconSpriteAssetURL = getIconSpriteAssetURL(context, iconStyle);
|
||||
|
||||
if (context.renderMode === 'symbol') {
|
||||
prefetchServerIconAsset(iconSpriteAssetURL);
|
||||
registerServerIconSymbol({
|
||||
style: iconStyle,
|
||||
icon,
|
||||
symbolId,
|
||||
});
|
||||
|
||||
return (
|
||||
<svg
|
||||
ref={ref}
|
||||
{...rest}
|
||||
viewBox="0 0 512 512"
|
||||
data-gb-icon-instance={iconInstanceId}
|
||||
style={{
|
||||
overflow: 'visible',
|
||||
...(size ? { width: size, height: size } : {}),
|
||||
...rest.style,
|
||||
}}
|
||||
className={`gb-icon ${className}`}
|
||||
>
|
||||
<title>{icon}</title>
|
||||
<use data-testid="symbol-use" href={`#${symbolId}`} width="100%" height="100%" />
|
||||
{context.symbolLoaderURL ? (
|
||||
<IconSymbolLoader
|
||||
instanceId={iconInstanceId}
|
||||
symbolId={symbolId}
|
||||
style={iconStyle}
|
||||
icon={icon}
|
||||
loaderURL={context.symbolLoaderURL}
|
||||
/>
|
||||
) : null}
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<svg
|
||||
@@ -73,7 +114,7 @@ export const Icon = React.forwardRef(function Icon(
|
||||
>
|
||||
<image
|
||||
data-testid="mask-image"
|
||||
href={url}
|
||||
href={iconAssetURL}
|
||||
width="100%"
|
||||
height="100%"
|
||||
preserveAspectRatio="xMidYMid meet"
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
const SVG_NAMESPACE = 'http://www.w3.org/2000/svg';
|
||||
const pendingSymbolLoads = new Map<string, Promise<boolean>>();
|
||||
|
||||
/**
|
||||
* Ensure a symbol referenced by an inline `<use>` exists after hydration, fetching it from the
|
||||
* same-origin symbol route only when the SSR sprite did not already include it.
|
||||
*/
|
||||
export function IconSymbolLoader(props: {
|
||||
instanceId: string;
|
||||
symbolId: string;
|
||||
style: string;
|
||||
icon: string;
|
||||
loaderURL: string;
|
||||
}) {
|
||||
const { instanceId, symbolId, style, icon, loaderURL } = props;
|
||||
|
||||
React.useEffect(() => {
|
||||
let mounted = true;
|
||||
|
||||
if (hasSymbol(symbolId)) {
|
||||
setIconState(instanceId, true);
|
||||
return;
|
||||
}
|
||||
|
||||
loadSymbol(symbolId, loaderURL, style, icon).then((loaded) => {
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
|
||||
setIconState(instanceId, loaded);
|
||||
});
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
};
|
||||
}, [icon, instanceId, loaderURL, style, symbolId]);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A root SVG element used to host the loaded symbols as a spritesheet. This is used as a reference point for the `<use>` elements in the icons.
|
||||
*/
|
||||
function getSpriteRoot(): SVGSVGElement {
|
||||
const existing = document.getElementById('gb-icon-sprite-root');
|
||||
if (existing instanceof SVGSVGElement) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
return createSpriteRoot();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the root SVG element for the spritesheet if it doesn't already exist, and appends it to the document body.
|
||||
*/
|
||||
function createSpriteRoot(): SVGSVGElement {
|
||||
const spriteRoot = document.createElementNS(SVG_NAMESPACE, 'svg');
|
||||
spriteRoot.setAttribute('id', 'gb-icon-sprite-root');
|
||||
spriteRoot.setAttribute('aria-hidden', 'true');
|
||||
spriteRoot.setAttribute('focusable', 'false');
|
||||
spriteRoot.setAttribute(
|
||||
'style',
|
||||
'position:absolute;width:0;height:0;overflow:hidden;pointer-events:none'
|
||||
);
|
||||
document.body.prepend(spriteRoot);
|
||||
|
||||
return spriteRoot;
|
||||
}
|
||||
|
||||
function hasSymbol(symbolId: string): boolean {
|
||||
return document.getElementById(symbolId) instanceof SVGElement;
|
||||
}
|
||||
|
||||
function buildSymbolURL(loaderURL: string, style: string, icon: string): string {
|
||||
const normalizedLoaderURL = loaderURL.endsWith('/') ? loaderURL.slice(0, -1) : loaderURL;
|
||||
|
||||
return `${normalizedLoaderURL}/${encodeURIComponent(style)}/${encodeURIComponent(icon)}`;
|
||||
}
|
||||
|
||||
function appendSymbolsFromDocument(markup: string): boolean {
|
||||
const parsed = new DOMParser().parseFromString(markup, 'image/svg+xml');
|
||||
const symbols = Array.from(parsed.querySelectorAll('symbol'));
|
||||
if (symbols.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const spriteRoot = getSpriteRoot();
|
||||
for (const symbol of symbols) {
|
||||
const symbolId = symbol.getAttribute('id');
|
||||
if (!symbolId || hasSymbol(symbolId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
spriteRoot.appendChild(document.importNode(symbol, true));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function loadSymbol(symbolId: string, loaderURL: string, style: string, icon: string) {
|
||||
if (hasSymbol(symbolId)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const cached = pendingSymbolLoads.get(symbolId);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const request = fetch(buildSymbolURL(loaderURL, style, icon), {
|
||||
credentials: 'same-origin',
|
||||
})
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const symbolMarkup = await response.text();
|
||||
if (hasSymbol(symbolId)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!appendSymbolsFromDocument(symbolMarkup)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return hasSymbol(symbolId);
|
||||
})
|
||||
.catch(() => false)
|
||||
.finally(() => {
|
||||
pendingSymbolLoads.delete(symbolId);
|
||||
});
|
||||
|
||||
pendingSymbolLoads.set(symbolId, request);
|
||||
return request;
|
||||
}
|
||||
|
||||
function setIconState(instanceId: string, loaded: boolean) {
|
||||
const icon = document.querySelector<SVGSVGElement>(
|
||||
`svg[data-gb-icon-instance="${instanceId}"]`
|
||||
);
|
||||
if (!icon) {
|
||||
return;
|
||||
}
|
||||
|
||||
const symbolUse = icon.querySelector<SVGUseElement>('[data-testid="symbol-use"]');
|
||||
if (symbolUse) {
|
||||
symbolUse.style.display = loaded ? '' : 'none';
|
||||
}
|
||||
icon.setAttribute('data-gb-icon-symbol-state', loaded ? 'loaded' : 'failed');
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import type { IconRenderMode } from './symbols';
|
||||
import { IconStyle } from './types';
|
||||
|
||||
const version = 2;
|
||||
@@ -17,10 +18,15 @@ export type IconsContextType = Partial<IconsAssetsLocation> & {
|
||||
assetsByStyles?: Record<string, IconsAssetsLocation>;
|
||||
/** Current default style for icons */
|
||||
iconStyle: IconStyle;
|
||||
/** Rendering strategy for icons */
|
||||
renderMode: IconRenderMode;
|
||||
/** Base URL used to lazily load prebuilt symbol documents introduced after hydration */
|
||||
symbolLoaderURL?: string;
|
||||
};
|
||||
|
||||
const IconsContext = React.createContext<IconsContextType>({
|
||||
iconStyle: IconStyle.Regular,
|
||||
renderMode: 'mask',
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -34,10 +40,17 @@ export function IconsProvider(props: React.PropsWithChildren<Partial<IconsContex
|
||||
assetsURLToken = parent.assetsURLToken,
|
||||
iconStyle = parent.iconStyle,
|
||||
assetsByStyles = parent.assetsByStyles,
|
||||
renderMode = parent.renderMode,
|
||||
symbolLoaderURL = parent.symbolLoaderURL,
|
||||
} = props;
|
||||
const value = React.useMemo(() => {
|
||||
return { assetsURL, assetsURLToken, iconStyle, assetsByStyles };
|
||||
}, [assetsURL, assetsURLToken, iconStyle, assetsByStyles]);
|
||||
const value = {
|
||||
assetsURL,
|
||||
assetsURLToken,
|
||||
iconStyle,
|
||||
assetsByStyles,
|
||||
renderMode,
|
||||
symbolLoaderURL,
|
||||
};
|
||||
|
||||
return <IconsContext.Provider value={value}>{children}</IconsContext.Provider>;
|
||||
}
|
||||
@@ -75,3 +88,11 @@ export function getIconAssetURL(context: IconsContextType, style: string, icon:
|
||||
const iconName = typeof icon === 'string' ? icon : String(icon);
|
||||
return getAssetURL(location, `svgs/${style}/${iconName}.svg`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the URL for the sprite document of an icon style.
|
||||
*/
|
||||
export function getIconSpriteAssetURL(context: IconsContextType, style: string): string {
|
||||
const location = context.assetsByStyles?.[style] ?? context;
|
||||
return getAssetURL(location, `sprites/${style}.svg`);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
export * from './Icon';
|
||||
export * from './types';
|
||||
export * from './symbols';
|
||||
export * from './getIconStyle';
|
||||
export * from './IconsProvider';
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
export type IconRenderMode = 'mask' | 'symbol';
|
||||
|
||||
export interface RegisteredIconSymbol {
|
||||
style: string;
|
||||
icon: string;
|
||||
symbolId: string;
|
||||
}
|
||||
|
||||
const REGISTERED_SYMBOLS_KEY = Symbol.for('gitbook.icons.registeredSymbols');
|
||||
const PREFETCHED_ICON_ASSETS_KEY = Symbol.for('gitbook.icons.prefetchedAssets');
|
||||
|
||||
function sanitizeSymbolFragment(fragment: string): string {
|
||||
return fragment.replace(/[^a-zA-Z0-9_-]/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the DOM id used by both SSR-emitted symbols and the lazy symbol loader.
|
||||
*/
|
||||
export function getIconSymbolId(style: string, icon: string): string {
|
||||
return `gb-icon-${sanitizeSymbolFragment(style)}-${sanitizeSymbolFragment(icon)}`;
|
||||
}
|
||||
|
||||
function getRegisteredSymbolsStore(): Map<string, RegisteredIconSymbol> {
|
||||
const store = globalThis as typeof globalThis & {
|
||||
[REGISTERED_SYMBOLS_KEY]?: Map<string, RegisteredIconSymbol>;
|
||||
};
|
||||
|
||||
if (!store[REGISTERED_SYMBOLS_KEY]) {
|
||||
store[REGISTERED_SYMBOLS_KEY] = new Map<string, RegisteredIconSymbol>();
|
||||
}
|
||||
|
||||
return store[REGISTERED_SYMBOLS_KEY];
|
||||
}
|
||||
|
||||
function getPrefetchedAssetsStore(): Map<string, Promise<void>> {
|
||||
const store = globalThis as typeof globalThis & {
|
||||
[PREFETCHED_ICON_ASSETS_KEY]?: Map<string, Promise<void>>;
|
||||
};
|
||||
|
||||
if (!store[PREFETCHED_ICON_ASSETS_KEY]) {
|
||||
store[PREFETCHED_ICON_ASSETS_KEY] = new Map<string, Promise<void>>();
|
||||
}
|
||||
|
||||
return store[PREFETCHED_ICON_ASSETS_KEY];
|
||||
}
|
||||
|
||||
function shouldTrackSymbolRegistrations() {
|
||||
const runtime = globalThis as typeof globalThis & { Bun?: unknown };
|
||||
|
||||
return typeof window === 'undefined' || typeof runtime.Bun !== 'undefined';
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a symbol used during server rendering so the app can emit a deduplicated sprite subset.
|
||||
*/
|
||||
export function registerServerIconSymbol(symbol: RegisteredIconSymbol): void {
|
||||
if (!shouldTrackSymbolRegistrations()) {
|
||||
return;
|
||||
}
|
||||
|
||||
getRegisteredSymbolsStore().set(`${symbol.style}/${symbol.icon}`, symbol);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start fetching a server-side icon asset during SSR so sprite generation can reuse the in-flight
|
||||
* or warm request instead of waiting until the end of the render.
|
||||
*/
|
||||
export function prefetchServerIconAsset(assetURL: string): void {
|
||||
if (typeof window !== 'undefined') {
|
||||
return;
|
||||
}
|
||||
|
||||
const prefetchedAssets = getPrefetchedAssetsStore();
|
||||
if (prefetchedAssets.has(assetURL)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const request = fetch(assetURL, { cache: 'force-cache' })
|
||||
.then(() => undefined)
|
||||
.catch(() => undefined);
|
||||
|
||||
prefetchedAssets.set(assetURL, request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the currently registered server-rendered symbols in insertion order.
|
||||
*/
|
||||
export function getRegisteredServerIconSymbols(): RegisteredIconSymbol[] {
|
||||
return [...getRegisteredSymbolsStore().values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the per-request symbol registry after the sprite subset has been emitted.
|
||||
*/
|
||||
export function clearRegisteredServerIconSymbols(): void {
|
||||
getRegisteredSymbolsStore().clear();
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["es2023"],
|
||||
"lib": ["es2023", "dom"],
|
||||
"module": "ESNext",
|
||||
"target": "es2022",
|
||||
"strict": true,
|
||||
|
||||
Reference in New Issue
Block a user