From 75bdff3ff3c1a882f4bb4eee1ec75091cc8fca77 Mon Sep 17 00:00:00 2001 From: conico974 Date: Thu, 20 Aug 2026 12:18:17 +0200 Subject: [PATCH] Add support for AI agent detection and robots directives in markdown (#4524) --- .changeset/markdown-agent-robots-tag.md | 5 ++ packages/gitbook/src/lib/context.ts | 13 ++++ packages/gitbook/src/middleware.ts | 12 +++- packages/gitbook/src/routes/markdownPage.ts | 40 +++++++++-- packages/gitbook/tests/markdown.test.ts | 76 +++++++++++++++++++-- 5 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 .changeset/markdown-agent-robots-tag.md diff --git a/.changeset/markdown-agent-robots-tag.md b/.changeset/markdown-agent-robots-tag.md new file mode 100644 index 000000000..1fc034bf5 --- /dev/null +++ b/.changeset/markdown-agent-robots-tag.md @@ -0,0 +1,5 @@ +--- +"gitbook": patch +--- + +Serve an indexable `X-Robots-Tag` on markdown pages requested by AI agents diff --git a/packages/gitbook/src/lib/context.ts b/packages/gitbook/src/lib/context.ts index df6affcdc..36d667fb3 100644 --- a/packages/gitbook/src/lib/context.ts +++ b/packages/gitbook/src/lib/context.ts @@ -93,6 +93,13 @@ export type SiteURLData = Pick< * Should never be set for the main site. RND-11571. */ embedTheme?: CustomizationDefaultThemeMode; + + /** + * Whether the request comes from a detected AI agent. Used to serve an indexable + * `X-Robots-Tag` on markdown pages. Only set for markdown routes, to avoid splitting + * the static cache of the other routes. + */ + isAiAgent?: boolean; }; /** @@ -193,6 +200,9 @@ export type GitBookSiteContext = GitBookSpaceContext & { /** Whether to display agent instructions in the markdown output. Defaults to true when undefined. */ displayAgentInstructions?: boolean; + + /** Whether the request comes from a detected AI agent. Only set for markdown routes. */ + isAiAgent?: boolean; }; /** @@ -275,6 +285,7 @@ export async function fetchSiteContextByURLLookup( noIndexSearch: data.noIndexSearch ?? false, isLoggedInVisitor: data.isLoggedInVisitor ?? false, displayAgentInstructions: data.displayAgentInstructions, + isAiAgent: data.isAiAgent, }); } @@ -297,6 +308,7 @@ export async function fetchSiteContextByIds( noIndexSearch: boolean; isLoggedInVisitor: boolean; displayAgentInstructions?: boolean; + isAiAgent?: boolean; } ): Promise { const { dataFetcher } = baseContext; @@ -425,6 +437,7 @@ export async function fetchSiteContextByIds( noIndexSearch: ids.noIndexSearch, isLoggedInVisitor: ids.isLoggedInVisitor, displayAgentInstructions: ids.displayAgentInstructions, + isAiAgent: ids.isAiAgent, }; } diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index 40d595796..0c6b8b4d6 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -459,8 +459,11 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) { pathname, routeType: routeTypeFromPathname, events, + isAiAgent, } = encodePathInSiteContent(siteURLData, request); routeType = routeTypeFromPathname ?? routeType; + // Only set for markdown routes, so it becomes part of their static cache key. + stableSiteURLData.isAiAgent = isAiAgent; // Apply a forced theme (`?theme=`/cookie). For the docs embed we thread it through the // route context (`embedTheme`) so those routes stay statically rendered — it becomes part @@ -772,6 +775,8 @@ function encodePathInSiteContent( pathname: string; routeType?: 'static' | 'dynamic'; events?: ServerInsightsEventInput[] | undefined; + /** Only set for markdown routes, where the output depends on the visitor being an agent. */ + isAiAgent?: boolean; } { let pathname = removeLeadingSlash(removeTrailingSlash(siteURLData.pathname)); @@ -884,9 +889,8 @@ function encodePathInSiteContent( const aiAgentDetection = isAIAgent(request); // Using heuristic detection incorrectly detects some legitimate bot requests as AI agents (e.g. Slackbot) // We don't want to serve markdown for these requests as it can cause issues like breaking slack unfurling. - const shouldServeMarkdown = - (aiAgentDetection.detected && aiAgentDetection.method !== 'heuristic') || - acceptsMarkdown(request); + const isAiAgent = aiAgentDetection.detected && aiAgentDetection.method !== 'heuristic'; + const shouldServeMarkdown = isAiAgent || acceptsMarkdown(request); if (pathname.match(MARKDOWN_PATH_REGEX) || shouldServeMarkdown) { const pagePathWithoutMD = pathname.replace(MARKDOWN_PATH_REGEX, ''); const searchParams = new URL(request.url).searchParams; @@ -903,6 +907,8 @@ function encodePathInSiteContent( }` : `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`, routeType: 'static', + // Left undefined for non-agents to avoid splitting the static cache for them. + isAiAgent: isAiAgent || undefined, // TODO: track pageId / spaceId when possible // We don't do it at the moment as we can't easily extract it from the URL. events: ask diff --git a/packages/gitbook/src/routes/markdownPage.ts b/packages/gitbook/src/routes/markdownPage.ts index bd4e59522..76ac13975 100644 --- a/packages/gitbook/src/routes/markdownPage.ts +++ b/packages/gitbook/src/routes/markdownPage.ts @@ -7,6 +7,7 @@ import { linkerWithMarkdownPages } from '@/lib/links'; import { renderLLMsTxtMarkdownDirective } from '@/lib/llms-directive'; import { getMarkdownForPage } from '@/lib/markdownPage'; import { type ResolvedPagePath, getSimilarPages } from '@/lib/pages'; +import { isPageIndexable, isSiteIndexable } from '@/lib/seo'; import { resolveSiteSpacePagePathDocumentOrGroup } from '@/lib/sites'; /** @@ -27,17 +28,40 @@ export async function servePageMarkdown(baseContext: GitBookSiteContext, pagePat ); if (!pageLookup) { // Generates a markdown body for missing pages. Return this with a 200 status (not 404) because agents discard 404 response bodies.= - return renderNotFoundMarkdown(context, pagePath); + return { + markdown: renderNotFoundMarkdown(context, pagePath), + robots: 'noindex, nofollow', + }; } + const robots = getMarkdownRobots(context, pageLookup); + const markdownPage = await getMarkdownForPage(context, pageLookup); if (baseContext.displayAgentInstructions === false) { - return markdownPage; + return { markdown: markdownPage, robots }; } - return `${renderLLMsTxtMarkdownDirective(context, pageLookup.page)}\n\n${markdownPage}${renderAskFooter(context, pageLookup)}`; + return { + markdown: `${renderLLMsTxtMarkdownDirective(context, pageLookup.page)}\n\n${markdownPage}${renderAskFooter(context, pageLookup)}`, + robots, + }; }); } +/** + * Robots directive for a markdown page: the markdown version is only indexable for AI agents, + * and only when the page itself is indexable. + */ +function getMarkdownRobots( + context: GitBookSiteContext, + pageLookup: ResolvedPagePath +) { + if (!isSiteIndexable(context) || !isPageIndexable(pageLookup.ancestors, pageLookup.page)) { + return 'noindex, nofollow'; + } + + return context.isAiAgent ? 'index, follow' : 'noindex'; +} + function renderNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) { const similarPages = getSimilarPages(context.revision.pages, pagePath, 5); const sitemapUrl = context.linker.toAbsoluteURL(context.linker.toPathInSite('sitemap.md')); @@ -138,13 +162,17 @@ Use this mechanism when the answer is not explicitly present in the current page /** * Return a markdown content. */ -export async function serveMarkdown(fn: () => Promise) { +export async function serveMarkdown( + fn: () => Promise +) { try { - const markdown = await fn(); + const result = await fn(); + const { markdown, robots } = + typeof result === 'string' ? { markdown: result, robots: 'noindex' } : result; return new Response(markdown, { headers: { 'Content-Type': 'text/markdown; charset=utf-8', - 'X-Robots-Tag': 'noindex', + 'X-Robots-Tag': robots, Vary: 'Accept', }, }); diff --git a/packages/gitbook/tests/markdown.test.ts b/packages/gitbook/tests/markdown.test.ts index d918e9e38..20740d9a1 100644 --- a/packages/gitbook/tests/markdown.test.ts +++ b/packages/gitbook/tests/markdown.test.ts @@ -85,7 +85,9 @@ describe('search parameters for indexing crawlers', () => { describe('markdown pages', () => { it('should expose a markdown page with the .md extension', async () => { const response = await fetch( - getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/text-page.md') + getContentTestURL( + 'https://gitbook.gitbook.io/test-gitbook-open/text-page.md?x-gitbook-search-indexation=1' + ) ); const text = await response.text(); @@ -97,7 +99,9 @@ describe('markdown pages', () => { it('should expose a markdown page with the accept header', async () => { const response = await fetch( - getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/text-page'), + getContentTestURL( + 'https://gitbook.gitbook.io/test-gitbook-open/text-page?x-gitbook-search-indexation=1' + ), { headers: { Accept: 'text/markdown', @@ -120,13 +124,15 @@ describe('markdown pages', () => { expect(response.status).toBe(200); expect(response.headers.get('content-type')).toContain('text/markdown'); - expect(response.headers.get('x-robots-tag')).toBe('noindex'); + expect(response.headers.get('x-robots-tag')).toBe('noindex, nofollow'); expect(text).toContain('# Page Not Found'); }); it('should rewrite links to markdown URLs', async () => { const response = await fetch( - getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/blocks/links.md') + getContentTestURL( + 'https://gitbook.gitbook.io/test-gitbook-open/blocks/links.md?x-gitbook-search-indexation=1' + ) ); const text = await response.text(); @@ -249,3 +255,65 @@ describe('markdown ask responses', () => { { timeout: 30_000 } ); }); + +describe('markdown robots directives', () => { + // Preview deployments block indexation unless this param is set. + const INDEXATION_PARAM = 'x-gitbook-search-indexation=1'; + // Share link sites are never indexable, whatever the visitor is. + const SHARE_LINK_PAGE_URL = + 'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/3.0/other-page'; + + it('should be indexable for an AI agent requesting a .md page', async () => { + const response = await fetch(getContentTestURL(`${TEST_PAGE_URL}.md?${INDEXATION_PARAM}`), { + headers: { 'User-Agent': 'ClaudeBot/1.0' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBe('index, follow'); + }); + + it('should be indexable for an AI agent served markdown on the page URL', async () => { + const response = await fetch(getContentTestURL(`${TEST_PAGE_URL}?${INDEXATION_PARAM}`), { + headers: { 'User-Agent': 'GPTBot/1.2' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBe('index, follow'); + }); + + it('should stay noindex for a non-agent requesting a .md page', async () => { + const response = await fetch(getContentTestURL(`${TEST_PAGE_URL}.md?${INDEXATION_PARAM}`), { + headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBe('noindex'); + }); + + it('should be noindex, nofollow for an AI agent on a non-indexable site', async () => { + const response = await fetch( + getContentTestURL(`${SHARE_LINK_PAGE_URL}.md?${INDEXATION_PARAM}`), + { headers: { 'User-Agent': 'ClaudeBot/1.0' } } + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBe('noindex, nofollow'); + }); + + it('should be noindex, nofollow for an AI agent on a missing page', async () => { + const response = await fetch( + getContentTestURL( + `https://gitbook.gitbook.io/test-gitbook-open/missing-page.md?${INDEXATION_PARAM}` + ), + { headers: { 'User-Agent': 'ClaudeBot/1.0' } } + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBe('noindex, nofollow'); + }); +});