Add support for AI agent detection and robots directives in markdown (#4524)

.changeset/markdown-agent-robots-tag.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Serve an indexable `X-Robots-Tag` on markdown pages requested by AI agents

packages/gitbook/src/lib/context.ts

@@ -93,6 +93,13 @@ export type SiteURLData = Pick<
      * Should never be set for the main site. RND-11571.
      */
     embedTheme?: CustomizationDefaultThemeMode;
+
+    /**
+     * Whether the request comes from a detected AI agent. Used to serve an indexable
+     * `X-Robots-Tag` on markdown pages. Only set for markdown routes, to avoid splitting
+     * the static cache of the other routes.
+     */
+    isAiAgent?: boolean;
 };
 
 /**
@@ -193,6 +200,9 @@ export type GitBookSiteContext = GitBookSpaceContext & {
 
     /** Whether to display agent instructions in the markdown output. Defaults to true when undefined. */
     displayAgentInstructions?: boolean;
+
+    /** Whether the request comes from a detected AI agent. Only set for markdown routes. */
+    isAiAgent?: boolean;
 };
 
 /**
@@ -275,6 +285,7 @@ export async function fetchSiteContextByURLLookup(
         noIndexSearch: data.noIndexSearch ?? false,
         isLoggedInVisitor: data.isLoggedInVisitor ?? false,
         displayAgentInstructions: data.displayAgentInstructions,
+        isAiAgent: data.isAiAgent,
     });
 }
 
@@ -297,6 +308,7 @@ export async function fetchSiteContextByIds(
         noIndexSearch: boolean;
         isLoggedInVisitor: boolean;
         displayAgentInstructions?: boolean;
+        isAiAgent?: boolean;
     }
 ): Promise<GitBookSiteContext> {
     const { dataFetcher } = baseContext;
@@ -425,6 +437,7 @@ export async function fetchSiteContextByIds(
         noIndexSearch: ids.noIndexSearch,
         isLoggedInVisitor: ids.isLoggedInVisitor,
         displayAgentInstructions: ids.displayAgentInstructions,
+        isAiAgent: ids.isAiAgent,
     };
 }
 

packages/gitbook/src/middleware.ts

@@ -459,8 +459,11 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
             pathname,
             routeType: routeTypeFromPathname,
             events,
+            isAiAgent,
         } = encodePathInSiteContent(siteURLData, request);
         routeType = routeTypeFromPathname ?? routeType;
+        // Only set for markdown routes, so it becomes part of their static cache key.
+        stableSiteURLData.isAiAgent = isAiAgent;
 
         // Apply a forced theme (`?theme=`/cookie). For the docs embed we thread it through the
         // route context (`embedTheme`) so those routes stay statically rendered — it becomes part
@@ -772,6 +775,8 @@ function encodePathInSiteContent(
     pathname: string;
     routeType?: 'static' | 'dynamic';
     events?: ServerInsightsEventInput[] | undefined;
+    /** Only set for markdown routes, where the output depends on the visitor being an agent. */
+    isAiAgent?: boolean;
 } {
     let pathname = removeLeadingSlash(removeTrailingSlash(siteURLData.pathname));
 
@@ -884,9 +889,8 @@ function encodePathInSiteContent(
             const aiAgentDetection = isAIAgent(request);
             // Using heuristic detection incorrectly detects some legitimate bot requests as AI agents (e.g. Slackbot)
             // We don't want to serve markdown for these requests as it can cause issues like breaking slack unfurling.
-            const shouldServeMarkdown =
-                (aiAgentDetection.detected && aiAgentDetection.method !== 'heuristic') ||
-                acceptsMarkdown(request);
+            const isAiAgent = aiAgentDetection.detected && aiAgentDetection.method !== 'heuristic';
+            const shouldServeMarkdown = isAiAgent || acceptsMarkdown(request);
             if (pathname.match(MARKDOWN_PATH_REGEX) || shouldServeMarkdown) {
                 const pagePathWithoutMD = pathname.replace(MARKDOWN_PATH_REGEX, '');
                 const searchParams = new URL(request.url).searchParams;
@@ -903,6 +907,8 @@ function encodePathInSiteContent(
                               }`
                             : `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`,
                     routeType: 'static',
+                    // Left undefined for non-agents to avoid splitting the static cache for them.
+                    isAiAgent: isAiAgent || undefined,
                     // TODO: track pageId / spaceId when possible
                     // We don't do it at the moment as we can't easily extract it from the URL.
                     events: ask

packages/gitbook/src/routes/markdownPage.ts

@@ -7,6 +7,7 @@ import { linkerWithMarkdownPages } from '@/lib/links';
 import { renderLLMsTxtMarkdownDirective } from '@/lib/llms-directive';
 import { getMarkdownForPage } from '@/lib/markdownPage';
 import { type ResolvedPagePath, getSimilarPages } from '@/lib/pages';
+import { isPageIndexable, isSiteIndexable } from '@/lib/seo';
 import { resolveSiteSpacePagePathDocumentOrGroup } from '@/lib/sites';
 
 /**
@@ -27,17 +28,40 @@ export async function servePageMarkdown(baseContext: GitBookSiteContext, pagePat
         );
         if (!pageLookup) {
             // Generates a markdown body for missing pages. Return this with a 200 status (not 404) because agents discard 404 response bodies.=
-            return renderNotFoundMarkdown(context, pagePath);
+            return {
+                markdown: renderNotFoundMarkdown(context, pagePath),
+                robots: 'noindex, nofollow',
+            };
         }
 
+        const robots = getMarkdownRobots(context, pageLookup);
+
         const markdownPage = await getMarkdownForPage(context, pageLookup);
         if (baseContext.displayAgentInstructions === false) {
-            return markdownPage;
+            return { markdown: markdownPage, robots };
         }
-        return `${renderLLMsTxtMarkdownDirective(context, pageLookup.page)}\n\n${markdownPage}${renderAskFooter(context, pageLookup)}`;
+        return {
+            markdown: `${renderLLMsTxtMarkdownDirective(context, pageLookup.page)}\n\n${markdownPage}${renderAskFooter(context, pageLookup)}`,
+            robots,
+        };
     });
 }
 
+/**
+ * Robots directive for a markdown page: the markdown version is only indexable for AI agents,
+ * and only when the page itself is indexable.
+ */
+function getMarkdownRobots(
+    context: GitBookSiteContext,
+    pageLookup: ResolvedPagePath<RevisionPageDocument | RevisionPageGroup>
+) {
+    if (!isSiteIndexable(context) || !isPageIndexable(pageLookup.ancestors, pageLookup.page)) {
+        return 'noindex, nofollow';
+    }
+
+    return context.isAiAgent ? 'index, follow' : 'noindex';
+}
+
 function renderNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) {
     const similarPages = getSimilarPages(context.revision.pages, pagePath, 5);
     const sitemapUrl = context.linker.toAbsoluteURL(context.linker.toPathInSite('sitemap.md'));
@@ -138,13 +162,17 @@ Use this mechanism when the answer is not explicitly present in the current page
 /**
  * Return a markdown content.
  */
-export async function serveMarkdown(fn: () => Promise<string>) {
+export async function serveMarkdown(
+    fn: () => Promise<string | { markdown: string; robots: string }>
+) {
     try {
-        const markdown = await fn();
+        const result = await fn();
+        const { markdown, robots } =
+            typeof result === 'string' ? { markdown: result, robots: 'noindex' } : result;
         return new Response(markdown, {
             headers: {
                 'Content-Type': 'text/markdown; charset=utf-8',
-                'X-Robots-Tag': 'noindex',
+                'X-Robots-Tag': robots,
                 Vary: 'Accept',
             },
         });

packages/gitbook/tests/markdown.test.ts

@@ -85,7 +85,9 @@ describe('search parameters for indexing crawlers', () => {
 describe('markdown pages', () => {
     it('should expose a markdown page with the .md extension', async () => {
         const response = await fetch(
-            getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/text-page.md')
+            getContentTestURL(
+                'https://gitbook.gitbook.io/test-gitbook-open/text-page.md?x-gitbook-search-indexation=1'
+            )
         );
         const text = await response.text();
 
@@ -97,7 +99,9 @@ describe('markdown pages', () => {
 
     it('should expose a markdown page with the accept header', async () => {
         const response = await fetch(
-            getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/text-page'),
+            getContentTestURL(
+                'https://gitbook.gitbook.io/test-gitbook-open/text-page?x-gitbook-search-indexation=1'
+            ),
             {
                 headers: {
                     Accept: 'text/markdown',
@@ -120,13 +124,15 @@ describe('markdown pages', () => {
 
         expect(response.status).toBe(200);
         expect(response.headers.get('content-type')).toContain('text/markdown');
-        expect(response.headers.get('x-robots-tag')).toBe('noindex');
+        expect(response.headers.get('x-robots-tag')).toBe('noindex, nofollow');
         expect(text).toContain('# Page Not Found');
     });
 
     it('should rewrite links to markdown URLs', async () => {
         const response = await fetch(
-            getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/blocks/links.md')
+            getContentTestURL(
+                'https://gitbook.gitbook.io/test-gitbook-open/blocks/links.md?x-gitbook-search-indexation=1'
+            )
         );
         const text = await response.text();
 
@@ -249,3 +255,65 @@ describe('markdown ask responses', () => {
         { timeout: 30_000 }
     );
 });
+
+describe('markdown robots directives', () => {
+    // Preview deployments block indexation unless this param is set.
+    const INDEXATION_PARAM = 'x-gitbook-search-indexation=1';
+    // Share link sites are never indexable, whatever the visitor is.
+    const SHARE_LINK_PAGE_URL =
+        'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/3.0/other-page';
+
+    it('should be indexable for an AI agent requesting a .md page', async () => {
+        const response = await fetch(getContentTestURL(`${TEST_PAGE_URL}.md?${INDEXATION_PARAM}`), {
+            headers: { 'User-Agent': 'ClaudeBot/1.0' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+        expect(response.headers.get('x-robots-tag')).toBe('index, follow');
+    });
+
+    it('should be indexable for an AI agent served markdown on the page URL', async () => {
+        const response = await fetch(getContentTestURL(`${TEST_PAGE_URL}?${INDEXATION_PARAM}`), {
+            headers: { 'User-Agent': 'GPTBot/1.2' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+        expect(response.headers.get('x-robots-tag')).toBe('index, follow');
+    });
+
+    it('should stay noindex for a non-agent requesting a .md page', async () => {
+        const response = await fetch(getContentTestURL(`${TEST_PAGE_URL}.md?${INDEXATION_PARAM}`), {
+            headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+        expect(response.headers.get('x-robots-tag')).toBe('noindex');
+    });
+
+    it('should be noindex, nofollow for an AI agent on a non-indexable site', async () => {
+        const response = await fetch(
+            getContentTestURL(`${SHARE_LINK_PAGE_URL}.md?${INDEXATION_PARAM}`),
+            { headers: { 'User-Agent': 'ClaudeBot/1.0' } }
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+        expect(response.headers.get('x-robots-tag')).toBe('noindex, nofollow');
+    });
+
+    it('should be noindex, nofollow for an AI agent on a missing page', async () => {
+        const response = await fetch(
+            getContentTestURL(
+                `https://gitbook.gitbook.io/test-gitbook-open/missing-page.md?${INDEXATION_PARAM}`
+            ),
+            { headers: { 'User-Agent': 'ClaudeBot/1.0' } }
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+        expect(response.headers.get('x-robots-tag')).toBe('noindex, nofollow');
+    });
+});