Block crawler from using ask endpoint (#4457)

packages/gitbook/src/lib/indexing-crawlers.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'bun:test';
+import { isAITrainingOrIndexingRequest } from './indexing-crawlers';
+
+describe('isAITrainingOrIndexingRequest', () => {
+    it('detects the configured AI training and indexing crawlers on ask and search endpoints', () => {
+        for (const [userAgent, parameter] of [
+            ['Meta-ExternalAgent/1.1', 'ask'],
+            ['meta-webindexer/1.0', 'q'],
+            ['Amazonbot/0.1', 'ask'],
+        ] as const) {
+            expect(
+                isAITrainingOrIndexingRequest(
+                    new Request(`https://docs.example.com/page?${parameter}=query`, {
+                        headers: { 'User-Agent': userAgent },
+                    })
+                )
+            ).toBe(true);
+        }
+    });
+
+    it('does not apply to regular pages or unlisted crawlers', () => {
+        expect(
+            isAITrainingOrIndexingRequest(
+                new Request('https://docs.example.com/page', {
+                    headers: { 'User-Agent': 'Meta-ExternalAgent/1.1' },
+                })
+            )
+        ).toBe(false);
+        expect(
+            isAITrainingOrIndexingRequest(
+                new Request('https://docs.example.com/page?ask=Question', {
+                    headers: {
+                        'User-Agent':
+                            'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+                    },
+                })
+            )
+        ).toBe(false);
+        expect(
+            isAITrainingOrIndexingRequest(
+                new Request('https://docs.example.com/page?ask=Question&q=query', {
+                    headers: { 'User-Agent': 'GPTBot/1.2' },
+                })
+            )
+        ).toBe(false);
+    });
+});

packages/gitbook/src/lib/indexing-crawlers.ts

@@ -0,0 +1,25 @@
+const AI_TRAINING_OR_INDEXING_USER_AGENT_PATTERNS = [
+    // We try to be conservative here, and only act on bot causing excessive load
+    'meta-externalagent',
+    'meta-webindexer',
+    'amazonbot',
+] as const;
+
+function isAITrainingOrIndexingCrawler(request: Request): boolean {
+    const userAgent = request.headers.get('user-agent')?.toLowerCase() ?? '';
+    return AI_TRAINING_OR_INDEXING_USER_AGENT_PATTERNS.some((pattern) =>
+        userAgent.includes(pattern)
+    );
+}
+
+/**
+ * Detect AI training and indexing crawlers accessing an internal search endpoint.
+ */
+export function isAITrainingOrIndexingRequest(request: Request): boolean {
+    if (!isAITrainingOrIndexingCrawler(request)) {
+        return false;
+    }
+
+    const searchParams = new URL(request.url).searchParams;
+    return searchParams.has('ask') || searchParams.has('q');
+}

packages/gitbook/src/middleware.ts

@@ -28,6 +28,7 @@ import {
 } from '@/lib/data';
 import { isGitBookAssetsHostURL, isGitBookHostURL } from '@/lib/env';
 import { getImageResizingContextId } from '@/lib/images';
+import { isAITrainingOrIndexingRequest } from '@/lib/indexing-crawlers';
 import { MiddlewareHeaders } from '@/lib/middleware';
 import {
     createOAuthProtectedResourceMetadataResponse,
@@ -151,6 +152,13 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
 
     const { url: siteRequestURL, mode } = match;
 
+    if (isAITrainingOrIndexingRequest(request)) {
+        return new Response('This endpoint is not intended for AI training or indexing.', {
+            status: 403,
+            headers: { 'content-type': 'text/plain; charset=utf-8' },
+        });
+    }
+
     // Normalize URL after extracting the URL from the request to make sure the client is redirected to the proper one
     const normalizationResponse = normalizeRequestURL(siteRequestURL);
     if (normalizationResponse) {

packages/gitbook/tests/markdown.test.ts

@@ -50,6 +50,37 @@ describe('markdown serving based on user agent', () => {
     });
 });
 
+describe('search parameters for indexing crawlers', () => {
+    const ASK_QUESTION = 'This question must not reach Ask AI';
+    const SEARCH_QUERY = 'This query must not reach search';
+
+    it('should reject Ask AI requests from Meta external agents', async () => {
+        const response = await fetch(
+            getContentTestURL(
+                `${TEST_PAGE_URL}?ask=${encodeURIComponent(ASK_QUESTION)}&goal=Read%20the%20docs`
+            ),
+            { headers: { 'User-Agent': 'meta-externalagent/1.1' } }
+        );
+        expect(response.status).toBe(403);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+        expect(await response.text()).toBe(
+            'This endpoint is not intended for AI training or indexing.'
+        );
+    });
+
+    it('should reject search requests from Amazonbot', async () => {
+        const response = await fetch(
+            getContentTestURL(`${TEST_PAGE_URL}?q=${encodeURIComponent(SEARCH_QUERY)}`),
+            { headers: { 'User-Agent': 'Amazonbot/0.1' } }
+        );
+        expect(response.status).toBe(403);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+        expect(await response.text()).toBe(
+            'This endpoint is not intended for AI training or indexing.'
+        );
+    });
+});
+
 describe('markdown pages', () => {
     it('should expose a markdown page with the .md extension', async () => {
         const response = await fetch(