From ca9a1dd39697c49f56ac6f8910a7d4aad35704a8 Mon Sep 17 00:00:00 2001 From: conico974 Date: Mon, 3 Aug 2026 11:47:17 +0200 Subject: [PATCH] Block crawler from using ask endpoint (#4457) --- .../gitbook/src/lib/indexing-crawlers.test.ts | 47 +++++++++++++++++++ packages/gitbook/src/lib/indexing-crawlers.ts | 25 ++++++++++ packages/gitbook/src/middleware.ts | 8 ++++ packages/gitbook/tests/markdown.test.ts | 31 ++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 packages/gitbook/src/lib/indexing-crawlers.test.ts create mode 100644 packages/gitbook/src/lib/indexing-crawlers.ts diff --git a/packages/gitbook/src/lib/indexing-crawlers.test.ts b/packages/gitbook/src/lib/indexing-crawlers.test.ts new file mode 100644 index 000000000..8bf07f5d0 --- /dev/null +++ b/packages/gitbook/src/lib/indexing-crawlers.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'bun:test'; +import { isAITrainingOrIndexingRequest } from './indexing-crawlers'; + +describe('isAITrainingOrIndexingRequest', () => { + it('detects the configured AI training and indexing crawlers on ask and search endpoints', () => { + for (const [userAgent, parameter] of [ + ['Meta-ExternalAgent/1.1', 'ask'], + ['meta-webindexer/1.0', 'q'], + ['Amazonbot/0.1', 'ask'], + ] as const) { + expect( + isAITrainingOrIndexingRequest( + new Request(`https://docs.example.com/page?${parameter}=query`, { + headers: { 'User-Agent': userAgent }, + }) + ) + ).toBe(true); + } + }); + + it('does not apply to regular pages or unlisted crawlers', () => { + expect( + isAITrainingOrIndexingRequest( + new Request('https://docs.example.com/page', { + headers: { 'User-Agent': 'Meta-ExternalAgent/1.1' }, + }) + ) + ).toBe(false); + expect( + isAITrainingOrIndexingRequest( + new Request('https://docs.example.com/page?ask=Question', { + headers: { + 'User-Agent': + 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + }, + }) + ) + ).toBe(false); + expect( + isAITrainingOrIndexingRequest( + new Request('https://docs.example.com/page?ask=Question&q=query', { + headers: { 'User-Agent': 'GPTBot/1.2' }, + }) + ) + ).toBe(false); + }); +}); diff --git a/packages/gitbook/src/lib/indexing-crawlers.ts b/packages/gitbook/src/lib/indexing-crawlers.ts new file mode 100644 index 000000000..382c3953d --- /dev/null +++ b/packages/gitbook/src/lib/indexing-crawlers.ts @@ -0,0 +1,25 @@ +const AI_TRAINING_OR_INDEXING_USER_AGENT_PATTERNS = [ + // We try to be conservative here, and only act on bot causing excessive load + 'meta-externalagent', + 'meta-webindexer', + 'amazonbot', +] as const; + +function isAITrainingOrIndexingCrawler(request: Request): boolean { + const userAgent = request.headers.get('user-agent')?.toLowerCase() ?? ''; + return AI_TRAINING_OR_INDEXING_USER_AGENT_PATTERNS.some((pattern) => + userAgent.includes(pattern) + ); +} + +/** + * Detect AI training and indexing crawlers accessing an internal search endpoint. + */ +export function isAITrainingOrIndexingRequest(request: Request): boolean { + if (!isAITrainingOrIndexingCrawler(request)) { + return false; + } + + const searchParams = new URL(request.url).searchParams; + return searchParams.has('ask') || searchParams.has('q'); +} diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index e45ad1e80..176b6273b 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -28,6 +28,7 @@ import { } from '@/lib/data'; import { isGitBookAssetsHostURL, isGitBookHostURL } from '@/lib/env'; import { getImageResizingContextId } from '@/lib/images'; +import { isAITrainingOrIndexingRequest } from '@/lib/indexing-crawlers'; import { MiddlewareHeaders } from '@/lib/middleware'; import { createOAuthProtectedResourceMetadataResponse, @@ -151,6 +152,13 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) { const { url: siteRequestURL, mode } = match; + if (isAITrainingOrIndexingRequest(request)) { + return new Response('This endpoint is not intended for AI training or indexing.', { + status: 403, + headers: { 'content-type': 'text/plain; charset=utf-8' }, + }); + } + // Normalize URL after extracting the URL from the request to make sure the client is redirected to the proper one const normalizationResponse = normalizeRequestURL(siteRequestURL); if (normalizationResponse) { diff --git a/packages/gitbook/tests/markdown.test.ts b/packages/gitbook/tests/markdown.test.ts index d2d55b676..da28916f2 100644 --- a/packages/gitbook/tests/markdown.test.ts +++ b/packages/gitbook/tests/markdown.test.ts @@ -50,6 +50,37 @@ describe('markdown serving based on user agent', () => { }); }); +describe('search parameters for indexing crawlers', () => { + const ASK_QUESTION = 'This question must not reach Ask AI'; + const SEARCH_QUERY = 'This query must not reach search'; + + it('should reject Ask AI requests from Meta external agents', async () => { + const response = await fetch( + getContentTestURL( + `${TEST_PAGE_URL}?ask=${encodeURIComponent(ASK_QUESTION)}&goal=Read%20the%20docs` + ), + { headers: { 'User-Agent': 'meta-externalagent/1.1' } } + ); + expect(response.status).toBe(403); + expect(response.headers.get('content-type')).toContain('text/plain'); + expect(await response.text()).toBe( + 'This endpoint is not intended for AI training or indexing.' + ); + }); + + it('should reject search requests from Amazonbot', async () => { + const response = await fetch( + getContentTestURL(`${TEST_PAGE_URL}?q=${encodeURIComponent(SEARCH_QUERY)}`), + { headers: { 'User-Agent': 'Amazonbot/0.1' } } + ); + expect(response.status).toBe(403); + expect(response.headers.get('content-type')).toContain('text/plain'); + expect(await response.text()).toBe( + 'This endpoint is not intended for AI training or indexing.' + ); + }); +}); + describe('markdown pages', () => { it('should expose a markdown page with the .md extension', async () => { const response = await fetch(