Add support for ChatGPT Search (OAI-SearchBot) user agent (#4633)

.changeset/friendly-bots-search.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Serve Markdown responses as plain text to OpenAI's ChatGPT Search crawler.

packages/gitbook/src/lib/chatgpt.ts

@@ -6,6 +6,7 @@ export function isChatGPTRequest(request: Pick<Request, 'headers'>): boolean {
     return (
         normalizedUserAgent.includes('chatgpt-user') ||
         normalizedUserAgent.includes('chatgpt agent') ||
+        normalizedUserAgent.includes('oai-searchbot') ||
         normalizedSignatureAgent.includes('chatgpt.com')
     );
 }

packages/gitbook/src/lib/markdown-content-type.test.ts

@@ -9,6 +9,14 @@ describe('ChatGPT Markdown compatibility', () => {
     it('detects ChatGPT requests without matching other agents', () => {
         expect(isChatGPTRequest(requestWith({ 'user-agent': 'ChatGPT-User/1.0' }))).toBe(true);
         expect(isChatGPTRequest(requestWith({ 'user-agent': 'ChatGPT Agent' }))).toBe(true);
+        expect(
+            isChatGPTRequest(
+                requestWith({
+                    'user-agent':
+                        'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot',
+                })
+            )
+        ).toBe(true);
         expect(
             isChatGPTRequest(
                 requestWith({

packages/gitbook/tests/llms.test.ts

@@ -55,6 +55,22 @@ describe('llms.txt', () => {
         expect(await response.text()).toContain('# E2E Tests GitBook Open');
     });
 
+    it('should serve plain text to ChatGPT Search', async () => {
+        const response = await fetch(
+            getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms.txt'),
+            {
+                headers: {
+                    'User-Agent':
+                        'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot',
+                },
+            }
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+        expect(await response.text()).toContain('# E2E Tests GitBook Open');
+    });
+
     it('should expose llms.txt from sitemap.md', async () => {
         const response = await fetch(
             getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/sitemap.md')

packages/gitbook/tests/markdown.test.ts

@@ -38,6 +38,18 @@ describe('markdown serving based on user agent', () => {
         expect(response.headers.get('content-type')).toContain('text/plain');
     });
 
+    it('should serve plain text to ChatGPT Search', async () => {
+        const response = await fetch(getContentTestURL(TEST_PAGE_URL), {
+            headers: {
+                'User-Agent':
+                    'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot',
+            },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+    });
+
     it('should NOT serve markdown to Slackbot (heuristic detection only)', async () => {
         const response = await fetch(getContentTestURL(TEST_PAGE_URL), {
             headers: {