diff --git a/.changeset/friendly-bots-search.md b/.changeset/friendly-bots-search.md new file mode 100644 index 000000000..6429969a0 --- /dev/null +++ b/.changeset/friendly-bots-search.md @@ -0,0 +1,5 @@ +--- +"gitbook": patch +--- + +Serve Markdown responses as plain text to OpenAI's ChatGPT Search crawler. diff --git a/packages/gitbook/src/lib/chatgpt.ts b/packages/gitbook/src/lib/chatgpt.ts index ee0e3083e..d46a02ca3 100644 --- a/packages/gitbook/src/lib/chatgpt.ts +++ b/packages/gitbook/src/lib/chatgpt.ts @@ -6,6 +6,7 @@ export function isChatGPTRequest(request: Pick): boolean { return ( normalizedUserAgent.includes('chatgpt-user') || normalizedUserAgent.includes('chatgpt agent') || + normalizedUserAgent.includes('oai-searchbot') || normalizedSignatureAgent.includes('chatgpt.com') ); } diff --git a/packages/gitbook/src/lib/markdown-content-type.test.ts b/packages/gitbook/src/lib/markdown-content-type.test.ts index 835a8f3a5..24f41f294 100644 --- a/packages/gitbook/src/lib/markdown-content-type.test.ts +++ b/packages/gitbook/src/lib/markdown-content-type.test.ts @@ -9,6 +9,14 @@ describe('ChatGPT Markdown compatibility', () => { it('detects ChatGPT requests without matching other agents', () => { expect(isChatGPTRequest(requestWith({ 'user-agent': 'ChatGPT-User/1.0' }))).toBe(true); expect(isChatGPTRequest(requestWith({ 'user-agent': 'ChatGPT Agent' }))).toBe(true); + expect( + isChatGPTRequest( + requestWith({ + 'user-agent': + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot', + }) + ) + ).toBe(true); expect( isChatGPTRequest( requestWith({ diff --git a/packages/gitbook/tests/llms.test.ts b/packages/gitbook/tests/llms.test.ts index 91ad1777c..735ad593e 100644 --- a/packages/gitbook/tests/llms.test.ts +++ b/packages/gitbook/tests/llms.test.ts @@ -55,6 +55,22 @@ describe('llms.txt', () => { expect(await response.text()).toContain('# E2E Tests GitBook Open'); }); + it('should serve plain text to ChatGPT Search', async () => { + const response = await fetch( + getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/llms.txt'), + { + headers: { + 'User-Agent': + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot', + }, + } + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/plain'); + expect(await response.text()).toContain('# E2E Tests GitBook Open'); + }); + it('should expose llms.txt from sitemap.md', async () => { const response = await fetch( getContentTestURL('https://gitbook.gitbook.io/test-gitbook-open/sitemap.md') diff --git a/packages/gitbook/tests/markdown.test.ts b/packages/gitbook/tests/markdown.test.ts index 07405d8b8..17986462e 100644 --- a/packages/gitbook/tests/markdown.test.ts +++ b/packages/gitbook/tests/markdown.test.ts @@ -38,6 +38,18 @@ describe('markdown serving based on user agent', () => { expect(response.headers.get('content-type')).toContain('text/plain'); }); + it('should serve plain text to ChatGPT Search', async () => { + const response = await fetch(getContentTestURL(TEST_PAGE_URL), { + headers: { + 'User-Agent': + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot', + }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/plain'); + }); + it('should NOT serve markdown to Slackbot (heuristic detection only)', async () => { const response = await fetch(getContentTestURL(TEST_PAGE_URL), { headers: {