Always allow LLMs to read the content (#4264)

.changeset/allow-llm-crawling-non-indexed-sites.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Allow user-triggered AI assistants (ChatGPT-User, Claude-User, Perplexity-User) to read pages on non-indexable sites, so end-users can pull content into an LLM without an MCP connection. Search engines and training crawlers remain blocked.

packages/gitbook/src/routes/robots.ts

@@ -1,5 +1,14 @@
 import { type GitBookSiteContext, checkIsRootSiteContext } from '@/lib/context';
 import { isSiteIndexable } from '@/lib/seo';
+import { SiteVisibility } from '@gitbook/api';
+
+/**
+ * User-agents of AI assistants that fetch pages live in response to a user prompt.
+ * These are allowed to read pages even when the site is not indexable in search
+ * engines, so end-users can pull content into an LLM without needing an MCP
+ * connection — including when they ask their assistant to fetch a specific URL.
+ */
+const AI_USER_AGENTS = ['ChatGPT-User', 'Claude-User', 'Perplexity-User'];
 
 /**
  * Generate a robots.txt for a site.
@@ -9,6 +18,7 @@ export async function serveRobotsTxt(context: GitBookSiteContext) {
 
     const isRoot = checkIsRootSiteContext(context);
     const isIndexable = isSiteIndexable(context);
+    const isSitePublic = context.site.visibility === SiteVisibility.Public;
 
     const sitemapPath = linker.toPathInSpace(isRoot ? '/sitemap.xml' : '/sitemap-pages.xml');
     const sitemapUrl = linker.toAbsoluteURL(sitemapPath);
@@ -27,7 +37,23 @@ export async function serveRobotsTxt(context: GitBookSiteContext) {
               'Allow: /',
               `Sitemap: ${sitemapUrl}`,
           ]
-        : ['User-agent: *', 'Content-Signal: ai-train=no, search=no, ai-input=no', 'Disallow: /'];
+        : [
+              // Allow user-triggered AI assistants to read pages even when the
+              // site is not indexable, so end-users can pull content into an LLM.
+              // Training crawlers and search engines remain blocked.
+              // If site is not public, we don't allow it.
+              ...(isSitePublic
+                  ? AI_USER_AGENTS.flatMap((userAgent) => [
+                        `User-agent: ${userAgent}`,
+                        'Content-Signal: ai-train=no, search=no, ai-input=yes',
+                        'Allow: /',
+                        '',
+                    ])
+                  : []),
+              'User-agent: *',
+              'Content-Signal: ai-train=no, search=no, ai-input=no',
+              'Disallow: /',
+          ];
 
     return new Response(`${lines.join('\n')}\n`, { headers: { 'Content-Type': 'text/plain' } });
 }

packages/gitbook/tests/robots.test.ts

@@ -29,4 +29,18 @@ describe('robots.txt', () => {
         expect(content).toContain('Disallow: /\n');
         expect(content).toContain('Content-Signal: ai-train=no, search=no, ai-input=no');
     });
+
+    it('disallow user-triggered AI assistants to read non-public sites', async () => {
+        const response = await fetch(
+            getContentTestURL(
+                'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/robots.txt?x-gitbook-search-indexation=1'
+            )
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+        const content = await response.text();
+        expect(content).toContain('Disallow: /\n');
+        expect(content).toContain('Content-Signal: ai-train=no, search=no, ai-input=no');
+    });
 });