Add negotiator for markdown acceptance in middleware (#4238)

Co-authored-by: Copilot <copilot@github.com>

bun.lock

@@ -164,6 +164,7 @@
         "micromark-extension-frontmatter": "^2.0.0",
         "micromark-extension-gfm": "^3.0.0",
         "motion": "^12.23.24",
+        "negotiator": "^1.0.0",
         "next": "^16.2.3",
         "next-themes": "^0.4.6",
         "nuqs": "^2.2.3",

packages/gitbook/package.json

@@ -56,6 +56,7 @@
         "micromark-extension-frontmatter": "^2.0.0",
         "micromark-extension-gfm": "^3.0.0",
         "motion": "^12.23.24",
+        "negotiator": "^1.0.0",
         "next": "^16.2.3",
         "next-themes": "^0.4.6",
         "nuqs": "^2.2.3",

packages/gitbook/src/middleware.ts

@@ -3,7 +3,7 @@ import {
     SiteInsightsDisplayContext,
     SiteInsightsLLMSVariant,
 } from '@gitbook/api';
-import { acceptsMarkdown, isAIAgent } from '@vercel/agent-readability';
+import { isAIAgent } from '@vercel/agent-readability';
 import { cookies } from 'next/headers';
 import type { NextRequest } from 'next/server';
 import { NextResponse } from 'next/server';
@@ -44,6 +44,7 @@ import {
 } from '@/lib/visitors';
 import { waitUntil } from '@/lib/waitUntil';
 import { serveResizedImage } from '@/routes/image';
+import Negotiator from 'negotiator';
 import {
     type ServerInsightsEventInput,
     serveProxyAnalyticsEvent,
@@ -498,6 +499,13 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
         response.headers.set('x-gitbook-route-type', routeType);
         response.headers.set('x-gitbook-route-site', siteURLWithoutProtocol);
 
+        // AI related headers
+        // This one is technically useless, but is used by a bunch of scoring systems
+        response.headers.set(
+            'vary',
+            'rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch, accept-encoding, accept'
+        );
+
         // When we use adaptive content, we want to ensure that the cache is not used at all on the client side.
         // Vercel already set this header, this is needed in OpenNext.
         if (siteURLData.contextId && !siteRequestURL.pathname.endsWith('~gitbook/site-index')) {
@@ -849,3 +857,20 @@ async function writeResponseCookies<R extends NextResponse>(
 
     return response;
 }
+
+function acceptsMarkdown(request: Request): boolean {
+    const acceptHeader = request.headers.get('accept') || '';
+
+    const negotiator = new Negotiator({ headers: { accept: acceptHeader } });
+    const mediaTypes = negotiator.mediaTypes();
+
+    // Media types are in order of preference, so we check if the client has markdown as one of its favorites,
+    // but text/html and */* should take precedence.
+    const markdownIndex = mediaTypes.findIndex(
+        (type) => type === 'text/markdown' || type === 'text/x-markdown'
+    );
+    if (markdownIndex === -1) return false;
+
+    const htmlIndex = mediaTypes.findIndex((type) => type === 'text/html' || type === '*/*');
+    return htmlIndex === -1 || markdownIndex < htmlIndex;
+}

packages/gitbook/src/routes/markdownPage.ts

@@ -126,6 +126,7 @@ export async function serveMarkdown(fn: () => Promise<string>) {
             headers: {
                 'Content-Type': 'text/markdown; charset=utf-8',
                 'X-Robots-Tag': 'noindex',
+                Vary: 'Accept',
             },
         });
     } catch (error) {
@@ -134,6 +135,7 @@ export async function serveMarkdown(fn: () => Promise<string>) {
             status: exposable.code,
             headers: {
                 'Content-Type': 'text/plain; charset=utf-8',
+                Vary: 'Accept',
             },
         });
     }

packages/gitbook/tests/markdown.test.ts

@@ -105,6 +105,73 @@ describe('markdown pages', () => {
     });
 });
 
+describe('Accept header content negotiation', () => {
+    const PAGE_URL = getContentTestURL(TEST_PAGE_URL);
+
+    it('should NOT serve markdown for Accept: text/html', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: 'text/html' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/html');
+    });
+
+    it('should NOT serve markdown for Accept: */*', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: '*/*' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/html');
+    });
+
+    it('should NOT serve markdown when text/html is preferred over text/markdown (Accept: text/html, text/markdown)', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: 'text/html, text/markdown' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/html');
+    });
+
+    it('should serve markdown when text/markdown is preferred over text/html (Accept: text/markdown, text/html)', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: 'text/markdown, text/html' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+    });
+
+    it('should serve markdown when text/markdown has a higher q-value (Accept: text/html;q=0.9, text/markdown)', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: 'text/html;q=0.9, text/markdown' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+    });
+
+    it('should NOT serve markdown when text/markdown has a lower q-value (Accept: text/html, text/markdown;q=0.9)', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: 'text/html, text/markdown;q=0.9' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/html');
+    });
+
+    it('should serve markdown for Accept: text/x-markdown', async () => {
+        const response = await fetch(PAGE_URL, {
+            headers: { Accept: 'text/x-markdown' },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+    });
+});
+
 describe('markdown ask responses', () => {
     const ASK_QUESTION = 'What is GitBook?';
     const ASK_QUESTION_HEADING = `# ${ASK_QUESTION}`;