From 8bfced2e0de48569ec7a69589eb344795ec4213d Mon Sep 17 00:00:00 2001 From: conico974 Date: Wed, 6 May 2026 14:59:38 +0200 Subject: [PATCH] Add negotiator for markdown acceptance in middleware (#4238) Co-authored-by: Copilot --- bun.lock | 1 + packages/gitbook/package.json | 1 + packages/gitbook/src/middleware.ts | 27 ++++++++- packages/gitbook/src/routes/markdownPage.ts | 2 + packages/gitbook/tests/markdown.test.ts | 67 +++++++++++++++++++++ 5 files changed, 97 insertions(+), 1 deletion(-) diff --git a/bun.lock b/bun.lock index 4ec219d5c..7f33fe6c4 100644 --- a/bun.lock +++ b/bun.lock @@ -164,6 +164,7 @@ "micromark-extension-frontmatter": "^2.0.0", "micromark-extension-gfm": "^3.0.0", "motion": "^12.23.24", + "negotiator": "^1.0.0", "next": "^16.2.3", "next-themes": "^0.4.6", "nuqs": "^2.2.3", diff --git a/packages/gitbook/package.json b/packages/gitbook/package.json index f67cb7c6d..7392b9d8a 100644 --- a/packages/gitbook/package.json +++ b/packages/gitbook/package.json @@ -56,6 +56,7 @@ "micromark-extension-frontmatter": "^2.0.0", "micromark-extension-gfm": "^3.0.0", "motion": "^12.23.24", + "negotiator": "^1.0.0", "next": "^16.2.3", "next-themes": "^0.4.6", "nuqs": "^2.2.3", diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index cdf560a42..168388cb2 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -3,7 +3,7 @@ import { SiteInsightsDisplayContext, SiteInsightsLLMSVariant, } from '@gitbook/api'; -import { acceptsMarkdown, isAIAgent } from '@vercel/agent-readability'; +import { isAIAgent } from '@vercel/agent-readability'; import { cookies } from 'next/headers'; import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; @@ -44,6 +44,7 @@ import { } from '@/lib/visitors'; import { waitUntil } from '@/lib/waitUntil'; import { serveResizedImage } from '@/routes/image'; +import Negotiator from 'negotiator'; import { type ServerInsightsEventInput, serveProxyAnalyticsEvent, @@ -498,6 +499,13 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) { response.headers.set('x-gitbook-route-type', routeType); response.headers.set('x-gitbook-route-site', siteURLWithoutProtocol); + // AI related headers + // This one is technically useless, but is used by a bunch of scoring systems + response.headers.set( + 'vary', + 'rsc, next-router-state-tree, next-router-prefetch, next-router-segment-prefetch, accept-encoding, accept' + ); + // When we use adaptive content, we want to ensure that the cache is not used at all on the client side. // Vercel already set this header, this is needed in OpenNext. if (siteURLData.contextId && !siteRequestURL.pathname.endsWith('~gitbook/site-index')) { @@ -849,3 +857,20 @@ async function writeResponseCookies( return response; } + +function acceptsMarkdown(request: Request): boolean { + const acceptHeader = request.headers.get('accept') || ''; + + const negotiator = new Negotiator({ headers: { accept: acceptHeader } }); + const mediaTypes = negotiator.mediaTypes(); + + // Media types are in order of preference, so we check if the client has markdown as one of its favorites, + // but text/html and */* should take precedence. + const markdownIndex = mediaTypes.findIndex( + (type) => type === 'text/markdown' || type === 'text/x-markdown' + ); + if (markdownIndex === -1) return false; + + const htmlIndex = mediaTypes.findIndex((type) => type === 'text/html' || type === '*/*'); + return htmlIndex === -1 || markdownIndex < htmlIndex; +} diff --git a/packages/gitbook/src/routes/markdownPage.ts b/packages/gitbook/src/routes/markdownPage.ts index 72acf9e0d..fd9d8f91f 100644 --- a/packages/gitbook/src/routes/markdownPage.ts +++ b/packages/gitbook/src/routes/markdownPage.ts @@ -126,6 +126,7 @@ export async function serveMarkdown(fn: () => Promise) { headers: { 'Content-Type': 'text/markdown; charset=utf-8', 'X-Robots-Tag': 'noindex', + Vary: 'Accept', }, }); } catch (error) { @@ -134,6 +135,7 @@ export async function serveMarkdown(fn: () => Promise) { status: exposable.code, headers: { 'Content-Type': 'text/plain; charset=utf-8', + Vary: 'Accept', }, }); } diff --git a/packages/gitbook/tests/markdown.test.ts b/packages/gitbook/tests/markdown.test.ts index 72a1dcce6..0360e675f 100644 --- a/packages/gitbook/tests/markdown.test.ts +++ b/packages/gitbook/tests/markdown.test.ts @@ -105,6 +105,73 @@ describe('markdown pages', () => { }); }); +describe('Accept header content negotiation', () => { + const PAGE_URL = getContentTestURL(TEST_PAGE_URL); + + it('should NOT serve markdown for Accept: text/html', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: 'text/html' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/html'); + }); + + it('should NOT serve markdown for Accept: */*', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: '*/*' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/html'); + }); + + it('should NOT serve markdown when text/html is preferred over text/markdown (Accept: text/html, text/markdown)', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: 'text/html, text/markdown' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/html'); + }); + + it('should serve markdown when text/markdown is preferred over text/html (Accept: text/markdown, text/html)', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: 'text/markdown, text/html' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); + + it('should serve markdown when text/markdown has a higher q-value (Accept: text/html;q=0.9, text/markdown)', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: 'text/html;q=0.9, text/markdown' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); + + it('should NOT serve markdown when text/markdown has a lower q-value (Accept: text/html, text/markdown;q=0.9)', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: 'text/html, text/markdown;q=0.9' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/html'); + }); + + it('should serve markdown for Accept: text/x-markdown', async () => { + const response = await fetch(PAGE_URL, { + headers: { Accept: 'text/x-markdown' }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); +}); + describe('markdown ask responses', () => { const ASK_QUESTION = 'What is GitBook?'; const ASK_QUESTION_HEADING = `# ${ASK_QUESTION}`;