Add test to cover longest URL match for nested site spaces

.changeset/mcp-getpage-markdown-urls.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Fix the MCP `getPage` tool returning "Page not found" for the Markdown URL of a page (`<page>.md`).

.changeset/proxy-normalized-redirect.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Fix encoded or non-normalized links on proxied sites redirecting to the internal proxy path instead of the public URL.

packages/gitbook/src/lib/data/urls.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'bun:test';
 
-import { getURLLookup, getURLLookupPathname, normalizeURL } from './urls';
+import { getURLLookup, getURLLookupPathname, normalizeRequestURL, normalizeURL } from './urls';
 
 describe('getURLLookupPathname', () => {
     const previewRoot = 'https://sites.gitbook.com/preview/site_example/section';
@@ -162,6 +162,51 @@ describe('getURLLookup', () => {
     });
 });
 
+describe('normalizeRequestURL', () => {
+    it('does not redirect a normalized URL', () => {
+        const url = new URL('https://docs.mycompany.com/hello');
+        expect(normalizeRequestURL(url, 'url-host')).toEqual({ isNormalized: true });
+    });
+
+    it('redirects a non-normalized URL to the normalized one', () => {
+        const result = normalizeRequestURL(
+            new URL('https://docs.mycompany.com/hello/tes%74'),
+            'url-host'
+        );
+        expect(result).toBeInstanceOf(Response);
+        const response = result as Response;
+        expect(response.status).toBe(302);
+        expect(response.headers.get('location')).toBe('https://docs.mycompany.com/hello/test');
+    });
+
+    it('does not redirect a non-normalized proxied URL to the internal proxy URL', () => {
+        const url = new URL(
+            'https://proxy.gitbook.site/sites/site_abc/getting-started/quickstar%74'
+        );
+        expect(normalizeRequestURL(url, 'url-host')).toEqual({ isNormalized: false });
+    });
+
+    it('redirects a non-normalized proxied URL in url mode', () => {
+        const result = normalizeRequestURL(
+            new URL('https://proxy.gitbook.site/sites/site_abc/quickstar%74'),
+            'url'
+        );
+        expect(result).toBeInstanceOf(Response);
+        expect((result as Response).headers.get('location')).toBe(
+            'https://proxy.gitbook.site/sites/site_abc/quickstart'
+        );
+    });
+
+    it.each([
+        'https://docs.mycompany.com/hello/%ZZ',
+        'https://proxy.gitbook.site/sites/site_abc/%ZZ',
+    ])('returns a 400 for an invalid URL %s', (url) => {
+        const result = normalizeRequestURL(new URL(url), 'url-host');
+        expect(result).toBeInstanceOf(Response);
+        expect((result as Response).status).toBe(400);
+    });
+});
+
 describe('normalizeURL', () => {
     it('should remove trailing slashes', () => {
         expect(normalizeURL(new URL('https://docs.mycompany.com/hello/'))).toEqual(

packages/gitbook/src/lib/data/urls.ts

@@ -1,5 +1,5 @@
 import { joinPath, removeTrailingSlash } from '../paths';
-import { isProxyRootRequest } from '../proxy';
+import { isProxyRequest, isProxyRootRequest } from '../proxy';
 import { DataFetcherError, getExposableError } from './errors';
 
 /**
@@ -105,21 +105,28 @@ export function getURLLookupPathname(
 }
 
 /**
- * Normalize the URL in a request and redirect if the normalized URL is different from the original one.
+ * Normalize the URL in a request before the lookup. Returns an error response if it is invalid,
+ * or a redirect to the normalized URL if it can be built from the request URL.
  */
-export function normalizeRequestURL(url: URL): Response | null {
+export function normalizeRequestURL(
+    url: URL,
+    mode: 'url' | 'url-host'
+): Response | { isNormalized: boolean } {
+    let normalizedURL: URL;
     try {
-        const normalizedURL = normalizeURL(url);
-
-        if (normalizedURL.toString() !== url.toString()) {
-            return Response.redirect(normalizedURL.toString(), 302);
-        }
-
-        return null;
+        normalizedURL = normalizeURL(url);
     } catch (error) {
         const sanitized = getExposableError(error);
         return new Response(sanitized.message, { status: sanitized.code });
     }
+
+    const isNormalized = normalizedURL.toString() === url.toString();
+    // A proxied site's public URL is only known after the lookup, which redirects to the canonical URL.
+    if (!isNormalized && !(mode !== 'url' && isProxyRequest(url))) {
+        return Response.redirect(normalizedURL.toString(), 302);
+    }
+
+    return { isNormalized };
 }
 
 /**

packages/gitbook/src/lib/pages.test.ts

@@ -50,6 +50,16 @@ describe('extractPagePath', () => {
     it('returns empty string when URL equals root base URL', () => {
         expect(extractPagePath('https://docs.example.com/', 'https://docs.example.com/')).toBe('');
     });
+
+    it('extracts path from the markdown URL of a page', () => {
+        expect(
+            extractPagePath('https://docs.example.com/api/guides/installation.md', baseURL)
+        ).toBe('guides/installation');
+        expect(extractPagePath('https://docs.example.com/api/.md', baseURL)).toBe('');
+        expect(extractPagePath('https://docs.example.com/api/notes.md.html', baseURL)).toBe(
+            'notes.md.html'
+        );
+    });
 });
 
 describe('resolveFirstDocument', () => {

packages/gitbook/src/lib/pages.ts

@@ -412,7 +412,8 @@ function flattenPages(
  * Returns the path segment after the base, or undefined if the URL doesn't match.
  */
 export function extractPagePath(url: string, baseURL: string): string | undefined {
-    const urlPath = getURLPathname(url);
+    // Pages are also served as Markdown at `<page>.md`, the form llms.txt links to.
+    const urlPath = getURLPathname(url)?.replace(/\.md$/, '');
     const basePath = getURLPathname(baseURL);
 
     if (urlPath === undefined || basePath === undefined) {

packages/gitbook/src/lib/sites.test.ts

@@ -14,6 +14,7 @@ import { createLinker } from './links';
 import {
     filterSiteSpacesByLocale,
     findSiteSpaceBy,
+    findSiteSpaceByUrl,
     getFallbackSiteSpacePath,
     getLinkerForSiteSpace,
     getSiteStructureSections,
@@ -114,6 +115,30 @@ describe('site structure traversal', () => {
     });
 });
 
+describe('findSiteSpaceByUrl', () => {
+    const docsSpace = {
+        id: 'docs',
+        urls: { published: 'https://example.com/docs' },
+    } as SiteSpace;
+    const v2Space = {
+        id: 'v2',
+        urls: { published: 'https://example.com/docs/v2' },
+    } as SiteSpace;
+
+    it.each([
+        { order: 'broader space first', spaces: [docsSpace, v2Space] },
+        { order: 'nested space first', spaces: [v2Space, docsSpace] },
+    ])('chooses the longest matching published URL with $order', ({ spaces }) => {
+        const match = findSiteSpaceByUrl(
+            { type: 'siteSpaces', structure: spaces },
+            'https://example.com/docs/v2/install'
+        );
+
+        expect(match?.siteSpace).toBe(v2Space);
+        expect(match?.pagePath).toBe('install');
+    });
+});
+
 describe('filterSiteSpacesByLocale', () => {
     it('returns all spaces on a single-language site', () => {
         const spaces = [makeSiteSpace(undefined), makeSiteSpace(undefined)];

packages/gitbook/src/middleware.ts

@@ -169,10 +169,11 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
     }
 
     // Normalize URL after extracting the URL from the request to make sure the client is redirected to the proper one
-    const normalizationResponse = normalizeRequestURL(siteRequestURL);
-    if (normalizationResponse) {
-        return normalizationResponse;
+    const normalization = normalizeRequestURL(siteRequestURL, mode);
+    if (normalization instanceof Response) {
+        return normalization;
     }
+    const isNormalizedURL = normalization.isNormalized;
 
     const imagesContextId = getImageResizingContextId(siteRequestURL);
     /**
@@ -338,6 +339,7 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
         //
         // Make sure the URL is clean of any va token after a successful lookup,
         // and of any visitor.* params that may have been passed to the URL.
+        // Proxied requests that were not normalized are also redirected here, to the canonical URL.
         //
         // We only redirect if the request is not coming from the revalidation worker, as we don't
         // want to redirect in that case. It can carry unsigned claims without any token, so we rely
@@ -348,7 +350,7 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
         //
         const normalizedVisitorURL = normalizeVisitorURL(incomingURL);
         if (
-            normalizedVisitorURL.toString() !== incomingURL.toString() &&
+            (!isNormalizedURL || normalizedVisitorURL.toString() !== incomingURL.toString()) &&
             !isRevalidationRequest(request.headers)
         ) {
             if (visitorToken?.source === 'url') {

packages/gitbook/tests/mcp.test.ts

@@ -55,6 +55,29 @@ it(
     { timeout: 15_000 }
 );
 
+it(
+    'should get a page through MCP from its markdown URL',
+    async () => {
+        const client = await connectMCPClient(
+            getContentTestURL(
+                'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/~gitbook/mcp/auth'
+            )
+        );
+
+        const response = await client.callTool({
+            name: 'getPage',
+            arguments: {
+                url: 'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/3.0/other-page.md',
+            },
+        });
+
+        expect(response.isError).toBeFalsy();
+        // @ts-expect-error - response.content is of type unknown
+        expect(response.content[0]?.text).toContain('# Other Page');
+    },
+    { timeout: 15_000 }
+);
+
 describe('MCP on a site behind visitor authentication', () => {
     const VA_SITE_URL = 'https://gitbook-open-e2e-sites.gitbook.io/va-site-redirects-fallback';