Add support for resolving missing page paths and redirecting to markdown versions

packages/gitbook/src/components/SitePage/fetch.test.ts

@@ -6,7 +6,8 @@ import type { GitBookSiteContext } from '@/lib/context';
 
 mock.module('server-only', () => ({}));
 
-const { fetchPageData, getLowercasePathnameRedirect } = await import('./fetch');
+const { fetchPageData, getLowercasePathnameRedirect, resolveMissingPagePath } =
+    await import('./fetch');
 const { normalizeURL } = await import('@/lib/data/urls');
 
 const page = {
@@ -94,6 +95,70 @@ describe('fetchPageData', () => {
     });
 });
 
+describe('resolveMissingPagePath', () => {
+    function createRedirectContext(options: {
+        siteRedirect?: { target: string; permanent?: boolean };
+        spaceRedirectPageId?: string;
+    }) {
+        const getSiteRedirectBySource = mock(async ({ source }: { source: string }) =>
+            options.siteRedirect && source === '/old'
+                ? {
+                      data: {
+                          target: options.siteRedirect.target,
+                          redirect: { permanent: options.siteRedirect.permanent ?? false },
+                      },
+                  }
+                : { error: { code: 404, message: 'Not found' } }
+        );
+        const getRevisionPageByPath = mock(async () =>
+            options.spaceRedirectPageId
+                ? { data: { id: options.spaceRedirectPageId } }
+                : { error: { code: 404, message: 'Not found' } }
+        );
+
+        return {
+            organizationId: 'org-1',
+            site: { id: 'site-1' },
+            space: { id: 'space-1', revision: 'revision-1' },
+            revisionId: 'revision-1',
+            revision: { pages: [page] },
+            linker: {
+                toPathInSpace: (path: string) => path,
+                toRelativePathInSite: (path: string) => path,
+                toLinkForContent: (url: string) => new URL(url).pathname,
+            },
+            dataFetcher: { getSiteRedirectBySource, getRevisionPageByPath },
+        } as unknown as GitBookSiteContext;
+    }
+
+    it('resolves a site redirect', async () => {
+        const context = createRedirectContext({
+            siteRedirect: { target: 'https://docs.example.com/new', permanent: true },
+        });
+
+        expect(await resolveMissingPagePath(context, 'old')).toEqual({
+            type: 'redirect',
+            destination: '/new',
+            permanent: true,
+        });
+    });
+
+    it('resolves a space redirect to a page', async () => {
+        const context = createRedirectContext({ spaceRedirectPageId: page.id });
+
+        expect(await resolveMissingPagePath(context, 'old')).toEqual({
+            type: 'page',
+            page: { page, ancestors: [] },
+        });
+    });
+
+    it('returns undefined when nothing matches', async () => {
+        const context = createRedirectContext({});
+
+        expect(await resolveMissingPagePath(context, 'old')).toBeUndefined();
+    });
+});
+
 describe('getLowercasePathnameRedirect', () => {
     it('redirects ASCII paths with uppercase letters', () => {
         expect(getLowercasePathnameRedirect('Foo/Bar')).toBe('foo/bar');

packages/gitbook/src/components/SitePage/fetch.ts

@@ -2,13 +2,14 @@ import { permanentRedirect, redirect } from 'next/navigation';
 
 import {
     CustomizationPageActionType,
+    type RevisionPageDocument,
     SITE_REDIRECT_SOURCE_PATH_MAX_LENGTH,
     SITE_REDIRECT_SOURCE_PATH_PATTERN,
 } from '@gitbook/api';
 
 import type { GitBookSiteContext } from '@/lib/context';
 import { getDataOrNull } from '@/lib/data';
-import { resolvePageId } from '@/lib/pages';
+import { type ResolvedPagePath, resolvePageId } from '@/lib/pages';
 import { withLeadingSlash } from '@/lib/paths';
 import { resolveSiteSpacePagePath } from '@/lib/sites';
 
@@ -70,7 +71,7 @@ export async function fetchPageData(context: GitBookSiteContext, params: PagePar
  * If the path can't be found, we try to resolve it from the API to handle redirects.
  */
 async function resolvePage(context: GitBookSiteContext, params: PagePathParams | PageIdParams) {
-    const { organizationId, site, space, revision, shareKey, linker, revisionId } = context;
+    const { revision } = context;
 
     if ('pageId' in params) {
         return resolvePageId(revision.pages, params.pageId);
@@ -85,72 +86,102 @@ async function resolvePage(context: GitBookSiteContext, params: PagePathParams |
         return page;
     }
 
+    const fallback = await resolveMissingPagePath(context, rawPathname);
+    if (fallback?.type === 'redirect') {
+        return fallback.permanent
+            ? permanentRedirect(fallback.destination)
+            : redirect(fallback.destination);
+    }
+
+    return fallback?.page;
+}
+
+export type MissingPagePathResolution =
+    | {
+          type: 'redirect';
+          /** Destination as returned by `linker.toLinkForContent` (absolute path or URL). */
+          destination: string;
+          permanent: boolean;
+      }
+    | {
+          type: 'page';
+          page: ResolvedPagePath<RevisionPageDocument>;
+      };
+
+/**
+ * Resolve a pathname that doesn't match any page of the revision, using site-level and space-level redirects.
+ */
+export async function resolveMissingPagePath(
+    context: GitBookSiteContext,
+    rawPathname: string
+): Promise<MissingPagePathResolution | undefined> {
+    const { organizationId, site, space, revision, shareKey, linker, revisionId } = context;
+
     // We don't test path that are too long as GitBook doesn't support them and will return a 404 anyway.
     // API has a limit of less than 512 characters for the source path, so we use the same limit here.
-    if (rawPathname.length < SITE_REDIRECT_SOURCE_PATH_MAX_LENGTH) {
-        const SITE_REDIRECT_SOURCE_PATH_REGEX = new RegExp(SITE_REDIRECT_SOURCE_PATH_PATTERN);
-        const redirectPathname = withLeadingSlash(rawPathname);
-        // If a page can't be found, we try with the API, in case we have a redirect at site level.
-        const redirectSources = new Set(
-            [
-                // Test the pathname relative to the root
-                // For example hello/world -> section/variant/hello/world
-                linker.toRelativePathInSite(linker.toPathInSpace(redirectPathname)),
-                // Test the pathname relative to the content/space
-                // For example hello/world -> /hello/world
-                redirectPathname,
-            ]
-                .map(toSiteRedirectSourceCandidate)
-                .filter((source) => SITE_REDIRECT_SOURCE_PATH_REGEX.test(source))
-        );
+    if (rawPathname.length >= SITE_REDIRECT_SOURCE_PATH_MAX_LENGTH) {
+        return undefined;
+    }
 
-        for (const source of redirectSources) {
-            // We try to resolve the site redirect
-            const resolvedSiteRedirect =
-                source.length < SITE_REDIRECT_SOURCE_PATH_MAX_LENGTH &&
-                (await getDataOrNull(
-                    context.dataFetcher.getSiteRedirectBySource({
-                        organizationId,
-                        siteId: site.id,
-                        source,
-                        siteShareKey: shareKey,
-                    })
-                ));
-            if (resolvedSiteRedirect) {
-                const destination = linker.toLinkForContent(resolvedSiteRedirect.target);
-                const isPublicLiveContext =
-                    !shareKey &&
-                    !context.changeRequest &&
-                    !context.preview &&
-                    context.revisionId === context.space.revision &&
-                    !context.isLoggedInVisitor;
-                if (
+    const SITE_REDIRECT_SOURCE_PATH_REGEX = new RegExp(SITE_REDIRECT_SOURCE_PATH_PATTERN);
+    const redirectPathname = withLeadingSlash(rawPathname);
+    // If a page can't be found, we try with the API, in case we have a redirect at site level.
+    const redirectSources = new Set(
+        [
+            // Test the pathname relative to the root
+            // For example hello/world -> section/variant/hello/world
+            linker.toRelativePathInSite(linker.toPathInSpace(redirectPathname)),
+            // Test the pathname relative to the content/space
+            // For example hello/world -> /hello/world
+            redirectPathname,
+        ]
+            .map(toSiteRedirectSourceCandidate)
+            .filter((source) => SITE_REDIRECT_SOURCE_PATH_REGEX.test(source))
+    );
+
+    for (const source of redirectSources) {
+        // We try to resolve the site redirect
+        const resolvedSiteRedirect =
+            source.length < SITE_REDIRECT_SOURCE_PATH_MAX_LENGTH &&
+            (await getDataOrNull(
+                context.dataFetcher.getSiteRedirectBySource({
+                    organizationId,
+                    siteId: site.id,
+                    source,
+                    siteShareKey: shareKey,
+                })
+            ));
+        if (resolvedSiteRedirect) {
+            const isPublicLiveContext =
+                !shareKey &&
+                !context.changeRequest &&
+                !context.preview &&
+                context.revisionId === context.space.revision &&
+                !context.isLoggedInVisitor;
+            return {
+                type: 'redirect',
+                destination: linker.toLinkForContent(resolvedSiteRedirect.target),
+                permanent: Boolean(
                     resolvedSiteRedirect.redirect?.permanent &&
                     !resolvedSiteRedirect.redirect.draft &&
                     isPublicLiveContext
-                ) {
-                    return permanentRedirect(destination);
-                }
-                return redirect(destination);
-            }
-        }
-
-        // If page still can't be found, we try with the API, in case we have a redirect at space level.
-        // We use the raw pathname to handle special/malformed redirects setup by users in the GitSync.
-        // The page rendering will take care of redirecting to a normalized pathname.
-        const resolved = await getDataOrNull(
-            context.dataFetcher.getRevisionPageByPath({
-                spaceId: space.id,
-                revisionId: revisionId,
-                path: rawPathname,
-            })
-        );
-        if (resolved) {
-            return resolvePageId(revision.pages, resolved.id);
+                ),
+            };
         }
     }
 
-    return undefined;
+    // If page still can't be found, we try with the API, in case we have a redirect at space level.
+    // We use the raw pathname to handle special/malformed redirects setup by users in the GitSync.
+    // The page rendering will take care of redirecting to a normalized pathname.
+    const resolved = await getDataOrNull(
+        context.dataFetcher.getRevisionPageByPath({
+            spaceId: space.id,
+            revisionId: revisionId,
+            path: rawPathname,
+        })
+    );
+    const page = resolved ? resolvePageId(revision.pages, resolved.id) : undefined;
+    return page ? { type: 'page', page } : undefined;
 }
 
 /**

packages/gitbook/src/routes/markdownPage.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it, mock } from 'bun:test';
+
+import type { RevisionPageDocument } from '@gitbook/api';
+
+import type { GitBookSiteContext } from '@/lib/context';
+import { createLinker } from '@/lib/links';
+
+mock.module('server-only', () => ({}));
+
+const { servePageMarkdown, toMarkdownDestination } = await import('./markdownPage');
+
+const page = {
+    id: 'page-1',
+    title: 'New page',
+    kind: 'sheet',
+    type: 'document',
+    path: 'new-page',
+    slug: 'new-page',
+    pages: [],
+} as unknown as RevisionPageDocument;
+
+function createContext(options: {
+    siteRedirect?: { target: string; permanent?: boolean };
+    spaceRedirectPageId?: string;
+}) {
+    return {
+        organizationId: 'org-1',
+        site: { id: 'site-1' },
+        siteSpace: { id: 'site-space-1' },
+        space: { id: 'space-1', revision: 'revision-1' },
+        revisionId: 'revision-1',
+        revision: { pages: [page] },
+        linker: createLinker({
+            host: 'docs.example.com',
+            siteBasePath: '/docs/',
+            spaceBasePath: '/docs/',
+        }),
+        dataFetcher: {
+            getSiteRedirectBySource: async ({ source }: { source: string }) =>
+                options.siteRedirect && source === '/old-page'
+                    ? {
+                          data: {
+                              target: options.siteRedirect.target,
+                              redirect: { permanent: options.siteRedirect.permanent ?? false },
+                          },
+                      }
+                    : { error: { code: 404, message: 'Not found' } },
+            getRevisionPageByPath: async () =>
+                options.spaceRedirectPageId
+                    ? { data: { id: options.spaceRedirectPageId } }
+                    : { error: { code: 404, message: 'Not found' } },
+        },
+    } as unknown as GitBookSiteContext;
+}
+
+describe('servePageMarkdown', () => {
+    it('redirects to the markdown version of a site redirect target', async () => {
+        const context = createContext({
+            siteRedirect: { target: 'https://docs.example.com/docs/new-page', permanent: true },
+        });
+
+        const response = await servePageMarkdown(context, 'old-page');
+
+        expect(response.status).toBe(308);
+        expect(response.headers.get('Location')).toBe('/docs/new-page.md');
+    });
+
+    it('redirects to the markdown version of a space redirect target', async () => {
+        const context = createContext({ spaceRedirectPageId: page.id });
+
+        const response = await servePageMarkdown(context, 'old-page');
+
+        expect(response.status).toBe(307);
+        expect(response.headers.get('Location')).toBe('/docs/new-page.md');
+    });
+});
+
+describe('toMarkdownDestination', () => {
+    it('appends .md to same-site paths', () => {
+        expect(toMarkdownDestination('/docs/new-page')).toBe('/docs/new-page.md');
+        expect(toMarkdownDestination('/docs/new-page/?a=1#b')).toBe('/docs/new-page.md?a=1#b');
+    });
+
+    it('leaves markdown paths and external URLs untouched', () => {
+        expect(toMarkdownDestination('/docs/new-page.md')).toBe('/docs/new-page.md');
+        expect(toMarkdownDestination('https://example.com/page')).toBe('https://example.com/page');
+    });
+});

packages/gitbook/src/routes/markdownPage.ts

@@ -1,5 +1,6 @@
 import type { RevisionPageDocument, RevisionPageGroup } from '@gitbook/api';
 
+import { resolveMissingPagePath } from '@/components/SitePage/fetch';
 import { isAIEnabled } from '@/components/utils/isAIChatEnabled';
 import type { GitBookSiteContext } from '@/lib/context';
 import { getExposableError } from '@/lib/data';
@@ -22,12 +23,36 @@ export async function servePageMarkdown(baseContext: GitBookSiteContext, pagePat
             linker: linkerWithMarkdownPages(baseContext.linker),
         };
 
-        const pageLookup = resolveSiteSpacePagePathDocumentOrGroup(
-            context.siteSpace,
-            context.revision.pages,
-            pagePath
-        );
+        const pageLookup =
+            resolveSiteSpacePagePathDocumentOrGroup(
+                context.siteSpace,
+                context.revision.pages,
+                pagePath
+            ) ??
+            // Page paths are lowercase, match the case-insensitive lookup of HTML pages.
+            resolveSiteSpacePagePathDocumentOrGroup(
+                context.siteSpace,
+                context.revision.pages,
+                pagePath.toLowerCase()
+            );
         if (!pageLookup) {
+            const fallback = await resolveMissingPagePath(baseContext, pagePath);
+            if (fallback?.type === 'redirect') {
+                return markdownRedirect(
+                    toMarkdownDestination(fallback.destination),
+                    fallback.permanent
+                );
+            }
+            if (fallback?.type === 'page') {
+                return markdownRedirect(
+                    context.linker.toPathForPage({
+                        pages: context.revision.pages,
+                        page: fallback.page.page,
+                    }),
+                    false
+                );
+            }
+
             // Generates a markdown body for missing pages. Return this with a 200 status (not 404) because agents discard 404 response bodies.=
             return {
                 markdown: renderNotFoundMarkdown(context, pagePath),
@@ -63,6 +88,32 @@ function getMarkdownRobots(
     return context.isAiAgent ? 'index, follow' : 'noindex';
 }
 
+/**
+ * Point a redirect destination to its markdown version, so agents keep receiving markdown.
+ * Destinations outside the site are returned as full URLs and left untouched.
+ */
+export function toMarkdownDestination(destination: string): string {
+    if (!destination.startsWith('/')) {
+        return destination;
+    }
+
+    const url = new URL(destination, 'https://gitbook.invalid');
+    const pathname = url.pathname.replace(/\/+$/, '');
+    if (!pathname || pathname.endsWith('.md')) {
+        return destination;
+    }
+
+    return `${pathname}.md${url.search}${url.hash}`;
+}
+
+function markdownRedirect(location: string, permanent: boolean) {
+    // Same status codes as Next's `redirect` / `permanentRedirect`.
+    return new Response(null, {
+        status: permanent ? 308 : 307,
+        headers: { Location: location, Vary: 'Accept' },
+    });
+}
+
 function renderNotFoundMarkdown(context: GitBookSiteContext, pagePath: string) {
     const similarPages = getSimilarPages(context.revision.pages, pagePath, 5);
     const sitemapUrl = context.linker.toAbsoluteURL(context.linker.toPathInSite('sitemap.md'));
@@ -164,11 +215,14 @@ Use this mechanism when the answer is not explicitly present in the current page
  * Return a markdown content.
  */
 export async function serveMarkdown(
-    fn: () => Promise<string | { markdown: string; robots: string }>,
+    fn: () => Promise<string | { markdown: string; robots: string } | Response>,
     isChatGPT?: boolean
 ) {
     try {
         const result = await fn();
+        if (result instanceof Response) {
+            return result;
+        }
         const { markdown, robots } =
             typeof result === 'string' ? { markdown: result, robots: 'noindex' } : result;
         return new Response(markdown, {