Prevent robots indexation for pages where it's disabled (#2464)

.changeset/thick-dogs-bake.md

@@ -0,0 +1,5 @@
+---
+'gitbook': minor
+---
+
+Prevent search indexation for pages where it's configured as disabled

.github/workflows/ci.yaml

@@ -93,6 +93,12 @@ jobs:
               env:
                   BASE_URL: ${{needs.deploy.outputs.deployment_url}}
                   ARGOS_TOKEN: ${{ secrets.ARGOS_TOKEN }}
+            - uses: actions/upload-artifact@v4
+              if: ${{ !cancelled() }}
+              with:
+                  name: playwright-test-results
+                  path: packages/gitbook/test-results/
+                  retention-days: 3
     pagespeed-testing:
         runs-on: ubuntu-latest
         name: PageSpeed Testing

packages/gitbook/e2e/pages.spec.ts

@@ -658,6 +658,57 @@ const testCases: TestsCase[] = [
             },
         })),
     },
+    {
+        name: 'SEO',
+        baseUrl: 'https://gitbook.gitbook.io/test-gitbook-open/',
+        tests: [
+            {
+                name: `Index by default`,
+                url: '?x-gitbook-search-indexation=true',
+                screenshot: false,
+                run: async (page) => {
+                    const metaRobots = page.locator('meta[name="robots"]');
+                    await expect(metaRobots).toHaveAttribute('content', 'index, follow');
+                },
+            },
+            {
+                name: `Don't index noIndex`,
+                url: 'page-options/page-no-index?x-gitbook-search-indexation=true',
+                screenshot: false,
+                run: async (page) => {
+                    const metaRobots = page.locator('meta[name="robots"]');
+                    await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow');
+                },
+            },
+            {
+                name: `Don't index descendant of noIndex`,
+                url: 'page-options/page-no-index/descendant-of-page-no-index?x-gitbook-search-indexation=true',
+                screenshot: false,
+                run: async (page) => {
+                    const metaRobots = page.locator('meta[name="robots"]');
+                    await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow');
+                },
+            },
+            {
+                name: `Don't index noRobotsIndex`,
+                url: 'page-options/page-no-robots-index?x-gitbook-search-indexation=true',
+                screenshot: false,
+                run: async (page) => {
+                    const metaRobots = page.locator('meta[name="robots"]');
+                    await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow');
+                },
+            },
+            {
+                name: `Don't index descendant of noRobotsIndex`,
+                url: 'page-options/page-no-robots-index/descendant-of-page-no-robots-index?x-gitbook-search-indexation=true',
+                screenshot: false,
+                run: async (page) => {
+                    const metaRobots = page.locator('meta[name="robots"]');
+                    await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow');
+                },
+            },
+        ],
+    },
 ];
 
 for (const testCase of testCases) {

packages/gitbook/package.json

@@ -15,7 +15,7 @@
         "copy:icons": "gitbook-icons ./public/~gitbook/static/icons"
     },
     "dependencies": {
-        "@gitbook/api": "^0.60.0",
+        "@gitbook/api": "0.60.0",
         "@gitbook/icons": "workspace:*",
         "@gitbook/react-math": "workspace:*",
         "@gitbook/react-openapi": "workspace:*",

packages/gitbook/src/app/(space)/(content)/[[...pathname]]/page.tsx

@@ -8,6 +8,7 @@ import { PageBody, PageCover } from '@/components/PageBody';
 import { PageHrefContext, absoluteHref, pageHref } from '@/lib/links';
 import { getPagePath, resolveFirstDocument } from '@/lib/pages';
 import { ContentRefContext } from '@/lib/references';
+import { isSpaceIndexable, isPageIndexable } from '@/lib/seo';
 import { tcls } from '@/lib/tailwind';
 import { getContentTitle } from '@/lib/utils';
 
@@ -129,7 +130,7 @@ export async function generateMetadata({
     params: PagePathParams;
     searchParams: { fallback?: string };
 }): Promise<Metadata> {
-    const { space, pages, page, customization, parent } = await getPageDataWithFallback({
+    const { space, pages, page, customization, parent, ancestors } = await getPageDataWithFallback({
         pagePathParams: params,
         searchParams,
     });
@@ -152,6 +153,10 @@ export async function generateMetadata({
                     absoluteHref(`~gitbook/ogimage/${page.id}`, true),
             ],
         },
+        robots:
+            isSpaceIndexable({ space, parent }) && isPageIndexable(ancestors, page)
+                ? 'index, follow'
+                : 'noindex, nofollow',
     };
 }
 

packages/gitbook/src/app/(space)/(content)/layout.tsx

@@ -13,7 +13,7 @@ import { assetsDomain } from '@/lib/assets';
 import { buildVersion } from '@/lib/build';
 import { getContentSecurityPolicyNonce } from '@/lib/csp';
 import { absoluteHref, baseUrl } from '@/lib/links';
-import { shouldIndexSpace } from '@/lib/seo';
+import { isSpaceIndexable } from '@/lib/seo';
 import { getContentTitle } from '@/lib/utils';
 
 import { ClientContexts } from './ClientContexts';
@@ -133,7 +133,7 @@ export async function generateMetadata(): Promise<Metadata> {
                 },
             ],
         },
-        robots: shouldIndexSpace({ space, parent }) ? 'index, follow' : 'noindex, nofollow',
+        robots: isSpaceIndexable({ space, parent }) ? 'index, follow' : 'noindex, nofollow',
     };
 }
 

packages/gitbook/src/app/(space)/(core)/robots.txt/route.ts

@@ -3,7 +3,7 @@ import { NextRequest } from 'next/server';
 
 import { getCollection, getSite, getSpace } from '@/lib/api';
 import { absoluteHref } from '@/lib/links';
-import { shouldIndexSpace } from '@/lib/seo';
+import { isSpaceIndexable } from '@/lib/seo';
 
 import { getContentPointer } from '../../fetch';
 
@@ -28,7 +28,7 @@ export async function GET(req: NextRequest) {
     const lines = [
         `User-agent: *`,
         'Disallow: /~gitbook/',
-        ...(shouldIndexSpace({ space, parent })
+        ...(isSpaceIndexable({ space, parent })
             ? [`Allow: /`, `Sitemap: ${absoluteHref(`/sitemap.xml`, true)}`]
             : [`Disallow: /`]),
     ];

packages/gitbook/src/app/(space)/(core)/sitemap.xml/route.ts

@@ -5,6 +5,7 @@ import { NextRequest } from 'next/server';
 import { getSpaceContentData } from '@/lib/api';
 import { absoluteHref } from '@/lib/links';
 import { getPagePath } from '@/lib/pages';
+import { isPageIndexable } from '@/lib/seo';
 
 import { getContentPointer } from '../../fetch';
 
@@ -19,7 +20,8 @@ export async function GET(req: NextRequest) {
         pointer,
         'siteId' in pointer ? pointer.siteShareKey : undefined,
     );
-    const pages = flattenPages(rootPages, (page) => !page.hidden);
+
+    const pages = flattenPages(rootPages, (page) => !page.hidden && isPageIndexable([], page));
     const urls = pages.map(({ page, depth }) => {
         // Decay priority with depth
         const priority = Math.pow(2, -0.25 * depth);
@@ -72,14 +74,19 @@ type FlatPageEntry = { page: RevisionPageDocument; depth: number };
 
 function flattenPages(
     rootPags: RevisionPage[],
-    filter: (page: RevisionPageDocument) => boolean,
+    filter: (page: RevisionPageDocument | RevisionPageGroup) => boolean,
 ): FlatPageEntry[] {
     const flattenPage = (
         page: RevisionPageDocument | RevisionPageGroup,
         depth: number,
     ): FlatPageEntry[] => {
+        const allowed = filter(page);
+        if (!allowed) {
+            return [];
+        }
+
         return [
-            ...(page.type === 'document' && filter(page) ? [{ page, depth }] : []),
+            ...(page.type === 'document' ? [{ page, depth }] : []),
             ...page.pages.flatMap((child) =>
                 child.type === 'link' ? [] : flattenPage(child, depth + 1),
             ),

packages/gitbook/src/lib/seo.ts

@@ -1,10 +1,37 @@
-import { Collection, ContentVisibility, Site, SiteVisibility, Space } from '@gitbook/api';
+import {
+    Collection,
+    ContentVisibility,
+    RevisionPageDocument,
+    RevisionPageGroup,
+    Site,
+    SiteVisibility,
+    Space,
+} from '@gitbook/api';
 import { headers } from 'next/headers';
 
+/**
+ * Return true if a page is indexable in search.
+ */
+export function isPageIndexable(
+    ancestors: Array<RevisionPageDocument | RevisionPageGroup>,
+    page: RevisionPageDocument | RevisionPageGroup,
+): boolean {
+    // @ts-ignore - noIndex and noRobotsIndex are not in the type
+    // until we fix the deprecated APIs
+    return (
+        // @ts-ignore
+        !page.noIndex &&
+        // @ts-ignore
+        !page.noRobotsIndex &&
+        // @ts-ignore
+        ancestors.every((ancestor) => !ancestor.noIndex && !ancestor.noRobotsIndex)
+    );
+}
+
 /**
  * Return true if a space should be indexed by search engines.
  */
-export function shouldIndexSpace({
+export function isSpaceIndexable({
     space,
     parent,
 }: {

packages/gitbook/src/middleware.ts

@@ -216,6 +216,15 @@ export async function middleware(request: NextRequest) {
         }
     }
 
+    // For tests, we make it possible to enable search indexation
+    // using a query parameter.
+    const xGitBookSearchIndexation =
+        headers.get('x-gitbook-search-indexation') ??
+        url.searchParams.has('x-gitbook-search-indexation');
+    if (xGitBookSearchIndexation) {
+        headers.set('x-gitbook-search-indexation', 'true');
+    }
+
     if (resolved.revision) {
         headers.set('x-gitbook-content-revision', resolved.revision);
     }

packages/react-contentkit/package.json

@@ -10,7 +10,7 @@
     },
     "dependencies": {
         "classnames": "^2.5.1",
-        "@gitbook/api": "^0.58.0",
+        "@gitbook/api": "0.60.0",
         "assert-never": "^1.2.1"
     },
     "peerDependencies": {