diff --git a/.changeset/thick-dogs-bake.md b/.changeset/thick-dogs-bake.md new file mode 100644 index 000000000..b26fb4884 --- /dev/null +++ b/.changeset/thick-dogs-bake.md @@ -0,0 +1,5 @@ +--- +'gitbook': minor +--- + +Prevent search indexation for pages where it's configured as disabled diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9cf0820bc..21d203291 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,6 +93,12 @@ jobs: env: BASE_URL: ${{needs.deploy.outputs.deployment_url}} ARGOS_TOKEN: ${{ secrets.ARGOS_TOKEN }} + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-test-results + path: packages/gitbook/test-results/ + retention-days: 3 pagespeed-testing: runs-on: ubuntu-latest name: PageSpeed Testing diff --git a/bun.lockb b/bun.lockb index 342942d7d..62159121e 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/packages/gitbook/e2e/pages.spec.ts b/packages/gitbook/e2e/pages.spec.ts index 4f5b981c3..90a7446fc 100644 --- a/packages/gitbook/e2e/pages.spec.ts +++ b/packages/gitbook/e2e/pages.spec.ts @@ -658,6 +658,57 @@ const testCases: TestsCase[] = [ }, })), }, + { + name: 'SEO', + baseUrl: 'https://gitbook.gitbook.io/test-gitbook-open/', + tests: [ + { + name: `Index by default`, + url: '?x-gitbook-search-indexation=true', + screenshot: false, + run: async (page) => { + const metaRobots = page.locator('meta[name="robots"]'); + await expect(metaRobots).toHaveAttribute('content', 'index, follow'); + }, + }, + { + name: `Don't index noIndex`, + url: 'page-options/page-no-index?x-gitbook-search-indexation=true', + screenshot: false, + run: async (page) => { + const metaRobots = page.locator('meta[name="robots"]'); + await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow'); + }, + }, + { + name: `Don't index descendant of noIndex`, + url: 'page-options/page-no-index/descendant-of-page-no-index?x-gitbook-search-indexation=true', + screenshot: false, + run: async (page) => { + const metaRobots = page.locator('meta[name="robots"]'); + await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow'); + }, + }, + { + name: `Don't index noRobotsIndex`, + url: 'page-options/page-no-robots-index?x-gitbook-search-indexation=true', + screenshot: false, + run: async (page) => { + const metaRobots = page.locator('meta[name="robots"]'); + await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow'); + }, + }, + { + name: `Don't index descendant of noRobotsIndex`, + url: 'page-options/page-no-robots-index/descendant-of-page-no-robots-index?x-gitbook-search-indexation=true', + screenshot: false, + run: async (page) => { + const metaRobots = page.locator('meta[name="robots"]'); + await expect(metaRobots).toHaveAttribute('content', 'noindex, nofollow'); + }, + }, + ], + }, ]; for (const testCase of testCases) { diff --git a/packages/gitbook/package.json b/packages/gitbook/package.json index c1b4a1ac1..a594313d0 100644 --- a/packages/gitbook/package.json +++ b/packages/gitbook/package.json @@ -15,7 +15,7 @@ "copy:icons": "gitbook-icons ./public/~gitbook/static/icons" }, "dependencies": { - "@gitbook/api": "^0.60.0", + "@gitbook/api": "0.60.0", "@gitbook/icons": "workspace:*", "@gitbook/react-math": "workspace:*", "@gitbook/react-openapi": "workspace:*", diff --git a/packages/gitbook/src/app/(space)/(content)/[[...pathname]]/page.tsx b/packages/gitbook/src/app/(space)/(content)/[[...pathname]]/page.tsx index 947d84693..b5ab1424e 100644 --- a/packages/gitbook/src/app/(space)/(content)/[[...pathname]]/page.tsx +++ b/packages/gitbook/src/app/(space)/(content)/[[...pathname]]/page.tsx @@ -8,6 +8,7 @@ import { PageBody, PageCover } from '@/components/PageBody'; import { PageHrefContext, absoluteHref, pageHref } from '@/lib/links'; import { getPagePath, resolveFirstDocument } from '@/lib/pages'; import { ContentRefContext } from '@/lib/references'; +import { isSpaceIndexable, isPageIndexable } from '@/lib/seo'; import { tcls } from '@/lib/tailwind'; import { getContentTitle } from '@/lib/utils'; @@ -129,7 +130,7 @@ export async function generateMetadata({ params: PagePathParams; searchParams: { fallback?: string }; }): Promise { - const { space, pages, page, customization, parent } = await getPageDataWithFallback({ + const { space, pages, page, customization, parent, ancestors } = await getPageDataWithFallback({ pagePathParams: params, searchParams, }); @@ -152,6 +153,10 @@ export async function generateMetadata({ absoluteHref(`~gitbook/ogimage/${page.id}`, true), ], }, + robots: + isSpaceIndexable({ space, parent }) && isPageIndexable(ancestors, page) + ? 'index, follow' + : 'noindex, nofollow', }; } diff --git a/packages/gitbook/src/app/(space)/(content)/layout.tsx b/packages/gitbook/src/app/(space)/(content)/layout.tsx index 301f034bc..5114af386 100644 --- a/packages/gitbook/src/app/(space)/(content)/layout.tsx +++ b/packages/gitbook/src/app/(space)/(content)/layout.tsx @@ -13,7 +13,7 @@ import { assetsDomain } from '@/lib/assets'; import { buildVersion } from '@/lib/build'; import { getContentSecurityPolicyNonce } from '@/lib/csp'; import { absoluteHref, baseUrl } from '@/lib/links'; -import { shouldIndexSpace } from '@/lib/seo'; +import { isSpaceIndexable } from '@/lib/seo'; import { getContentTitle } from '@/lib/utils'; import { ClientContexts } from './ClientContexts'; @@ -133,7 +133,7 @@ export async function generateMetadata(): Promise { }, ], }, - robots: shouldIndexSpace({ space, parent }) ? 'index, follow' : 'noindex, nofollow', + robots: isSpaceIndexable({ space, parent }) ? 'index, follow' : 'noindex, nofollow', }; } diff --git a/packages/gitbook/src/app/(space)/(core)/robots.txt/route.ts b/packages/gitbook/src/app/(space)/(core)/robots.txt/route.ts index 4b4e9f6df..dc3d6aa96 100644 --- a/packages/gitbook/src/app/(space)/(core)/robots.txt/route.ts +++ b/packages/gitbook/src/app/(space)/(core)/robots.txt/route.ts @@ -3,7 +3,7 @@ import { NextRequest } from 'next/server'; import { getCollection, getSite, getSpace } from '@/lib/api'; import { absoluteHref } from '@/lib/links'; -import { shouldIndexSpace } from '@/lib/seo'; +import { isSpaceIndexable } from '@/lib/seo'; import { getContentPointer } from '../../fetch'; @@ -28,7 +28,7 @@ export async function GET(req: NextRequest) { const lines = [ `User-agent: *`, 'Disallow: /~gitbook/', - ...(shouldIndexSpace({ space, parent }) + ...(isSpaceIndexable({ space, parent }) ? [`Allow: /`, `Sitemap: ${absoluteHref(`/sitemap.xml`, true)}`] : [`Disallow: /`]), ]; diff --git a/packages/gitbook/src/app/(space)/(core)/sitemap.xml/route.ts b/packages/gitbook/src/app/(space)/(core)/sitemap.xml/route.ts index 398e2af69..f0a23df27 100644 --- a/packages/gitbook/src/app/(space)/(core)/sitemap.xml/route.ts +++ b/packages/gitbook/src/app/(space)/(core)/sitemap.xml/route.ts @@ -5,6 +5,7 @@ import { NextRequest } from 'next/server'; import { getSpaceContentData } from '@/lib/api'; import { absoluteHref } from '@/lib/links'; import { getPagePath } from '@/lib/pages'; +import { isPageIndexable } from '@/lib/seo'; import { getContentPointer } from '../../fetch'; @@ -19,7 +20,8 @@ export async function GET(req: NextRequest) { pointer, 'siteId' in pointer ? pointer.siteShareKey : undefined, ); - const pages = flattenPages(rootPages, (page) => !page.hidden); + + const pages = flattenPages(rootPages, (page) => !page.hidden && isPageIndexable([], page)); const urls = pages.map(({ page, depth }) => { // Decay priority with depth const priority = Math.pow(2, -0.25 * depth); @@ -72,14 +74,19 @@ type FlatPageEntry = { page: RevisionPageDocument; depth: number }; function flattenPages( rootPags: RevisionPage[], - filter: (page: RevisionPageDocument) => boolean, + filter: (page: RevisionPageDocument | RevisionPageGroup) => boolean, ): FlatPageEntry[] { const flattenPage = ( page: RevisionPageDocument | RevisionPageGroup, depth: number, ): FlatPageEntry[] => { + const allowed = filter(page); + if (!allowed) { + return []; + } + return [ - ...(page.type === 'document' && filter(page) ? [{ page, depth }] : []), + ...(page.type === 'document' ? [{ page, depth }] : []), ...page.pages.flatMap((child) => child.type === 'link' ? [] : flattenPage(child, depth + 1), ), diff --git a/packages/gitbook/src/lib/seo.ts b/packages/gitbook/src/lib/seo.ts index 43fdbfb9d..1eb84ebbb 100644 --- a/packages/gitbook/src/lib/seo.ts +++ b/packages/gitbook/src/lib/seo.ts @@ -1,10 +1,37 @@ -import { Collection, ContentVisibility, Site, SiteVisibility, Space } from '@gitbook/api'; +import { + Collection, + ContentVisibility, + RevisionPageDocument, + RevisionPageGroup, + Site, + SiteVisibility, + Space, +} from '@gitbook/api'; import { headers } from 'next/headers'; +/** + * Return true if a page is indexable in search. + */ +export function isPageIndexable( + ancestors: Array, + page: RevisionPageDocument | RevisionPageGroup, +): boolean { + // @ts-ignore - noIndex and noRobotsIndex are not in the type + // until we fix the deprecated APIs + return ( + // @ts-ignore + !page.noIndex && + // @ts-ignore + !page.noRobotsIndex && + // @ts-ignore + ancestors.every((ancestor) => !ancestor.noIndex && !ancestor.noRobotsIndex) + ); +} + /** * Return true if a space should be indexed by search engines. */ -export function shouldIndexSpace({ +export function isSpaceIndexable({ space, parent, }: { diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index aeb368632..9ba3e06ef 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -216,6 +216,15 @@ export async function middleware(request: NextRequest) { } } + // For tests, we make it possible to enable search indexation + // using a query parameter. + const xGitBookSearchIndexation = + headers.get('x-gitbook-search-indexation') ?? + url.searchParams.has('x-gitbook-search-indexation'); + if (xGitBookSearchIndexation) { + headers.set('x-gitbook-search-indexation', 'true'); + } + if (resolved.revision) { headers.set('x-gitbook-content-revision', resolved.revision); } diff --git a/packages/react-contentkit/package.json b/packages/react-contentkit/package.json index 69ea324d2..b2c914c01 100644 --- a/packages/react-contentkit/package.json +++ b/packages/react-contentkit/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "classnames": "^2.5.1", - "@gitbook/api": "^0.58.0", + "@gitbook/api": "0.60.0", "assert-never": "^1.2.1" }, "peerDependencies": {