mirror of
https://github.com/GitbookIO/gitbook.git
synced 2026-11-21 15:52:36 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ca21ff8c6c | |||
| 11eea68e37 | |||
| ed9b5b9c15 |
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"gitbook": patch
|
||||
---
|
||||
|
||||
Keep published revision URLs out of search results.
|
||||
@@ -76,3 +76,7 @@ export function getExtension(path: string): string {
|
||||
export function isGitBookInternalPath(path: string): boolean {
|
||||
return /(^|\/)~gitbook(\/|$)/.test(path);
|
||||
}
|
||||
|
||||
export function hasRevisionSelector(path: string): boolean {
|
||||
return /(?:^|\/)~\/revisions\/[^/]+/.test(path);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { type RevisionPageDocument, type RevisionPageGroup, SiteVisibility } from '@gitbook/api';
|
||||
|
||||
import type { GitBookSiteContext } from '@/lib/context';
|
||||
import { hasRevisionSelector } from '@/lib/paths';
|
||||
|
||||
/**
|
||||
* Return true if a page is indexable in search.
|
||||
@@ -29,8 +30,11 @@ export function isSiteIndexable(context: GitBookSiteContext) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Prevent indexation of preview of revisions / change-requests
|
||||
if (context.changeRequest || context.revisionId !== context.space.revision) {
|
||||
if (
|
||||
context.changeRequest ||
|
||||
context.revisionId !== context.space.revision ||
|
||||
hasRevisionSelector(context.linker.spaceBasePath)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ import {
|
||||
isOAuthProtectedResourceMetadataRequestForAuthEndpoint,
|
||||
isOAuthProtectedResourceRequest,
|
||||
} from '@/lib/oauth-protected';
|
||||
import { removeLeadingSlash, removeTrailingSlash } from '@/lib/paths';
|
||||
import { hasRevisionSelector, removeLeadingSlash, removeTrailingSlash } from '@/lib/paths';
|
||||
import {
|
||||
getPreviewCookieResponse,
|
||||
getPreviewRequestIdentifier,
|
||||
@@ -85,6 +85,8 @@ export const config = {
|
||||
|
||||
type URLWithMode = { url: URL; mode: 'url' | 'url-host' };
|
||||
|
||||
const REVISION_ROBOTS_TAG = 'noindex, nofollow';
|
||||
|
||||
export async function middleware(request: NextRequest) {
|
||||
try {
|
||||
const requestURL = new URL(request.url);
|
||||
@@ -168,6 +170,20 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const response = await serveSiteRequest(requestURL, request, match);
|
||||
if (
|
||||
response &&
|
||||
hasRevisionSelector(match.url.pathname) &&
|
||||
!response.headers.has('x-robots-tag')
|
||||
) {
|
||||
const revisionResponse = new Response(response.body, response);
|
||||
revisionResponse.headers.set('x-robots-tag', REVISION_ROBOTS_TAG);
|
||||
return revisionResponse;
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function serveSiteRequest(requestURL: URL, request: NextRequest, match: URLWithMode) {
|
||||
const { url: siteRequestURL, mode } = match;
|
||||
|
||||
if (isAITrainingOrIndexingRequest(request)) {
|
||||
@@ -180,6 +196,13 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
|
||||
// Normalize URL after extracting the URL from the request to make sure the client is redirected to the proper one
|
||||
const normalization = normalizeRequestURL(siteRequestURL, mode);
|
||||
if (normalization instanceof Response) {
|
||||
const location = normalization.headers.get('location');
|
||||
if (location && hasRevisionSelector(new URL(location).pathname)) {
|
||||
return NextResponse.redirect(location, {
|
||||
status: normalization.status,
|
||||
headers: { 'x-robots-tag': REVISION_ROBOTS_TAG },
|
||||
});
|
||||
}
|
||||
return normalization;
|
||||
}
|
||||
const isNormalizedURL = normalization.isNormalized;
|
||||
@@ -622,6 +645,8 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
|
||||
// noindex search/assistant deep links, kept crawlable so Google sees the directive.
|
||||
if (rewrittenURL.searchParams.has('ask') || rewrittenURL.searchParams.has('q')) {
|
||||
response.headers.set('x-robots-tag', 'none');
|
||||
} else if (hasRevisionSelector(siteURLData.basePath)) {
|
||||
response.headers.set('x-robots-tag', REVISION_ROBOTS_TAG);
|
||||
}
|
||||
|
||||
// Allow cross-origin requests from the same parent domain as the site.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, mock } from 'bun:test';
|
||||
|
||||
import type { RevisionPageDocument } from '@gitbook/api';
|
||||
import { type RevisionPageDocument, SiteVisibility } from '@gitbook/api';
|
||||
|
||||
import type { GitBookSiteContext } from '@/lib/context';
|
||||
import { createLinker } from '@/lib/links';
|
||||
@@ -26,7 +26,7 @@ function createContext(options: {
|
||||
return {
|
||||
displayAgentInstructions: false,
|
||||
organizationId: 'org-1',
|
||||
site: { id: 'site-1' },
|
||||
site: { id: 'site-1', visibility: SiteVisibility.Public },
|
||||
siteSpace: { id: 'site-space-1' },
|
||||
space: { id: 'space-1', revision: 'revision-1' },
|
||||
revisionId: 'revision-1',
|
||||
@@ -66,6 +66,23 @@ describe('servePageMarkdown', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('serves the current revision by selector with a noindex header for AI agents', async () => {
|
||||
const context = createContext({});
|
||||
const response = await servePageMarkdown(
|
||||
{
|
||||
...context,
|
||||
isAiAgent: true,
|
||||
linker: context.linker.fork({
|
||||
spaceBasePath: `/docs/~/revisions/${context.revisionId}/`,
|
||||
}),
|
||||
},
|
||||
'new-page'
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('X-Robots-Tag')).toBe('noindex, nofollow');
|
||||
});
|
||||
|
||||
it('redirects to the markdown version of a site redirect target', async () => {
|
||||
const context = createContext({
|
||||
siteRedirect: { target: 'https://docs.example.com/docs/new-page', permanent: true },
|
||||
|
||||
Reference in New Issue
Block a user