Load integration scripts and secure the app with CSP (#39)

* Set colorSchema metadata

* Try CSP

* Move colorScheme to generateViewport

* Bun install

* Use wildcard for img-src and connect-src

* Allow all unsafe-inline for now for style

* Remove nonce for style

* Remove nonce for style

* Optimize data fetching with parallel requests

* Try to reuse memory cache between middleware and rendering in CF

* Comment

* Fetch document in parallel of collections

* Cleanup

* Log if redis is enabled for caching

* Fix memory cache before redis one

* Optimize alternatives being tested
This commit is contained in:
Samy Pessé
2023-12-17 00:25:07 +01:00
committed by GitHub
parent 046e74b4a8
commit afcb0ecba6
10 changed files with 321 additions and 119 deletions
BIN
View File
Binary file not shown.
+1
View File
@@ -23,6 +23,7 @@
"ajv": "^8.12.0",
"assert-never": "^1.2.1",
"bun-types": "^1.0.7",
"content-security-policy-merger": "^1.0.0",
"framer-motion": "^10.16.14",
"js-cookie": "^3.0.5",
"jsontoxml": "^1.0.1",
+18 -3
View File
@@ -1,12 +1,15 @@
import { CustomizationHeaderPreset, CustomizationSettings } from '@gitbook/api';
import assertNever from 'assert-never';
import Script from 'next/script';
import colors from 'tailwindcss/colors';
import { fonts } from '@/fonts';
import { getSpaceContent } from '@/lib/api';
import { hexToRgb, shadesOfColor } from '@/lib/colors';
import { getContentSecurityPolicyNonce } from '@/lib/csp';
import { tcls } from '@/lib/tailwind';
import { PagePathParams, fetchPageData } from '../fetch';
import { PagePathParams } from '../fetch';
export default async function SpaceRootLayout(props: {
children: React.ReactNode;
@@ -14,13 +17,21 @@ export default async function SpaceRootLayout(props: {
}) {
const { params, children } = props;
const { customization } = await fetchPageData(params);
const { customization, scripts } = await getSpaceContent({
spaceId: params.spaceId,
});
const headerTheme = generateHeaderTheme(customization);
const nonce = getContentSecurityPolicyNonce();
return (
<html lang={customization.internationalization.locale}>
<head>
<style>{`
<style
nonce={
//Since I can't get the nonce to work for inline styles, we need to allow unsafe-inline
undefined
}
>{`
:root {
${generateColorVariable(
'primary-color',
@@ -56,6 +67,10 @@ export default async function SpaceRootLayout(props: {
)}
>
{children}
{scripts.map(({ script }) => (
<Script key={script} src={script} strategy="lazyOnload" nonce={nonce} />
))}
</body>
</html>
);
+24 -5
View File
@@ -1,9 +1,9 @@
import { Metadata } from 'next';
import { CustomizationThemeMode } from '@gitbook/api';
import { Metadata, Viewport } from 'next';
import { notFound, redirect } from 'next/navigation';
import { SpaceContent } from '@/components/SpaceContent';
import { getSpaceLanguage } from '@/intl/server';
import { getDocument } from '@/lib/api';
import { PageHrefContext, absoluteHref, baseUrl, pageHref } from '@/lib/links';
import { getPagePath } from '@/lib/pages';
@@ -18,8 +18,17 @@ export const runtime = 'edge';
export default async function Page(props: { params: PagePathParams }) {
const { params } = props;
const { content, space, customization, pages, page, collection, collectionSpaces, ancestors } =
await fetchPageData(params);
const {
content,
space,
customization,
pages,
page,
collection,
collectionSpaces,
ancestors,
document,
} = await fetchPageData(params);
const linksContext: PageHrefContext = {};
if (!page) {
@@ -28,7 +37,6 @@ export default async function Page(props: { params: PagePathParams }) {
redirect(pageHref(pages, page, linksContext));
}
const document = page.documentId ? await getDocument(space.id, page.documentId) : null;
const language = getSpaceLanguage(customization);
return (
@@ -48,6 +56,17 @@ export default async function Page(props: { params: PagePathParams }) {
);
}
export async function generateViewport({ params }: { params: PagePathParams }): Promise<Viewport> {
const { customization } = await fetchPageData(params);
return {
colorScheme: customization.themes.toggeable
? customization.themes.default === CustomizationThemeMode.Dark
? 'dark light'
: 'light dark'
: customization.themes.default,
};
}
export async function generateMetadata({ params }: { params: PagePathParams }): Promise<Metadata> {
const { space, page, customization } = await fetchPageData(params);
if (!page) {
+11 -11
View File
@@ -1,12 +1,11 @@
import { ContentVisibility, Space } from '@gitbook/api';
import {
getSpace,
getSpaceCustomization,
getCollectionSpaces,
getCollection,
ContentPointer,
getRevisionPages,
getSpaceContent,
getDocument,
} from '@/lib/api';
import { resolvePagePath, resolvePageId } from '@/lib/pages';
@@ -22,36 +21,37 @@ export interface PageIdParams extends SpaceParams {
/**
* Fetch all the data needed to render the content.
* Optimized to fetch in parallel as much as possible.
*/
export async function fetchPageData(params: PagePathParams | PageIdParams) {
const { spaceId } = params;
const content: ContentPointer = {
spaceId: params.spaceId,
changeRequestId: params.changeRequestId,
revisionId: params.revisionId,
};
const [space, pages, customization] = await Promise.all([
getSpace(spaceId),
getRevisionPages(content),
getSpaceCustomization(spaceId),
]);
const { space, pages, customization, scripts } = await getSpaceContent(content);
const collection = await fetchParentCollection(space);
const page =
'pageId' in params && params.pageId
? resolvePageId(pages, params.pageId)
: resolvePagePath(pages, getPathnameParam(params));
const [collection, document] = await Promise.all([
fetchParentCollection(space),
page && page.page.documentId ? await getDocument(space.id, page.page.documentId) : null,
]);
return {
content,
space,
pages,
customization,
scripts,
ancestors: [],
...page,
...collection,
document,
};
}
+55 -8
View File
@@ -1,11 +1,11 @@
import 'server-only';
import { AsyncLocalStorage } from 'node:async_hooks';
import {
ContentVisibility,
GitBookAPI,
GitBookAPIError,
PublishedContentLookup,
Space,
} from '@gitbook/api';
import assertNever from 'assert-never';
import { headers } from 'next/headers';
@@ -18,12 +18,18 @@ export interface ContentPointer {
revisionId?: string;
}
const apiSyncStorage = new AsyncLocalStorage<GitBookAPI>();
/**
* Create an API client for the current request.
*/
export function api(): GitBookAPI {
const headersList = headers();
const existing = apiSyncStorage.getStore();
if (existing) {
return existing;
}
const headersList = headers();
const apiEndpoint = headersList.get('x-gitbook-api') ?? undefined;
const apiToken = headersList.get('x-gitbook-token');
@@ -41,6 +47,13 @@ export function api(): GitBookAPI {
return gitbook;
}
/**
* Use an API client for an async function.
*/
export function withAPI<T>(client: GitBookAPI, fn: () => Promise<T>): Promise<T> {
return apiSyncStorage.run(client, fn);
}
export type PublishedContentWithCache = PublishedContentLookup & {
cacheMaxAge?: number;
cacheTags?: string[];
@@ -53,7 +66,6 @@ export const getPublishedContentByUrl = cache(
'api.getPublishedContentByUrl',
async (
url: string,
apiEndpoint: string | undefined,
visitorAuthToken: string | undefined,
options: {
signal?: AbortSignal;
@@ -65,11 +77,7 @@ export const getPublishedContentByUrl = cache(
// We call it as this logic is wrapped in an asynchronous cache that is not tied to the signal.
signal?.throwIfAborted();
const gitbook = new GitBookAPI({
endpoint: apiEndpoint,
});
const response = await gitbook.request<PublishedContentLookup>({
const response = await api().request<PublishedContentLookup>({
method: 'GET',
path: '/urls/published',
query: {
@@ -123,6 +131,24 @@ export const getSpace = cache('api.getSpace', async (spaceId: string) => {
});
});
/**
* List the scripts to load for the space.
*/
export const getSpaceIntegrationScripts = cache(
'api.getSpaceIntegrationScripts',
async (spaceId: string) => {
const response = await api().spaces.listSpaceIntegrationScripts(spaceId, {
...noCacheFetchOptions,
});
return cacheResponse(response, {
tags: [
getAPICacheTag({ tag: 'space', space: spaceId }),
getAPICacheTag({ tag: 'space-customization', space: spaceId }),
],
});
},
);
/**
* Get all the pages in the space.
*/
@@ -287,6 +313,27 @@ export const getCollectionSpaces = cache(
},
);
/**
* Fetch all the information about a space at once.
* This function executes the requests in parallel and should be used as early as possible
* instead of calling the individual functions.
*/
export async function getSpaceContent(pointer: ContentPointer) {
const [space, pages, customization, scripts] = await Promise.all([
getSpace(pointer.spaceId),
getRevisionPages(pointer),
getSpaceCustomization(pointer.spaceId),
getSpaceIntegrationScripts(pointer.spaceId),
]);
return {
space,
pages,
customization,
scripts,
};
}
/**
* Create a cache tag for the API.
*/
+50 -22
View File
@@ -11,15 +11,6 @@ const redis =
})
: null;
const memoryCache = new Map<
string,
{
data: any;
tags: string[];
expiresAt: number;
}
>();
export interface CacheResult<Result> {
data: Result;
@@ -60,12 +51,17 @@ export function cache<Args extends any[], Result>(
const fetchValue = async (key: string, ...args: Args) => {
// Read the cache
const startTime = now();
const hasMemoryHit = getMemoryCache().has(key);
const cachedValue = await getCacheValue(key);
const readCacheDuration = now() - startTime;
// Returns it if it exists
if (cachedValue !== null) {
console.log(`cache: ${key} hit in ${readCacheDuration.toFixed(0)}ms`);
console.log(
`cache: ${key} hit in ${readCacheDuration.toFixed(
0,
)}ms (memory: ${hasMemoryHit}, redis: ${!!redis})`,
);
return cachedValue;
}
@@ -76,7 +72,9 @@ export function cache<Args extends any[], Result>(
// Write it to the cache
// As soon as it'll be possible with next-on-pages, we should `waitUntil`
// to delay writing the cache after the response has been sent to the client.
await setCacheValue(key, result);
if (result.ttl && result.ttl > 0) {
await setCacheValue(key, result);
}
const writeCacheDuration = now() - startTime - readCacheDuration - fetchDuration;
console.log(
@@ -84,12 +82,15 @@ export function cache<Args extends any[], Result>(
0,
)}ms, read in ${readCacheDuration.toFixed(0)}ms, write in ${writeCacheDuration.toFixed(
0,
)}ms`,
)}ms (redis: ${!!redis})`,
);
return result.data;
};
// During development, for now it fetches data twice between the middleware and the handler.
// TODO: find a way to share the cache between the two.
const pendings = new Map<string, Promise<any>>();
return async (...args: Args) => {
@@ -106,21 +107,25 @@ export function cache<Args extends any[], Result>(
pendings.set(key, promise);
// Remove the pending request once it's done
promise.finally(() => pendings.delete(key));
return await promise;
try {
const result = await promise;
return result;
} finally {
pendings.delete(key);
}
};
}
/**
* Parse an HTTP response into a cache entry.
*/
export function parseCacheResponse<Result, DefaultData = Result>(
response: Response,
): {
export function parseCacheResponse(response: Response): {
ttl: number;
tags: string[];
} {
const ageHeader = response.headers.get('age');
const age = ageHeader ? parseInt(ageHeader, 10) : 0;
const cacheControlHeader = response.headers.get('cache-control');
const cacheControl = cacheControlHeader ? parseCacheControl(cacheControlHeader) : null;
const cacheTagHeader =
@@ -132,7 +137,7 @@ export function parseCacheResponse<Result, DefaultData = Result>(
};
if (cacheControl && cacheControl['max-age']) {
entry.ttl = cacheControl['max-age'];
entry.ttl = Math.max(0, cacheControl['max-age'] - age - 60);
}
return entry;
@@ -160,6 +165,7 @@ export function cacheResponse<Result, DefaultData = Result>(
*/
export async function invalidateCacheTags(tags: string[]) {
// Clear from memory cache
const memoryCache = getMemoryCache();
memoryCache.forEach((value, key) => {
if (value.tags.some((t) => tags.includes(t))) {
memoryCache.delete(key);
@@ -203,13 +209,12 @@ function getCacheTagKey(tag: string) {
* Get a value from the cache.
*/
async function getCacheValue(key: string): Promise<any | null> {
const memoryCache = getMemoryCache();
const memoryEntry = memoryCache.get(key);
if (memoryEntry) {
if (memoryEntry.expiresAt > Date.now()) {
return memoryEntry.data;
}
return null;
}
if (redis) {
@@ -220,7 +225,7 @@ async function getCacheValue(key: string): Promise<any | null> {
memoryCache.set(key, {
data: value,
tags: [],
expiresAt: Date.now() + 2 * 60,
expiresAt: Date.now() + 60 * 1000,
});
}
@@ -236,6 +241,7 @@ async function getCacheValue(key: string): Promise<any | null> {
async function setCacheValue(key: string, entry: CacheResult<any>) {
const ttl = entry.ttl ?? 60 * 60 * 24;
const memoryCache = getMemoryCache();
memoryCache.set(key, {
data: entry.data,
tags: entry.tags ?? [],
@@ -257,6 +263,28 @@ async function setCacheValue(key: string, entry: CacheResult<any>) {
}
}
/**
* With next-on-pages, the code seems to be isolated between the middleware and the handler.
* To share the cache between the two, we use a global variable.
*/
function getMemoryCache(): Map<
string,
{
data: any;
tags: string[];
expiresAt: number;
}
> {
// @ts-ignore
if (!globalThis.gitbookMemoryCache) {
// @ts-ignore
globalThis.gitbookMemoryCache = new Map();
}
// @ts-ignore
return globalThis.gitbookMemoryCache;
}
function now(): number {
return typeof performance !== 'undefined' ? performance.now() : Date.now();
}
+58
View File
@@ -0,0 +1,58 @@
import { SpaceIntegrationScript } from '@gitbook/api';
import { merge } from 'content-security-policy-merger';
import { headers } from 'next/headers';
import { filterOutNullable } from './typescript';
/**
* Get the current nonce for the current request.
*/
export function getContentSecurityPolicyNonce(): string {
const headersList = headers();
const nonce = headersList.get('x-nonce');
if (!nonce) {
throw new Error('No nonce found in headers');
}
return nonce;
}
/**
* Create a nonce for a Content Security Policy.
*/
export function createContentSecurityPolicyNonce(): string {
const nonce = Buffer.from(crypto.randomUUID()).toString('base64');
return nonce;
}
/**
* Generate a Content Security Policy header for a space.
*/
export function getContentSecurityPolicy(scripts: SpaceIntegrationScript[], nonce: string): string {
// We need to allow loading any image or download any file
// to support image and OpenAPI blocks where the content reference could be external.
//
// Since I can't get the nonce to work for inline styles, we need to allow unsafe-inline
const defaultCSP = `
default-src 'self';
script-src 'self' 'nonce-${nonce}' 'strict-dynamic' 'unsafe-inline' 'unsafe-eval' integrations.gitbook.com;
style-src 'self' fonts.googleapis.com 'unsafe-inline';
img-src * 'self' blob: data: files.gitbook.com;
connect-src * 'self' integrations.gitbook.com app.gitbook.com;
font-src 'self' fonts.gstatic.com;
object-src 'none';
base-uri 'self';
form-action 'self';
frame-ancestors 'none';
`;
const result = scripts
.map(({ contentSecurityPolicy }) => contentSecurityPolicy)
.filter(filterOutNullable)
.reduce((csp, policy) => merge(csp, policy), defaultCSP);
return result
.replace(/\n/g, ' ')
.replace(/\s{2,}/g, ' ')
.trim();
}
+101 -70
View File
@@ -1,6 +1,13 @@
import { GitBookAPI } from '@gitbook/api';
import { NextResponse, NextRequest } from 'next/server';
import { PublishedContentWithCache, getPublishedContentByUrl } from '@/lib/api';
import {
PublishedContentWithCache,
getPublishedContentByUrl,
getSpaceContent,
withAPI,
} from '@/lib/api';
import { createContentSecurityPolicyNonce, getContentSecurityPolicy } from '@/lib/csp';
export const config = {
matcher: '/((?!_next/static|_next/image|.revalidate).*)',
@@ -37,7 +44,12 @@ export async function middleware(request: NextRequest) {
console.log('resolving', inputURL.toString());
const resolved = await lookupSpaceForURL(mode, apiEndpoint, inputURL, visitorAuthToken);
const resolved = await withAPI(
new GitBookAPI({
endpoint: apiEndpoint,
}),
() => lookupSpaceForURL(mode, inputURL, visitorAuthToken),
);
if (!resolved) {
return new NextResponse(`Not found`, {
status: 404,
@@ -51,8 +63,24 @@ export async function middleware(request: NextRequest) {
console.log(`${request.method} ${resolved.space}${resolved.pathname}`);
const headers = new Headers(request.headers);
const nonce = createContentSecurityPolicyNonce();
const csp = await withAPI(
new GitBookAPI({
endpoint: apiEndpoint,
authToken: resolved.apiToken,
}),
async () => {
const content = await getSpaceContent({
spaceId: resolved.space,
});
return getContentSecurityPolicy(content.scripts, nonce);
},
);
const headers = new Headers(request.headers);
// https://nextjs.org/docs/app/building-your-application/configuring/content-security-policy
headers.set('x-nonce', nonce);
headers.set('content-security-policy', csp);
// Pass a x-forwarded-host and origin to ensure Next doesn't block server actions when proxied
headers.set('x-forwarded-host', inputURL.host);
headers.set('origin', inputURL.origin);
@@ -71,6 +99,9 @@ export async function middleware(request: NextRequest) {
},
});
// Add Content Security Policy header
response.headers.set('content-security-policy', csp);
// When content is authenticated, we store the state in a cookie.
if (visitorAuthToken) {
response.cookies.set(VISITOR_AUTH_COOKIE, visitorAuthToken);
@@ -121,7 +152,6 @@ function getInputURL(request: NextRequest): { url: URL; mode: URLLookupMode } {
async function lookupSpaceForURL(
mode: URLLookupMode,
apiEndpoint: string | undefined,
url: URL,
visitorAuthToken: string | undefined,
): Promise<PublishedContentWithCache | null> {
@@ -130,10 +160,10 @@ async function lookupSpaceForURL(
return await lookupSpaceInSingleMode(url);
}
case 'multi': {
return await lookupSpaceInMultiMode(url, apiEndpoint, visitorAuthToken);
return await lookupSpaceInMultiMode(url, visitorAuthToken);
}
case 'multi-path': {
return await lookupSpaceInMultiPathMode(url, apiEndpoint, visitorAuthToken);
return await lookupSpaceInMultiPathMode(url, visitorAuthToken);
}
default:
throw new Error(
@@ -175,10 +205,9 @@ async function lookupSpaceInSingleMode(url: URL): Promise<PublishedContentWithCa
*/
async function lookupSpaceInMultiMode(
url: URL,
apiEndpoint: string | undefined,
visitorAuthToken: string | undefined,
): Promise<PublishedContentWithCache | null> {
return lookupSpaceByAPI(url, apiEndpoint, visitorAuthToken);
return lookupSpaceByAPI(url, visitorAuthToken);
}
/**
@@ -187,7 +216,6 @@ async function lookupSpaceInMultiMode(
*/
async function lookupSpaceInMultiPathMode(
url: URL,
apiEndpoint: string | undefined,
visitorAuthToken: string | undefined,
): Promise<PublishedContentWithCache | null> {
const targetStr = `https://${url.pathname}`;
@@ -197,7 +225,7 @@ async function lookupSpaceInMultiPathMode(
}
const target = new URL(targetStr);
const lookup = await lookupSpaceByAPI(target, apiEndpoint, visitorAuthToken);
const lookup = await lookupSpaceByAPI(target, visitorAuthToken);
if (!lookup) {
return null;
}
@@ -231,7 +259,6 @@ async function lookupSpaceInMultiPathMode(
*/
async function lookupSpaceByAPI(
url: URL,
apiEndpoint: string | undefined,
visitorAuthToken: string | undefined,
): Promise<PublishedContentWithCache | null> {
const lookupAlternatives = computeLookupAlternatives(url);
@@ -242,65 +269,76 @@ async function lookupSpaceByAPI(
} alternatives`,
);
console.time('lookupSpaceByAPI');
try {
const abort = new AbortController();
const matches = await Promise.all(
lookupAlternatives.map(async (alternative) => {
try {
const data = await getPublishedContentByUrl(
alternative.url,
apiEndpoint,
visitorAuthToken,
{
signal: abort.signal,
},
);
const startTime = Date.now();
const abort = new AbortController();
const matches = await Promise.all(
lookupAlternatives.map(async (alternative) => {
try {
const data = await getPublishedContentByUrl(alternative.url, visitorAuthToken, {
signal: abort.signal,
});
if ('redirect' in data) {
if (alternative.url === url.toString()) {
return data;
}
return null;
if ('redirect' in data) {
if (alternative.url === url.toString()) {
return data;
}
// Cancel all other requests to speed up the lookup
abort.abort();
return {
space: data.space,
basePath: data.basePath,
pathname: joinPath(data.pathname, alternative.extraPath),
apiToken: data.apiToken,
cacheMaxAge: data.cacheMaxAge,
cacheTags: data.cacheTags,
} as PublishedContentWithCache;
} catch (error) {
// @ts-ignore
if (error.name === 'AbortError') {
return null;
}
throw error;
return null;
}
}),
);
return matches.find((match) => match !== null) ?? null;
} finally {
console.timeEnd('lookupSpaceByAPI');
}
// Cancel all other requests to speed up the lookup
abort.abort();
return {
space: data.space,
basePath: data.basePath,
pathname: joinPath(data.pathname, alternative.extraPath),
apiToken: data.apiToken,
cacheMaxAge: data.cacheMaxAge,
cacheTags: data.cacheTags,
} as PublishedContentWithCache;
} catch (error) {
// @ts-ignore
if (error.name === 'AbortError') {
return null;
}
throw error;
}
}),
);
console.log(`lookup took ${Date.now() - startTime}ms`);
return matches.find((match) => match !== null) ?? null;
}
function computeLookupAlternatives(url: URL) {
const alternatives: Array<{ url: string; extraPath: string }> = [];
const pushAlternative = (url: URL, extraPath: string) => {
const existing = alternatives.find((alt) => alt.url === url.toString());
if (existing) {
if (existing.extraPath !== extraPath) {
throw new Error(
`Invalid extraPath ${extraPath} for url ${url.toString()}, already set to ${
existing.extraPath
}`,
);
}
return;
}
alternatives.push({
url: url.toString(),
extraPath,
});
};
// Match only with the host, if it can be a custom hostname
// It should cover most cases of custom domains, and with caching, it should be fast.
if (!url.hostname.includes('.gitbook.io')) {
alternatives.push({
url: url.origin,
extraPath: url.pathname,
});
const noPathURL = new URL(url);
noPathURL.pathname = '/';
pushAlternative(noPathURL, url.pathname.slice(1));
}
const pathSegments = url.pathname.slice(1).split('/');
@@ -311,10 +349,8 @@ function computeLookupAlternatives(url: URL) {
if (pathSegments.length > 0) {
const shortURL = new URL(url);
shortURL.pathname = pathSegments[0];
alternatives.push({
url: shortURL.toString(),
extraPath: pathSegments.slice(1).join('/'),
});
pushAlternative(shortURL, pathSegments.slice(1).join('/'));
}
// URL looks like a collection url (with /v/ in the path)
@@ -322,18 +358,13 @@ function computeLookupAlternatives(url: URL) {
const collectionURL = new URL(url);
const vIndex = pathSegments.indexOf('v');
collectionURL.pathname = pathSegments.slice(0, vIndex + 1).join('/');
alternatives.push({
url: collectionURL.toString(),
extraPath: pathSegments.slice(vIndex + 1).join('/'),
});
pushAlternative(collectionURL, pathSegments.slice(vIndex + 1).join('/'));
}
// Always try with the full URL
if (!alternatives.some((alt) => alt.url === url.toString())) {
alternatives.push({
url: url.toString(),
extraPath: '',
});
pushAlternative(url, '');
}
return alternatives;
+3
View File
@@ -0,0 +1,3 @@
declare module 'content-security-policy-merger' {
export function merge(...policies: string[]): string;
}