diff --git a/.changeset/bold-points-flow.md b/.changeset/bold-points-flow.md new file mode 100644 index 000000000..385444f9d --- /dev/null +++ b/.changeset/bold-points-flow.md @@ -0,0 +1,5 @@ +--- +"gitbook": patch +--- + +Track new events for site analytics. diff --git a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts index da1176d9b..475297000 100644 --- a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts +++ b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts @@ -10,5 +10,6 @@ export async function GET( { params }: { params: Promise } ) { const { context } = await getStaticSiteContext(await params); + return serveLLMsTxt(context, { withMarkdownPages: true }); } diff --git a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/mcp/route.ts b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/mcp/route.ts index ec79c0543..2a95bba41 100644 --- a/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/mcp/route.ts +++ b/packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/mcp/route.ts @@ -10,13 +10,26 @@ import { createMcpHandler } from 'mcp-handler'; import type { NextRequest } from 'next/server'; import { z } from 'zod'; -async function handler( - nextRequest: NextRequest, - { params }: { params: Promise } -) { +async function handler(request: NextRequest, { params }: { params: Promise }) { const { context } = await getStaticSiteContext(await params); const { dataFetcher, linker, site } = context; + waitUntil( + trackServerInsightsEvents({ + organizationId: context.organizationId, + siteId: context.site.id, + events: [ + { + type: 'mcp_request', + location: { + displayContext: SiteInsightsDisplayContext.Server, + }, + }, + ], + request, + }) + ); + const mcpHandler = createMcpHandler( (server) => { server.tool( @@ -49,7 +62,7 @@ async function handler( }, }, ], - request: nextRequest, + request, }) ); @@ -118,10 +131,9 @@ async function handler( const requestURL = new URL( context.linker.toAbsoluteURL(context.linker.toPathInSite('~gitbook/mcp')) ); - requestURL.search = nextRequest.nextUrl.search; + requestURL.search = request.nextUrl.search; - const request = new Request(requestURL, nextRequest); - return mcpHandler(request); + return mcpHandler(new Request(requestURL, request)); } export { handler as GET, handler as POST }; diff --git a/packages/gitbook/src/lib/data/errors.ts b/packages/gitbook/src/lib/data/errors.ts index 686860713..4a3b5f8b1 100644 --- a/packages/gitbook/src/lib/data/errors.ts +++ b/packages/gitbook/src/lib/data/errors.ts @@ -181,7 +181,7 @@ export function extractCacheControl(error: GitBookAPIError) { * Get a data fetcher exposable error from a JS error. * This function should never throw, even if the error is not in the expected format. In that case, it should return a generic error with code 500. */ -export function getExposableError(error: Error): DataFetcherErrorData { +export function getExposableError(error: unknown): DataFetcherErrorData { if (error instanceof GitBookAPIError) { const cache = extractCacheControl(error); diff --git a/packages/gitbook/src/lib/markdownPage.ts b/packages/gitbook/src/lib/markdownPage.ts index 38bd489cd..ccfce7272 100644 --- a/packages/gitbook/src/lib/markdownPage.ts +++ b/packages/gitbook/src/lib/markdownPage.ts @@ -1,6 +1,6 @@ import type { GitBookSiteContext } from '@/lib/context'; -import type { DataFetcherResponse } from '@/lib/data'; -import { resolvePagePathDocumentOrGroup } from '@/lib/pages'; +import { DataFetcherError } from '@/lib/data'; +import type { ResolvedPagePath } from '@/lib/pages'; import { getIndexablePages } from '@/lib/sitemap'; import { getMarkdownForPagesTree } from '@/routes/llms'; import { type RevisionPageDocument, type RevisionPageGroup, RevisionPageType } from '@gitbook/api'; @@ -14,37 +14,22 @@ import { gfm } from 'micromark-extension-gfm'; import { remove } from 'unist-util-remove'; import { type GitBookLinker, relativeToAbsoluteLinks } from './links'; -type MarkdownResult = DataFetcherResponse; - /** * Generate a markdown version of a page. * Handles both regular document pages and group pages (pages with child pages). */ export async function getMarkdownForPage( context: GitBookSiteContext, - pagePath: string -): Promise { - const pageLookup = resolvePagePathDocumentOrGroup(context.revision.pages, pagePath); - - if (!pageLookup) { - return { - error: { - message: `Page "${pagePath}" not found`, - code: 404, - }, - }; - } - + pageLookup: ResolvedPagePath +): Promise { const { page } = pageLookup; // Only handle documents and groups if (page.type !== RevisionPageType.Document && page.type !== RevisionPageType.Group) { - return { - error: { - message: `Page "${pagePath}" is not a document or group`, - code: 400, - }, - }; + throw new DataFetcherError( + `Page "${pageLookup.page.title}" is not a document or group`, + 400 + ); } // Handle group pages @@ -59,12 +44,7 @@ export async function getMarkdownForPage( }); if (error) { - return { - error: { - message: 'An error occurred while fetching the markdown for this page', - code: 500, - }, - }; + throw error; } const tree = fromPageMarkdown({ @@ -78,7 +58,7 @@ export async function getMarkdownForPage( return servePageGroup(context, page); } - return { data: toPageMarkdown(tree) }; + return toPageMarkdown(tree); } /** @@ -150,15 +130,10 @@ function isEmptyMarkdownPage(tree: Root): boolean { async function servePageGroup( context: GitBookSiteContext, page: RevisionPageDocument | RevisionPageGroup -): Promise { +): Promise { const siteSpaceUrl = context.space.urls.published; if (!siteSpaceUrl) { - return { - error: { - message: `Page "${page.title}" is not published`, - code: 404, - }, - }; + throw new DataFetcherError(`Page "${page.title}" is not published`, 404); } const indexablePages = getIndexablePages(page.pages); @@ -180,9 +155,7 @@ async function servePageGroup( ], }; - return { - data: toMarkdown(markdownTree, { - bullet: '-', - }), - }; + return toMarkdown(markdownTree, { + bullet: '-', + }); } diff --git a/packages/gitbook/src/lib/pages.ts b/packages/gitbook/src/lib/pages.ts index eecb821e4..50e25fac1 100644 --- a/packages/gitbook/src/lib/pages.ts +++ b/packages/gitbook/src/lib/pages.ts @@ -8,7 +8,7 @@ import { export type AncestorRevisionPage = RevisionPageDocument | RevisionPageGroup; -type ResolvedPagePath = { +export type ResolvedPagePath = { page: Page; ancestors: AncestorRevisionPage[]; }; diff --git a/packages/gitbook/src/lib/tracking.ts b/packages/gitbook/src/lib/tracking.ts index 5d5e3088c..57627569a 100644 --- a/packages/gitbook/src/lib/tracking.ts +++ b/packages/gitbook/src/lib/tracking.ts @@ -2,6 +2,7 @@ import type * as api from '@gitbook/api'; import type { headers as nextHeaders } from 'next/headers'; import { apiClient } from './data/api'; import { GITBOOK_DISABLE_TRACKING } from './env'; +import { getLogger } from './logger'; /** * Return true if events should be tracked on the site. @@ -74,6 +75,12 @@ export async function trackServerInsightsEvents(args: { events: ServerInsightsEventInput[]; request: Request; }) { + const logger = getLogger().subLogger('tracking'); + + logger.info( + `Tracking ${args.events.length} events for site ${args.siteId} (enabled=${!GITBOOK_DISABLE_TRACKING})` + ); + if (GITBOOK_DISABLE_TRACKING) { return; } diff --git a/packages/gitbook/src/middleware.ts b/packages/gitbook/src/middleware.ts index 58757f26a..021367d78 100644 --- a/packages/gitbook/src/middleware.ts +++ b/packages/gitbook/src/middleware.ts @@ -1,4 +1,8 @@ -import { CustomizationThemeMode } from '@gitbook/api'; +import { + CustomizationThemeMode, + SiteInsightsDisplayContext, + SiteInsightsLLMSVariant, +} from '@gitbook/api'; import Negotiator from 'negotiator'; import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; @@ -36,9 +40,14 @@ import { normalizeVisitorURL, serveVisitorClaimsDataRequest, } from '@/lib/visitors'; +import { waitUntil } from '@/lib/waitUntil'; import { serveResizedImage } from '@/routes/image'; import { cookies } from 'next/headers'; -import { serveProxyAnalyticsEvent } from './lib/tracking'; +import { + type ServerInsightsEventInput, + serveProxyAnalyticsEvent, + trackServerInsightsEvents, +} from './lib/tracking'; export const config = { matcher: [ '/((?!_next/static|_next/image|~gitbook/static|~gitbook/revalidate|~gitbook/monitoring|~scalar/proxy).*)', @@ -411,12 +420,24 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) { requestHeaders.set('origin', request.nextUrl.origin); const siteURLWithoutProtocol = `${siteCanonicalURL.host}${siteURLData.basePath}`; - const { pathname, routeType: routeTypeFromPathname } = encodePathInSiteContent( - siteURLData.pathname, - request - ); + const { + pathname, + routeType: routeTypeFromPathname, + events, + } = encodePathInSiteContent(siteURLData.pathname, request); routeType = routeTypeFromPathname ?? routeType; + if (events && events.length > 0) { + waitUntil( + trackServerInsightsEvents({ + organizationId: siteURLData.organization, + siteId: siteURLData.site, + events, + request, + }) + ); + } + const route = [ 'sites', routeType, @@ -633,6 +654,7 @@ function encodePathInSiteContent( ): { pathname: string; routeType?: 'static' | 'dynamic'; + events?: ServerInsightsEventInput[] | undefined; } { const pathname = removeLeadingSlash(removeTrailingSlash(rawPathname)); @@ -646,12 +668,32 @@ function encodePathInSiteContent( return { pathname: `~gitbook/rss/${encodePagePath(rssMatch[2])}`, routeType: 'static', + events: [ + { + type: 'rss_request', + location: { + displayContext: SiteInsightsDisplayContext.Server, + }, + }, + ], }; } // We skip encoding for paginated llms-full.txt pages (i.e. llms-full.txt/100) if (pathname.match(LLMS_FULL_PATH_REGEX)) { - return { pathname, routeType: 'static' }; + return { + pathname, + routeType: 'static', + events: [ + { + type: 'llms_request', + llmsVariant: SiteInsightsLLMSVariant.Full, + location: { + displayContext: SiteInsightsDisplayContext.Server, + }, + }, + ], + }; } // If the pathname is an embedded page @@ -667,16 +709,32 @@ function encodePathInSiteContent( case '~gitbook/embed/assistant': case '~gitbook/icon': return { pathname }; - case '~gitbook/mcp': + // LLMs.txt, sitemap, sitemap-pages and robots.txt are always static + // as they only depend on the site structure / pages. case 'llms.txt': case 'llms-full.txt': + return { + pathname, + routeType: 'static', + events: [ + { + type: 'llms_request', + llmsVariant: + pathname === 'llms.txt' + ? SiteInsightsLLMSVariant.Standard + : SiteInsightsLLMSVariant.Full, + location: { + displayContext: SiteInsightsDisplayContext.Server, + }, + }, + ], + }; + case '~gitbook/mcp': case 'sitemap.xml': case 'sitemap-pages.xml': case 'robots.txt': case '~gitbook/embed/script.js': case '~gitbook/embed/demo': - // LLMs.txt, sitemap, sitemap-pages and robots.txt are always static - // as they only depend on the site structure / pages. return { pathname, routeType: 'static' }; case '~gitbook/pdf': case '~gitbook/search': @@ -693,6 +751,16 @@ function encodePathInSiteContent( pathname: `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`, // The markdown content is always static and doesn't depend on the dynamic parameter (customization, theme, etc) routeType: 'static', + events: [ + { + type: 'page_markdown_request', + // TODO: track pageId / spaceId when possible + // We don't do it at the moment as we can't easily extract it from the URL. + location: { + displayContext: SiteInsightsDisplayContext.Server, + }, + }, + ], }; } return { pathname: encodePagePath(pathname) }; diff --git a/packages/gitbook/src/routes/markdownPage.ts b/packages/gitbook/src/routes/markdownPage.ts index c0329def4..bdd6b06fb 100644 --- a/packages/gitbook/src/routes/markdownPage.ts +++ b/packages/gitbook/src/routes/markdownPage.ts @@ -1,6 +1,7 @@ import type { GitBookSiteContext } from '@/lib/context'; -import { getDataOrNull } from '@/lib/data'; +import { DataFetcherError, getExposableError } from '@/lib/data'; import { getMarkdownForPage } from '@/lib/markdownPage'; +import { resolvePagePathDocumentOrGroup } from '@/lib/pages'; /** * Serve a markdown version of a page. @@ -8,26 +9,23 @@ import { getMarkdownForPage } from '@/lib/markdownPage'; */ export async function servePageMarkdown(context: GitBookSiteContext, pagePath: string) { try { - const result = await getDataOrNull(getMarkdownForPage(context, pagePath)); - if (!result) { - return new Response('Page not found', { - status: 404, - headers: { - 'Content-Type': 'text/plain; charset=utf-8', - }, - }); + const pageLookup = resolvePagePathDocumentOrGroup(context.revision.pages, pagePath); + if (!pageLookup) { + throw new DataFetcherError(`Page "${pagePath}" not found`, 404); } - return new Response(result, { + const markdown = await getMarkdownForPage(context, pageLookup); + + return new Response(markdown, { headers: { 'Content-Type': 'text/markdown; charset=utf-8', 'X-Robots-Tag': 'noindex', }, }); } catch (error) { - console.error('Error serving markdown page:', error); - return new Response('Internal Server Error', { - status: 500, + const exposable = getExposableError(error); + return new Response(exposable.message, { + status: exposable.code, headers: { 'Content-Type': 'text/plain; charset=utf-8', }, diff --git a/packages/gitbook/src/routes/rss.ts b/packages/gitbook/src/routes/rss.ts index d51c1e8cf..0435405e1 100644 --- a/packages/gitbook/src/routes/rss.ts +++ b/packages/gitbook/src/routes/rss.ts @@ -4,7 +4,8 @@ import { getPageDocument } from '@/lib/data/pages'; import { getBlocksByType, getNodeText, isHeadingBlock } from '@/lib/document'; import { resolvePagePathDocumentOrGroup } from '@/lib/pages'; import { joinPath } from '@/lib/paths'; -import { type RevisionPageDocument, RevisionPageType } from '@gitbook/api'; +import type { RevisionPageDocument } from '@gitbook/api'; +import { RevisionPageType } from '@gitbook/api'; import { Feed } from 'feed'; /**