Replace Puppeteer by Playwright (#224)

* Replace Puppeteer by Playwright

* Remove the limit of workers on ci

* Various fixes after review

* Document things

.github/actions/setup-playwright/action.yml

@@ -0,0 +1,35 @@
+name: 'Setup Playwright'
+description: 'Install Playwright and dependencies'
+runs:
+    using: 'composite'
+    steps:
+        # Run npm ci and get Playwright version
+        - name: 🏗 Prepare Playwright env
+          shell: bash
+          run: |
+              PLAYWRIGHT_VERSION=$(npm ls --json @playwright/test | jq --raw-output '.dependencies["@playwright/test"].version')
+              echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV
+
+        # Cache browser binaries, cache key is based on Playwright version and OS
+        - name: 🧰 Cache Playwright browser binaries
+          uses: actions/cache@v3
+          id: playwright-cache
+          with:
+              path: '~/.cache/ms-playwright'
+              key: '${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}'
+              restore-keys: |
+                  ${{ runner.os }}-playwright-
+
+        # Install browser binaries & OS dependencies if cache missed
+        - name: 🏗 Install Playwright browser binaries & OS dependencies
+          if: steps.playwright-cache.outputs.cache-hit != 'true'
+          shell: bash
+          run: |
+              bun x playwright install --with-deps chromium
+
+        # Install only the OS dependencies if cache hit
+        - name: 🏗 Install Playwright OS dependencies
+          if: steps.playwright-cache.outputs.cache-hit == 'true'
+          shell: bash
+          run: |
+              bun x playwright install-deps

.github/workflows/ci.yaml

@@ -12,7 +12,7 @@ jobs:
             deployment_url: ${{ steps.cloudflare.outputs.url }}
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies
@@ -72,39 +72,32 @@ jobs:
         needs: deploy
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies
               run: bun install --frozen-lockfile
-            - name: Run visual tests
-              run: bun ./tests/visual-testing.ts $DEPLOYMENT_URL
-              env:
-                  DEPLOYMENT_URL: ${{needs.deploy.outputs.deployment_url}}
-            - name: Upload to Argos
-              run: bun x argos upload --token $ARGOS_TOKEN ./screenshots
+            - name: Setup Playwright
+              uses: ./.github/actions/setup-playwright
+            - name: Run Playwright tests
+              run: bun e2e
               env:
+                  BASE_URL: ${{needs.deploy.outputs.deployment_url}}
                   ARGOS_TOKEN: ${{ secrets.ARGOS_TOKEN }}
-            - name: Upload failed tests as artifacts
-              uses: actions/upload-artifact@v3
-              if: ${{ failure() }}
-              with:
-                  name: error-screenshots
-                  path: screenshots/errors/
     pagespeed-testing:
         runs-on: ubuntu-latest
         name: PageSpeed Testing
         needs: deploy
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies
               run: bun install --frozen-lockfile
               env:
                   PUPPETEER_SKIP_DOWNLOAD: 1
-            - name: Run visual tests
+            - name: Run pagespeed tests
               run: bun ./tests/pagespeed-testing.ts $DEPLOYMENT_URL
               env:
                   DEPLOYMENT_URL: ${{needs.deploy.outputs.deployment_url}}
@@ -114,7 +107,7 @@ jobs:
         name: Format
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies
@@ -127,7 +120,7 @@ jobs:
         name: Lint
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies
@@ -140,20 +133,20 @@ jobs:
         name: Test
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies
               run: bun install --frozen-lockfile
               env:
                   PUPPETEER_SKIP_DOWNLOAD: 1
-            - run: bun test
+            - run: bun unit
     typecheck:
         runs-on: ubuntu-latest
         name: Typecheck
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
             - name: Setup bun
               uses: oven-sh/setup-bun@v1
             - name: Install dependencies

.gitignore

@@ -39,3 +39,7 @@ screenshots/
 
 # Sentry Config File
 .sentryclirc
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

.prettierignore

@@ -1 +1 @@
-.next
+.next

README.md

@@ -4,7 +4,7 @@ Next.js application to render GitBook published content.
 
 ## Development
 
-#### Start the local server
+### Start the local server
 
 Clone the repository and use [Bun](https://bun.sh/) to install dependencies and run the local development server.
 
@@ -20,12 +20,15 @@ bun dev
 
 Then open the space in your web browser, using `http://localhost:3000/<host>/<path>` (example: `http://localhost:3000/docs.gitbook.com`).
 
-#### Other development commands
+### Other development commands
 
 -   `bun format`: format the code
 -   `bun lint`: lint the code
 
-#### CI and testing
+### Testing and CI
+
+-   Run unit tests with `bun unit`.
+-   Run E2E tests with `bun e2e` (be sure to have your dev server running locally).
 
 All pull-requests will be tested against both visual and performances testing to prevent regressions.
 

e2e/pages.spec.ts

@@ -1,18 +1,18 @@
-import fs from 'node:fs/promises';
-import rison from 'rison';
-import puppeteer, { Page } from 'puppeteer';
-import { argosScreenshot } from '@argos-ci/puppeteer';
-import { getContentTestURL, getTargetURL } from './utils';
+import { argosScreenshot } from '@argos-ci/playwright';
 import {
     CustomizationHeaderPreset,
     CustomizationLocale,
     CustomizationSettings,
 } from '@gitbook/api';
+import { test, Page } from '@playwright/test';
+import rison from 'rison';
+
+import { getContentTestURL } from '../tests/utils';
 
 interface Test {
     name: string;
     url: string;
-    wait?: (page: Page) => Promise<any>;
+    run?: (page: Page) => Promise<unknown>;
 }
 
 interface TestsCase {
@@ -44,12 +44,12 @@ const testCases: TestsCase[] = [
             {
                 name: 'Search Results',
                 url: '?q=gitbook',
-                wait: (page) => page.waitForSelector('[data-test="search-results"]'),
+                run: (page) => page.waitForSelector('[data-test="search-results"]'),
             },
             {
                 name: 'AI Search',
                 url: '?q=What+is+GitBook%3F&ask=true',
-                wait: (page) => page.waitForSelector('[data-test="search-ask-answer"]'),
+                run: (page) => page.waitForSelector('[data-test="search-ask-answer"]'),
             },
             {
                 name: 'Not found',
@@ -293,50 +293,23 @@ const testCases: TestsCase[] = [
     },
 ];
 
-console.log(`Starting visual testing with ${getTargetURL()}...`);
-
-const browser = await puppeteer.launch({
-    headless: 'new',
-});
-
 for (const testCase of testCases) {
-    for (const test of testCase.tests) {
-        const page = await browser.newPage();
-        const contentUrl = new URL(test.url, testCase.baseUrl);
-        const url = getContentTestURL(contentUrl.toString());
-        const start = Date.now();
-
-        console.log(`Testing ${testCase.name} - ${test.name} (${url})...`);
-
-        const screenshotName = `${testCase.name} - ${test.name}.png`;
-
-        try {
-            await page.goto(url, { waitUntil: test.wait ? 'load' : 'networkidle2' });
-
-            if (test.wait) {
-                await test.wait(page);
-            }
-        } catch (error) {
-            await fs.mkdir('screenshots/errors', {
-                recursive: true,
+    test.describe(testCase.name, () => {
+        for (const testEntry of testCase.tests) {
+            test(testEntry.name, async ({ page, baseURL }) => {
+                const contentUrl = new URL(testEntry.url, testCase.baseUrl);
+                const url = getContentTestURL(contentUrl.toString(), baseURL);
+                await page.goto(url);
+                if (testEntry.run) {
+                    await testEntry.run(page);
+                }
+                await argosScreenshot(page, `${testCase.name} - ${testEntry.name}`, {
+                    viewports: ['macbook-13', 'iphone-x', 'ipad-2'],
+                });
             });
-
-            await page.screenshot({ path: `screenshots/errors/${screenshotName}` });
-            console.log(`❌ Failed in ${((Date.now() - start) / 1000).toFixed(2)}s`);
-            throw error;
         }
-
-        await argosScreenshot(page, screenshotName, {
-            viewports: ['macbook-13', 'iphone-x', 'ipad-2'],
-        });
-        console.log(`✅ Done in ${((Date.now() - start) / 1000).toFixed(2)}s`);
-        console.log('');
-        await page.close();
-    }
+    });
 }
-await browser.close();
-
-console.log('All done!');
 
 /**
  * Create a URL with customization settings.

package.json

@@ -10,7 +10,9 @@
         "lint": "next lint",
         "format": "prettier ./ --ignore-unknown --write",
         "format:check": "prettier ./ --ignore-unknown --list-different",
-        "typecheck": "tsc --noEmit"
+        "typecheck": "tsc --noEmit",
+        "unit": "bun test {src,packages}/**/*.test.ts",
+        "e2e": "playwright test"
     },
     "workspaces": [
         "packages/*"
@@ -61,9 +63,10 @@
     },
     "devDependencies": {
         "@argos-ci/cli": "^1.0.4",
-        "@argos-ci/puppeteer": "^1.2.1",
+        "@argos-ci/playwright": "^1.9.3",
         "@cloudflare/next-on-pages": "^1.9.0",
         "@cloudflare/workers-types": "^4.20231218.0",
+        "@playwright/test": "^1.42.1",
         "@types/js-cookie": "^3.0.6",
         "@types/js-yaml": "^4.0.9",
         "@types/jsontoxml": "^1.0.5",
@@ -83,7 +86,6 @@
         "postcss": "^8",
         "prettier": "^3.0.3",
         "psi": "^4.1.0",
-        "puppeteer": "^21.6.1",
         "tailwindcss": "^3.4.0",
         "typescript": "^5"
     }

playwright.config.ts

@@ -0,0 +1,23 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+    testDir: './e2e',
+    fullyParallel: true,
+    forbidOnly: !!process.env.CI,
+    retries: process.env.CI ? 2 : 0,
+    reporter: [
+        process.env.CI ? ['dot'] : ['list'],
+        ['@argos-ci/playwright/reporter', { uploadToArgos: !!process.env.CI }],
+    ],
+    projects: [
+        {
+            name: 'chromium',
+            use: { ...devices['Desktop Chrome'] },
+        },
+    ],
+    use: {
+        baseURL: process.env.BASE_URL || 'http://localhost:3000',
+        trace: 'on-first-retry',
+        screenshot: 'only-on-failure',
+    },
+});

tests/pagespeed-testing.ts

@@ -1,4 +1,5 @@
 import psi from 'psi';
+
 import { getContentTestURL, getTargetURL } from './utils';
 
 interface Test {

tests/utils.ts

@@ -11,11 +11,16 @@ export function getTargetURL() {
     return targetUrl;
 }
 
+export function getContentPathName(input: string): string {
+    const contentUrl = new URL(input);
+    return `${contentUrl.host}${contentUrl.pathname}`;
+}
+
 /**
  * Get the URL to load for a content
  */
-export function getContentTestURL(input: string): string {
-    const url = new URL(getTargetURL());
+export function getContentTestURL(input: string, baseUrl: string = getTargetURL()): string {
+    const url = new URL(baseUrl);
     const contentUrl = new URL(input);
 
     url.pathname = `${contentUrl.host}${contentUrl.pathname}`;