Visual testing stabilization (#2741)

packages/gitbook/e2e/pages.spec.ts

@@ -31,7 +31,7 @@ interface Test {
     cookies?: Parameters<BrowserContext['addCookies']>[0];
     run?: (page: Page) => Promise<unknown>; // The test to run
     fullPage?: boolean; // Whether the test should be fullscreened during testing
-    screenshot?: false; // Should a screenshot be stored
+    screenshot?: false | { threshold: number }; // Disable screenshot or set threshold
     only?: boolean; // Only run this test
 }
 
@@ -95,19 +95,26 @@ const testCases: TestsCase[] = [
             {
                 name: 'Search',
                 url: '?q=',
+                screenshot: false,
+                run: async (page) => {
+                    await expect(page.getByTestId('search-results')).toBeVisible();
+                    const allItems = await page.getByTestId('search-result-item').all();
+                    // Expect at least 3 questions
+                    await expect(allItems.length).toBeGreaterThan(2);
+                },
             },
             {
                 name: 'Search Results',
                 url: '?q=gitbook',
                 run: async (page) => {
-                    await page.waitForSelector('[data-test="search-results"]');
+                    await expect(page.getByTestId('search-results')).toBeVisible();
                 },
             },
             {
                 name: 'AI Search',
                 url: '?q=What+is+GitBook%3F&ask=true',
                 run: async (page) => {
-                    await page.waitForSelector('[data-test="search-ask-answer"]');
+                    await expect(page.getByTestId('search-ask-answer')).toBeVisible();
                 },
                 screenshot: false,
             },
@@ -273,14 +280,14 @@ const testCases: TestsCase[] = [
                 name: 'Search Results',
                 url: '?q=gitbook',
                 run: async (page) => {
-                    await page.waitForSelector('[data-test="search-results"]');
+                    await expect(page.getByTestId('search-results')).toBeVisible();
                 },
             },
             {
                 name: 'AI Search',
                 url: '?q=What+is+GitBook%3F&ask=true',
                 run: async (page) => {
-                    await page.waitForSelector('[data-test="search-ask-answer"]');
+                    await expect(page.getByTestId('search-ask-answer')).toBeVisible();
                 },
                 screenshot: false,
             },
@@ -336,6 +343,7 @@ const testCases: TestsCase[] = [
                 name: 'Inline Images',
                 url: 'blocks/inline-images',
                 run: waitForCookiesDialog,
+                screenshot: { threshold: 0.8 },
             },
             {
                 name: 'Tabs',
@@ -678,6 +686,7 @@ const testCases: TestsCase[] = [
                 run: async (page) => {
                     await expect(page.locator('h1')).toHaveText('SSO');
                 },
+                screenshot: false,
             },
         ],
     },
@@ -1347,6 +1356,7 @@ for (const testCase of testCases) {
                     await testEntry.run(page);
                 }
                 if (testEntry.screenshot !== false) {
+                    await scrollTOCToTop(page);
                     await argosScreenshot(page, `${testCase.name} - ${testEntry.name}`, {
                         viewports: ['macbook-16', 'macbook-13', 'iphone-x', 'ipad-2'],
                         argosCSS: `
@@ -1355,6 +1365,7 @@ for (const testCase of testCases) {
                             display: none !important;
                         }
                     `,
+                        threshold: testEntry.screenshot?.threshold ?? undefined,
                         fullPage: testEntry.fullPage ?? false,
                         beforeScreenshot: async () => {
                             await waitForIcons(page);
@@ -1466,3 +1477,12 @@ async function waitForIcons(page: Page) {
         );
     });
 }
+
+/**
+ * Scroll the table of contents to the top to stabilize the screenshot.
+ */
+async function scrollTOCToTop(page: Page) {
+    await page.evaluate(() => {
+        document.querySelector('[data-testid=toc-container]')?.scrollTo(0, 0);
+    });
+}

packages/gitbook/src/components/Search/SearchAskAnswer.tsx

@@ -141,7 +141,7 @@ function AnswerBody(props: { answer: AskAnswerResult }) {
     return (
         <>
             <div
-                data-test="search-ask-answer"
+                data-testid="search-ask-answer"
                 className={tcls(
                     'my-4',
                     'sm:mt-6',

packages/gitbook/src/components/Search/SearchQuestionResultItem.tsx

@@ -24,6 +24,7 @@ export const SearchQuestionResultItem = React.forwardRef(function SearchQuestion
         <Link
             ref={ref}
             onClick={onClick}
+            data-testid="search-result-item"
             className={tcls(
                 'flex',
                 'px-4',

packages/gitbook/src/components/Search/SearchResults.tsx

@@ -217,10 +217,7 @@ export const SearchResults = React.forwardRef(function SearchResults(
     }
 
     const noResults = (
-        <div
-            data-test="search-noresults"
-            className={tcls('text', 'text-dark/8', 'p-8', 'text-center', 'dark:text-light/8')}
-        >
+        <div className={tcls('text', 'text-dark/8', 'p-8', 'text-center', 'dark:text-light/8')}>
             {t(language, 'search_no_results', query)}
         </div>
     );
@@ -234,7 +231,7 @@ export const SearchResults = React.forwardRef(function SearchResults(
                 ) : null
             ) : (
                 <>
-                    <div data-test="search-results">
+                    <div data-testid="search-results">
                         {results.map((item, index) => {
                             switch (item.type) {
                                 case 'page': {

packages/gitbook/src/components/TableOfContents/TOCScroller.tsx

@@ -26,7 +26,12 @@ export function TOCScrollContainer(props: {
 
     return (
         <TOCScrollContainerRefContext.Provider value={scrollContainerRef}>
-            <div ref={scrollContainerRef} className={tcls(className)} style={style}>
+            <div
+                ref={scrollContainerRef}
+                data-testid="toc-container"
+                className={tcls(className)}
+                style={style}
+            >
                 {children}
             </div>
         </TOCScrollContainerRefContext.Provider>