feat(ci): Use Argos to snapshot html/css and accessibility tree (#12382)

Author: slorber

.github/workflows/argos.yml

@@ -32,7 +32,8 @@ jobs:
         (github.event_name != 'pull_request' && github.ref_name == 'main') ||
         (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'Argos'))
       )
-    runs-on: ubuntu-latest
+    # runs-on: ubuntu-latest
+    runs-on: 8-core-ubuntu
     steps:
       - name: Check out repository code
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -55,5 +56,8 @@ jobs:
       - name: Build website fast
         run: pnpm argos:build
 
-      - name: Take Argos screenshots
+      - name: Take and upload Argos screenshots
         run: pnpm argos:screenshot
+
+      - name: Upload Argos text snapshots
+        run: pnpm argos:upload-text-snapshots

argos/README.md

@@ -4,18 +4,31 @@ We use [Argos CI](https://argos-ci.com) to detect visual regressions on Docusaur
 
 This workspace can be run manually, but is generally run through the [Argos GitHub Action](../.github/workflows/argos.yml).
 
+## Workflow overview
+
 The workflow execute those following steps:
 
-- Build the website locally with `pnpm build:website:fast`
+- Build the website locally with `pnpm argos:build` (a variant of `pnpm build:website:fast`)
 - Start the website server with `pnpm serve:website` on [http://localhost:3000](http://localhost:3000)
 - Take screenshots of all pages found in `sitemap.xml` with Playwright
-- Upload all screenshots to [Argos CI](https://argos-ci.com)
+- Upload all screenshots to [Argos CI](https://argos-ci.com) using the [Playwright reporter](https://argos-ci.com/docs/reference/playwright)
+- Upload [HTML/CSS/JS files to also diff them on Argos](https://argos-ci.com/docs/learn/how-to-guides/visual-coverage/compare-non-image-files) using `pnpm argos:upload-text-snapshots`
 
 This workflow runs for `main` and PR branches, and add a commit status to each PR with a visual diff that we can easily inspect.
 
----
+## Run manually
+
+To run this manually, use:
+
+```bash
+pnpm argos:build
+pnpm argos:screenshot
+pnpm argos:upload-text-snapshots
+```
+
+The Playwright reporter will upload the screenshots to Argos.
 
-Some additional capabilities:
+## How to modify
 
 - Use [./tests/screenshot.spec.ts](./tests/screenshot.spec.ts) to customize the screenshots we take, eventually filter out some useless sitemap pages like versioned docs
 - Use [./tests/screenshot.css](./tests/screenshot.css) to hide flaky CSS elements: iframe, video, gif...

argos/package.json

@@ -6,12 +6,14 @@
   "private": true,
   "scripts": {
     "screenshot": "playwright test",
-    "upload": "pnpm dlx @argos-ci/cli upload ./screenshots/chromium",
+    "upload": "pnpm exec -- argos upload ./screenshots/chromium",
+    "upload-text-snapshots": "pnpm exec -- argos upload \"../website/build\" --build-name text-snapshots --files styles.css --files \"docs/**/*.html\" --files \"blog/**/*.html\"",
     "report": "playwright show-report"
   },
   "dependencies": {
-    "@argos-ci/playwright": "^7.0.6",
-    "@playwright/test": "^1.60.0",
+    "@argos-ci/cli": "^6.9.0",
+    "@argos-ci/playwright": "^7.4.6",
+    "@playwright/test": "^1.62.1",
     "cheerio": "^1.2.0"
   }
 }

argos/playwright.config.ts

@@ -5,18 +5,23 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {devices} from '@playwright/test';
-import type {PlaywrightTestConfig} from '@playwright/test';
+import {defineConfig, devices} from '@playwright/test';
+import {createArgosReporterOptions} from '@argos-ci/playwright/reporter';
+
+const argosOptions = createArgosReporterOptions({
+  uploadToArgos: true, // for now, we always upload even outside of CI
+  buildName: 'screenshots',
+});
 
 /**
  * See https://playwright.dev/docs/test-configuration.
  */
-const config: PlaywrightTestConfig = {
+export default defineConfig({
   testDir: './tests',
 
   timeout: 60000,
 
-  reporter: [['list'], ['@argos-ci/playwright/reporter']],
+  reporter: [['list'], ['@argos-ci/playwright/reporter', argosOptions]],
 
   // Run website production built
   // Need to run "pnpm build:website:fast" before
@@ -32,9 +37,12 @@ const config: PlaywrightTestConfig = {
       name: 'chromium',
       use: {
         ...devices['Desktop Chrome'],
+        // Recommended by Argos
+        // See https://argos-ci.com/docs/learn/reliability-and-flakiness/flaky-tests/stabilize-text-rendering
+        launchOptions: {
+          args: ['--disable-lcd-text', '--font-render-hinting=none'],
+        },
       },
     },
   ],
-};
-
-export default config;
+});

argos/tests/screenshot.spec.ts

@@ -169,7 +169,9 @@ function createPathnameTest(pathname: string) {
     await page.waitForFunction(waitForDocusaurusHydration);
     await page.addStyleTag({content: stylesheet});
     // await expect(page).toHaveScreenshot({ fullPage: true, ...options });
-    await argosScreenshot(page, pathnameToArgosName(pathname));
+    await argosScreenshot(page, pathnameToArgosName(pathname), {
+      ariaSnapshot: true,
+    });
   });
 }
 

package.json

@@ -35,9 +35,10 @@
     "crowdin:upload:website": "crowdin upload sources --config ./crowdin-v2.yaml",
     "crowdin:download": "crowdin download --config ./crowdin-v2.yaml",
     "crowdin:download:website": "pnpm crowdin:download --language fr --language ko --language pt-BR --language zh-CN --language ja",
-    "argos": "pnpm argos:build && pnpm argos:screenshot",
+    "argos": "pnpm argos:build && pnpm argos:screenshot && argos:upload-text-snapshots",
     "argos:build": "cross-env DOCUSAURUS_ARGOS_BUILD=true pnpm build:website:fast --dev",
     "argos:screenshot": "pnpm --filter argos screenshot",
+    "argos:upload-text-snapshots": "pnpm --filter argos upload-text-snapshots",
     "canary": "pnpm canary:bumpVersion && pnpm canary:publish",
     "canary:getCoreVersion": "node -p \"require('./packages/docusaurus/package.json').version.split('-')[0]\"",
     "canary:version": "echo `pnpm --silent canary:getCoreVersion`-canary-`git rev-list --count HEAD`+`git rev-parse --short HEAD`",