Merge pull request #1580 from coseeian/a11y-ci-scan

feat: run Playwright accessibility scan on pull requests

Author: ksen0

.github/workflows/test.yml

@@ -3,6 +3,10 @@ name: Test, typecheck, and lint
 on:
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -48,6 +52,7 @@ jobs:
         run: npm run check
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Checkout your repository using git
         uses: actions/checkout@v6
@@ -62,3 +67,17 @@ jobs:
           SKIP_BUILD_COMPRESS: 1
           NODE_OPTIONS: "--max_old_space_size=4096"
         run: npm run build
+      - name: Install Playwright browser
+        run: npx playwright install --with-deps chromium
+      - name: Run accessibility tests
+        continue-on-error: true
+        run: npm run test:a11y
+      - name: Upload Playwright report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: |
+            playwright-report/
+            test-results/
+          retention-days: 14

playwright.config.ts

@@ -2,16 +2,24 @@ import { defineConfig, devices } from "@playwright/test";
 
 const PORT = 4321;
 const BASE_URL = `http://localhost:${PORT}`;
+const isCI = Boolean(process.env.CI);
 
 export default defineConfig({
   testDir: "./test/a11y",
   testMatch: "**/*.spec.ts",
   outputDir: "./test-results",
   fullyParallel: true,
-  reporter: [
-    ["list"],
-    ['html', { outputFolder: 'playwright-report', open: 'never' }]
-  ],
+  forbidOnly: isCI,
+  reporter: isCI
+    ? [
+        ["github"],
+        ["list"],
+        ["html", { outputFolder: "playwright-report", open: "never" }],
+      ]
+    : [
+        ["list"],
+        ['html', { outputFolder: 'playwright-report', open: 'never' }]
+      ],
 
   use: {
     baseURL: BASE_URL,
@@ -34,10 +42,12 @@ export default defineConfig({
   ],
 
   webServer: {
-    command: "npm run dev",
+    // CI serves the dist/ built by an earlier step; see .github/workflows/test.yml.
+    command: isCI ? "npm run preview" : "npm run dev",
     url: BASE_URL,
-    reuseExistingServer: true,
+    reuseExistingServer: !isCI,
     timeout: 180_000,
+    stdout: isCI ? "pipe" : "ignore",
     env: {
       A11Y_TEST: "1",
     },

test/a11y/homepage.spec.ts

@@ -0,0 +1,36 @@
+import AxeBuilder from "@axe-core/playwright";
+import { expect, test } from "@playwright/test";
+
+const pagesToScan = [
+  { name: "home", path: "/" },
+  { name: "about", path: "/about/" },
+  { name: "community", path: "/community/" },
+  { name: "contribute", path: "/contribute/" },
+  { name: "education resources", path: "/education-resources/" },
+  { name: "events", path: "/events/" },
+  { name: "libraries", path: "/libraries/" },
+  { name: "people", path: "/people/" },
+  { name: "sketches", path: "/sketches/" },
+  { name: "search", path: "/search/" },
+  { name: "tutorials index", path: "/tutorials/" },
+  { name: "tutorial", path: "/tutorials/get-started/" },
+  { name: "examples index", path: "/examples/" },
+  { name: "example", path: "/examples/Shapes-And-Color-Shape-Primitives/" },
+  { name: "reference index", path: "/reference/" },
+  { name: "reference function", path: "/reference/p5/ellipse/" },
+  { name: "reference class method", path: "/reference/p5.Vector/add/" },
+  { name: "reference p5.sound", path: "/reference/p5.sound/" },
+  { name: "404", path: "/no-such-page/" },
+];
+
+test.describe("a11y", () => {
+  for (const { name, path } of pagesToScan) {
+    test(`${name} should not have any automatically detectable accessibility issues`, async ({
+      page,
+    }) => {
+      await page.goto(path);
+      const accessibilityScanResults = await new AxeBuilder({ page }).analyze();
+      expect(accessibilityScanResults.violations).toEqual([]);
+    });
+  }
+});