Merge pull request #3313 from Strategy11/cypress_sharding

Split cypress tests into shards and reduce e2e calls to form templates API

Author: Crabcyborg

.github/workflows/cypress.yml

@@ -20,7 +20,27 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 'run e2e tests')
     runs-on: ubuntu-latest
 
-    name: Cypress
+    strategy:
+      # Report every shard's result. The default would cancel the sibling
+      # shards as soon as one spec fails, hiding failures we want to see in
+      # the same run.
+      fail-fast: false
+      matrix:
+        # Each shard is a separate runner with its own wp-env stack. The
+        # specs assert on global state (searchFunctionality.cy.js creates a
+        # form named "Test Form" and asserts the list count is exactly 1), so
+        # two Cypress processes must never share one WordPress instance -
+        # parallelism has to be at the runner level, not within a job.
+        #
+        # Each shard re-pays the fixed setup cost (npm ci + wp-env booting
+        # WordPress and MySQL), so raising this past ~6 buys little.
+        shard: [ 0, 1, 2, 3 ]
+
+    name: Cypress (shard ${{ matrix.shard }})
+
+    env:
+      SHARD_TOTAL: 4
+
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -46,8 +66,53 @@ jobs:
         if: steps.cache-npm.outputs.cache-hit != 'true'
         run: npm ci --legacy-peer-deps --include=dev
 
+      # Serve the form-templates API from a fixture instead of calling
+      # formidableforms.com on every run. The cache key is the UTC date, so
+      # the fixture is refreshed at most once a day and all four shards share
+      # the one copy - without this, each shard fetches it separately. If the
+      # fetch fails the committed fixture is used, so the suite still runs
+      # when the API is unreachable.
+      - name: Get current date
+        id: date
+        run: echo "date=$( date -u +%Y-%m-%d )" >> "$GITHUB_OUTPUT"
+
+      - name: Cache form templates API response
+        id: cache-templates
+        uses: actions/cache@v4
+        with:
+          path: tests/mu-plugins/form-templates-api.json
+          key: form-templates-${{ steps.date.outputs.date }}
+
+      - name: Refresh form templates fixture
+        if: steps.cache-templates.outputs.cache-hit != 'true'
+        run: |
+          # Write to a temp file first so a partial or failed response never
+          # clobbers the committed fallback.
+          if curl -fsS --max-time 40 -A 'formidable-e2e' \
+            'https://formidableforms.com/wp-json/form-templates/v1/list?l=RlJFRVRFTVBMQVRFUyEhIQ%3D%3D' \
+            -o /tmp/form-templates-api.json && [ -s /tmp/form-templates-api.json ]; then
+            mv /tmp/form-templates-api.json tests/mu-plugins/form-templates-api.json
+            echo "Refreshed the form templates fixture from the API."
+          else
+            echo "Could not reach the templates API; using the committed fixture."
+          fi
+
       - name: Set up WP environment
         run: npm run env start
 
       - name: Test
-        run: npm run e2e:githubrun
+        run: |
+          specs="$( ./tests/bin/split-specs.sh "$SHARD_TOTAL" "${{ matrix.shard }}" )"
+
+          # An empty shard only happens if there are fewer specs than shards.
+          # Bail out rather than calling Cypress with an empty --spec, which
+          # would silently fall back to running the whole suite.
+          if [ -z "$specs" ]; then
+            echo "No specs assigned to shard ${{ matrix.shard }}; nothing to run."
+            exit 0
+          fi
+
+          echo "Shard ${{ matrix.shard }} of $SHARD_TOTAL running:"
+          echo "$specs" | tr ',' '\n' | sed 's/^/  /'
+
+          npm run e2e:githubrun -- --spec "$specs"

.wp-env.json

@@ -4,11 +4,15 @@
   "plugins": [
     "."
   ],
+  "mappings": {
+    "wp-content/mu-plugins": "./tests/mu-plugins"
+  },
   "env": {
     "tests": {
       "mappings": {
+        "wp-content/mu-plugins": "./tests/mu-plugins",
         "wp-cli.yml": "./tests/bin/wp-cli.yml"
       }
     }
   }
-}
+}

tests/bin/split-specs.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Print the comma-separated Cypress `--spec` list for one shard of a
+# parallel E2E run.
+#
+# Usage: tests/bin/split-specs.sh <total_shards> <shard_index>
+#   total_shards - how many shards the run is split across
+#   shard_index  - 0-based index of the shard to print
+#
+# Specs are discovered by globbing, not listed explicitly, so a newly added
+# spec is always picked up by some shard instead of being silently dropped.
+#
+# Balancing uses line count as a stand-in for runtime, assigning the largest
+# spec first to whichever shard is currently lightest (greedy longest-
+# processing-time). Plain round-robin lands the two biggest specs on the same
+# shard and leaves it running ~4.5x longer than the shortest one; since the
+# whole job is only as fast as its slowest shard, that wastes most of the
+# parallelism. Line count is a rough proxy - swap in real per-spec durations
+# here if the shards drift out of balance.
+
+set -euo pipefail
+
+total="${1:?usage: split-specs.sh <total_shards> <shard_index>}"
+index="${2:?usage: split-specs.sh <total_shards> <shard_index>}"
+
+if [ "$total" -lt 1 ] || [ "$index" -lt 0 ] || [ "$index" -ge "$total" ]; then
+	echo "split-specs.sh: shard index $index out of range for $total shards" >&2
+	exit 1
+fi
+
+cd "$( dirname "$0" )/../.."
+
+# Keep the sort locale-independent so every shard in a run agrees on the
+# ordering, and therefore on the partition.
+export LC_ALL=C
+
+find tests/cypress/e2e -type f \
+	\( -name '*.cy.js' -o -name '*.cy.jsx' -o -name '*.cy.ts' -o -name '*.cy.tsx' \) \
+	-exec sh -c 'for f do printf "%s\t%s\n" "$( wc -l < "$f" )" "$f"; done' sh {} + |
+	sort -t"$( printf '\t' )" -k1,1nr -k2,2 |
+	awk -F"$( printf '\t' )" -v total="$total" -v want="$index" '
+		{
+			best = 0;
+			for ( s = 1; s < total; s++ ) {
+				if ( load[ s ] < load[ best ] ) {
+					best = s;
+				}
+			}
+			load[ best ] += $1;
+			if ( best == want ) {
+				out = ( out == "" ? $2 : out "," $2 );
+			}
+		}
+		END { print out }
+	'

tests/mu-plugins/form-templates-api.json

@@ -0,0 +1,29 @@
+<?php
+// Forces the welcome-tour checklist off in the e2e test environment.
+//
+// On a fresh install the checklist renders expanded on its 'create-form'
+// step and its header overlays the form builder, covering
+// #frm-save-form-name-button so `cy.click()` fails actionability.
+//
+// The suite used to get away with this by accident: Cypress runs root-level
+// specs first, so admin-a11y.cy.js was always spec 1 of 18 and clicked the
+// checklist away, and because FrmWelcomeTourController stores its state in
+// the site-wide `frm-welcome-tour` option, every later spec inherited a
+// dismissed tour. Sharding removes that guarantee - whichever spec runs
+// first on a given shard now meets a virgin site - so the state has to be
+// set deterministically instead of inherited from a neighbour.
+//
+// Filtering rather than seeding the option means save_checklist() can't
+// write over it mid-run. `active_step_key` is required alongside
+// `dismissed`: get_usage_data() reads it whenever `dismissed` is truthy and
+// would otherwise emit an undefined-index warning into debug.log.
+add_filter(
+	'pre_option_frm-welcome-tour',
+	function () {
+		return array(
+			'completed_steps' => array(),
+			'active_step_key' => 'create-form',
+			'dismissed'       => true,
+		);
+	}
+);

tests/mu-plugins/frm-dismiss-welcome-tour.php

@@ -0,0 +1,57 @@
+<?php
+// Serves the form-templates API from a local fixture in the e2e test
+// environment, so the suite never depends on a live call to
+// formidableforms.com.
+//
+// Two things go wrong without this. A fresh wp-env has an empty template
+// cache, and FrmFormApi::get_api_info() returns an empty array outright when
+// another request for the same data is already in flight (the is_running()
+// guard) rather than waiting for it - so the templates list renders empty and
+// assertions like [frm-search-text="user registration"] never match. The
+// `Get Instant Access` signup flow then calls reset_cached(), clearing the
+// cache mid-run and re-opening the same window for whatever runs next.
+//
+// Sharding sharpened this: every shard is its own wp-env with its own cold
+// cache, so one live fetch per run became one per shard.
+//
+// Filtering pre_http_request rather than seeding the cache option keeps this
+// independent of FrmFormApi's cache internals - the option key is derived
+// from the license, and reset_cached() deletes it - and it is the standard
+// WordPress way to stub an outbound request.
+//
+// The fixture sits beside this file on purpose. The whole mu-plugins
+// directory is mapped to a known path inside the container, whereas the
+// plugin directory is named `formidable` locally and `formidable-forms` in
+// CI, so a path built from the plugin folder would break in one of the two.
+// WordPress only auto-loads .php from the mu-plugins root, so the .json
+// sibling is inert.
+add_filter(
+	'pre_http_request',
+	function ( $preempt, $args, $url ) {
+		if ( false === strpos( $url, '/wp-json/form-templates/v1/list' ) ) {
+			return $preempt;
+		}
+
+		$fixture = __DIR__ . '/form-templates-api.json';
+
+		if ( ! is_readable( $fixture ) ) {
+			// Let the real request through rather than failing the run with an
+			// empty template list, which is the confusing symptom this exists
+			// to prevent.
+			return $preempt;
+		}
+
+		return array(
+			'headers'  => array(),
+			'body'     => file_get_contents( $fixture ),
+			'response' => array(
+				'code'    => 200,
+				'message' => 'OK',
+			),
+			'cookies'  => array(),
+			'filename' => null,
+		);
+	},
+	10,
+	3
+);