Build/Test Tools: Retry failed and stalled downloads in CI setup.

From October 15 to 28, 13 of 523 trunk Performance Tests runs needed a rerun because a setup step failed, and three Playwright installs hung for six hours.

Add two composite actions. `download-with-retry` downloads with curl, retrying on any error and on stalled transfers. `run-with-retry` runs a command up to three times. Use them for the Gutenberg download in E2E tests and the WordPress Importer and mock data downloads in performance tests. Also use them for the WordPress downloads in the installation, upgrade, and performance baseline steps. Retry the global npm install in the `setup-npm` action.

Also cap the Playwright browser install and the mock data import at 15 minutes each, and skip archiving performance artifacts when the tests never ran.

Developed in: #15210

Props lucatume, johnbillion.
See #65845.


git-svn-id: https://develop.svn.wordpress.org/trunk@65266 602fd350-edb4-49c9-b593-d223f7449a82

Author: lancewillett

.github/actions/download-with-retry/action.yml

@@ -0,0 +1,23 @@
+name: 'Download with retry'
+description: 'Downloads a file with curl, retrying on any error.'
+
+inputs:
+  url:
+    description: 'The URL to download.'
+    required: true
+  output:
+    description: 'The path to save the file to, relative to the workspace.'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Download the file
+      shell: bash
+      # A stalled attempt (connecting for over 30 s, or under 1 KB/s for 30 s) fails and is
+      # retried: curl has no transfer timeout, and `timeout-minutes` would end the step unretried.
+      # No retry starts after 10 minutes, so a long `Retry-After` can't stretch the waits.
+      run: curl --fail --no-progress-meter --location --retry 5 --retry-all-errors --retry-max-time 600 --connect-timeout 30 --speed-limit 1024 --speed-time 30 --output "$OUTPUT" "$URL"
+      env:
+        URL: ${{ inputs.url }}
+        OUTPUT: ${{ inputs.output }}

.github/actions/run-with-retry/action.yml

@@ -0,0 +1,28 @@
+name: 'Run with retry'
+description: 'Runs a Bash command, retrying it when it fails. For downloads curl can retry itself, use download-with-retry.'
+
+inputs:
+  command:
+    description: 'The Bash command to run. It runs as code: read values from the `env` of the calling step, not from expressions.'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Run the command
+      shell: bash
+      # Waits 10 s, then 20 s, between attempts, like `tools/local-env/scripts/utils.js`.
+      run: |
+        for (( attempt = 1; attempt <= MAX_ATTEMPTS; attempt++ )); do
+          if bash -eo pipefail -c "$COMMAND"; then
+            exit 0
+          fi
+          echo "::warning::Command failed on attempt $attempt of $MAX_ATTEMPTS."
+          if (( attempt < MAX_ATTEMPTS )); then
+            sleep $(( attempt * 10 ))
+          fi
+        done
+        exit 1
+      env:
+        COMMAND: ${{ inputs.command }}
+        MAX_ATTEMPTS: 3

.github/actions/setup-npm/action.yml

@@ -18,11 +18,22 @@ runs:
       id: npm
       env:
         WORKING_DIRECTORY: ${{ inputs.working-directory }}
+        NPM_INSTALL_MAX_ATTEMPTS: 3
       run: |
         NPM_VERSION="$(jq -r '.devEngines.packageManager.version // empty' package.json)"
         if [ -n "$NPM_VERSION" ]; then
           # Contributors need the floor; CI stays on that major's newest release.
-          npm install --global "npm@${NPM_VERSION/>=/^}"
+          # The registry can reset the connection mid-download (ECONNRESET), so retry.
+          for (( attempt = 1; attempt <= NPM_INSTALL_MAX_ATTEMPTS; attempt++ )); do
+            if npm install --global "npm@${NPM_VERSION/>=/^}"; then
+              break
+            fi
+            echo "::warning::npm install failed on attempt $attempt of $NPM_INSTALL_MAX_ATTEMPTS."
+            if (( attempt == NPM_INSTALL_MAX_ATTEMPTS )); then
+              exit 1
+            fi
+            sleep $(( attempt * 10 ))
+          done
           echo "Required npm version: $NPM_VERSION"
         else
           echo "No devEngines.packageManager.version, keeping the bundled npm."

.github/workflows/end-to-end-tests.yml

@@ -44,6 +44,7 @@ on:
       - 'tools/vendors/**'
       - 'tools/webpack/**'
       - '.github/actions/setup-npm/**'
+      - '.github/actions/download-with-retry/**'
       - '.github/workflows/end-to-end-tests.yml'
       - '.github/workflows/reusable-end-to-end-tests*.yml'
   workflow_dispatch:

.github/workflows/install-testing.yml

@@ -13,6 +13,7 @@ on:
     # Always test the workflow after it's updated.
     paths:
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.github/workflows/install-testing.yml'
       - '.version-support-*.json'
@@ -21,6 +22,7 @@ on:
     # Always test the workflow when changes are suggested.
     paths:
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.version-support-*.json'
       - '.github/workflows/install-testing.yml'
@@ -143,11 +145,15 @@ jobs:
       # WP-CLI 2.x extracts tarballs with PharData, which truncates paths longer than 100 characters.
       # Zip archives are not affected, so download the zip for every version.
       - name: Download WordPress
-        run: |
-          case "${WP_VERSION}" in
-            nightly|trunk) wp core download --version=nightly ;;
-            *) wp core download "https://wordpress.org/wordpress-${WP_VERSION}.zip" ;;
-          esac
+        # WP-CLI requests WordPress from WordPress.org once, so a network error fails the job.
+        # It extracts only after a complete download, so a retry starts from an empty directory.
+        uses: $/.github/actions/run-with-retry
+        with:
+          command: |
+            case "${WP_VERSION}" in
+              nightly|trunk) wp core download --version=nightly ;;
+              *) wp core download "https://wordpress.org/wordpress-${WP_VERSION}.zip" ;;
+            esac
         env:
           WP_VERSION: ${{ inputs.wp-version || 'nightly' }}
 

.github/workflows/performance.yml

@@ -44,7 +44,9 @@ on:
       - 'tools/vendors/**'
       - 'tools/webpack/**'
       - '.github/actions/setup-npm/**'
+      - '.github/actions/download-with-retry/**'
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/workflows/performance.yml'
       - '.github/workflows/reusable-performance-*.yml'
   workflow_dispatch:

.github/workflows/reusable-end-to-end-tests.yml

@@ -136,14 +136,22 @@ jobs:
           LOCAL_SCRIPT_DEBUG: ${{ inputs.LOCAL_SCRIPT_DEBUG }}
         run: npm run env:install
 
+      - name: Download Gutenberg
+        if: ${{ inputs.install-gutenberg }}
+        # Downloaded on the runner so failed downloads are retried: WP-CLI tries once.
+        # Release branches run this workflow from trunk: `$/` resolves to this workflow's commit, not the caller's.
+        uses: $/.github/actions/download-with-retry
+        with:
+          url: https://downloads.wordpress.org/plugin/gutenberg.${{ inputs.gutenberg-version || 'latest-stable' }}.zip
+          output: gutenberg.zip
+
       - name: Install Gutenberg
         if: ${{ inputs.install-gutenberg }}
         run: |
-          npm run env:cli -- plugin install gutenberg \
-          ${{ inputs.gutenberg-version && '--version="${GUTENBERG_VERSION}"' || '' }} \
-          --path="/var/www/${LOCAL_DIR}"
-        env:
-          GUTENBERG_VERSION: ${{ inputs.gutenberg-version }}
+          npm run env:cli -- plugin install gutenberg.zip --path="/var/www/${LOCAL_DIR}"
+          rm gutenberg.zip
+          # Installing from a zip doesn't log the version; unpinned branches get the latest.
+          npm run env:cli -- plugin get gutenberg --field=version --path="/var/www/${LOCAL_DIR}"
 
       - name: Install additional languages
         run: |

.github/workflows/reusable-performance-test-v2.yml

@@ -159,6 +159,9 @@ jobs:
         run: npm ci
 
       - name: Install Playwright browsers
+        # A hung install would otherwise run until the 6 hour job limit. The cap
+        # leaves room for slow downloads.
+        timeout-minutes: 15
         run: npm exec --no -- playwright install --with-deps chromium
 
       - name: Start Docker environment
@@ -202,9 +205,11 @@ jobs:
 
       - name: Set the environment to the baseline version
         if: ${{ inputs.subject == 'base' }}
-        run: |
-          VERSION="${BASE_TAG%.0}"
-          wp core download --version="$VERSION" --force --path="${LOCAL_DIR}"
+        # WP-CLI requests the release from WordPress.org once, so a network error fails the job.
+        # It extracts only after a complete download, and `--force` lets a retry overwrite files.
+        uses: $/.github/actions/run-with-retry
+        with:
+          command: wp core download --version="${BASE_TAG%.0}" --force --path="${LOCAL_DIR}"
 
       - name: Install object cache drop-in
         if: ${{ inputs.memcached }}
@@ -236,12 +241,29 @@ jobs:
           npm run env:cli -- theme enable twentytwentyfour --network --path="/var/www/${LOCAL_DIR}"
           npm run env:cli -- theme enable twentytwentyfive --network --path="/var/www/${LOCAL_DIR}"
 
+      - name: Download WordPress Importer plugin
+        # Downloaded on the runner so failed downloads are retried: WP-CLI tries once.
+        uses: $/.github/actions/download-with-retry
+        with:
+          url: https://downloads.wordpress.org/plugin/wordpress-importer.latest-stable.zip
+          output: wordpress-importer.zip
+
       - name: Install WordPress Importer plugin
-        run: npm run env:cli -- plugin install wordpress-importer --activate --path="/var/www/${LOCAL_DIR}"
+        run: |
+          npm run env:cli -- plugin install wordpress-importer.zip --activate --path="/var/www/${LOCAL_DIR}"
+          rm wordpress-importer.zip
+
+      - name: Download mock data
+        uses: $/.github/actions/download-with-retry
+        with:
+          url: https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml
+          output: themeunittestdata.wordpress.xml
 
       - name: Import mock data
+        # The importer waits up to 300 s per attachment download and skips the ones that
+        # fail: the cap leaves room for a stalled attachment, not a hung import.
+        timeout-minutes: 15
         run: |
-          curl -O https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml
           npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path="/var/www/${LOCAL_DIR}"
           rm themeunittestdata.wordpress.xml
 
@@ -271,13 +293,15 @@ jobs:
           cp ./tests/performance/wp-content/mu-plugins/server-timing.php "./${LOCAL_DIR}/wp-content/mu-plugins/server-timing.php"
 
       - name: Run performance tests
+        id: performance-tests
         run: npm run test:performance
         env:
           TEST_RESULTS_PREFIX: ${{ inputs.subject != 'current' && inputs.subject || '' }}
 
       - name: Archive artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        if: always()
+        # Without test results, a failed setup step would report a second failure here.
+        if: ${{ always() && steps.performance-tests.outcome != 'skipped' }}
         with:
           name: performance-${{ inputs.multisite && 'multisite' || 'single' }}-${{ inputs.memcached && 'memcached' || 'default' }}-${{ inputs.subject }}
           path: artifacts

.github/workflows/reusable-upgrade-testing.yml

@@ -95,7 +95,11 @@ jobs:
         uses: $/.github/actions/setup-wp-cli
 
       - name: Download WordPress ${{ inputs.wp }}
-        run: wp core download --version="${WP_VERSION}"
+        # WP-CLI requests the release from WordPress.org once, so a network error fails the job.
+        # It extracts only after a complete download, so a retry starts from an empty directory.
+        uses: $/.github/actions/run-with-retry
+        with:
+          command: wp core download --version="${WP_VERSION}"
         env:
           WP_VERSION: ${{ inputs.wp }}
 

.github/workflows/upgrade-develop-testing.yml

@@ -15,6 +15,7 @@ on:
       - '*.php' # PHP files in the root, like wp-config-sample.php.
       # Confirm any changes to relevant workflow files.
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/actions/smoke-check/**'
       - '.github/actions/setup-npm/**'
       - '.github/workflows/upgrade-develop-testing.yml'
@@ -31,6 +32,7 @@ on:
       # Confirm any changes to relevant workflow files.
       - '.github/actions/setup-npm/**'
       - '.github/actions/setup-wp-cli/**'
+      - '.github/actions/run-with-retry/**'
       - '.github/workflows/upgrade-develop-testing.yml'
       - '.github/workflows/reusable-upgrade-testing.yml'
   schedule: