GitHub Actions: document DISABLED_TESTS and count the dogfood plugins (#195)

Author: drowaudio

.github/actions/README.md

@@ -42,6 +42,23 @@ Every `pluginval validate` option can also be set as an environment variable on
 the step (e.g. `env: { TIMEOUT_MS: 60000 }`), which is the way to pass values
 containing spaces.
 
+To skip tests, set `DISABLED_TESTS` to their names, comma-separated (or to an
+absolute path to a file listing one name per line). Test names contain spaces,
+so they can't be given as `--disabled-tests` in `args`. For example, at strictness
+8 and above "Discrete parameter state restoration" fails for plugins using
+`juce::AudioParameterBool`, until [JUCE snaps its value to 0 or 1](https://forum.juce.com/t/pluginval-can-set-an-audioparameterbools-to-odd-values/50971):
+
+```yaml
+- uses: Tracktion/pluginval/validate@v2
+  env:
+    DISABLED_TESTS: Discrete parameter state restoration
+  with:
+    plugins: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3
+    strictness-level: 10
+```
+
+Run `pluginval strictness-help <level>` to list the test names.
+
 ## acceptance
 
 Runs the [acceptance tests](../../docs/Acceptance%20testing.md) for every config

.github/workflows/acceptance.yml

@@ -81,6 +81,14 @@ jobs:
             *)       echo "pluginval=build/pluginval_artefacts/Release/pluginval" >> "$GITHUB_OUTPUT" ;;
           esac
 
+          # Every dogfood plugin is validated, so count them rather than hard-coding
+          # the number the check step expects.
+          shopt -s nullglob
+          glob='build/tests/test_plugins/*/*_artefacts/Release/VST3/*.vst3'
+          plugins=( $glob )
+          echo "plugin-glob=$glob" >> "$GITHUB_OUTPUT"
+          echo "plugin-count=${#plugins[@]}" >> "$GITHUB_OUTPUT"
+
           # Tone generator configs with the plugin removed, so it has to come from
           # the action's "plugin" input. One is nested and a reference manifest is
           # added, to check the recursion and that manifests aren't run as configs.
@@ -104,7 +112,7 @@ jobs:
         with:
           pluginval-path: ${{ steps.fixtures.outputs.pluginval }}
           args: --skip-gui-tests
-          plugins: build/tests/test_plugins/*/*_artefacts/Release/VST3/*.vst3
+          plugins: ${{ steps.fixtures.outputs.plugin-glob }}
 
       - name: "Action: validate (missing plugin, expected to fail)"
         id: validate-missing
@@ -156,7 +164,8 @@ jobs:
         shell: bash
         run: |
           check() { echo "$1: got '$2', expected '$3'"; [ "$2" = "$3" ] || exit 1; }
-          check "validate passed"            "${{ steps.validate.outputs.passed }}"           4
+          [ "${{ steps.fixtures.outputs.plugin-count }}" -gt 0 ] || { echo "No dogfood plugins found"; exit 1; }
+          check "validate passed"            "${{ steps.validate.outputs.passed }}"           "${{ steps.fixtures.outputs.plugin-count }}"
           check "validate failed"            "${{ steps.validate.outputs.failed }}"           0
           check "validate-missing outcome"   "${{ steps.validate-missing.outcome }}"          failure
           check "validate-missing passed"    "${{ steps.validate-missing.outputs.passed }}"   1