Skip to content

Commit f9d152a

Browse files
committed
Run the E2E test scenarios as bashunit tests, sharded across runners
Each script of the E2E workflow matrix becomes a bashunit test function next to its fixture, in e2e/<dir>/<dir>_test.sh. bashunit collects them, and --shard spreads them over a few jobs: the result cache tests run in 4 jobs instead of 118, the other E2E tests in 2 shards plus 9 single-test rows that need a PHP setup of their own (instead of 47 jobs), and the Windows result cache tests in 1 job instead of 5. Tags pick the pool: result-cache tests need the patched Error.php, own-setup tests keep their own matrix row, windows tests run on windows-latest. .bashunitrc turns on strict mode, so a failing command fails the test as it failed the step before, and loads e2e/bootstrap.sh: a test runs in its fixture directory, with a TMPDIR of its own, and the directory is put back once the test ends. Several tests share and change one directory, and each of them used to start on a fresh checkout. Run them locally from the repository root: e2e/bashunit test e2e Amp-Thread-ID: https://ampcode.com/threads/T-01a1b596-a698-7607-9f34-a62afb8c4731
1 parent 702f3b0 commit f9d152a

141 files changed

Lines changed: 4010 additions & 2169 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.bashunitrc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# E2E tests in e2e/, run from the repository root: e2e/bashunit test e2e
2+
BASHUNIT_BOOTSTRAP=e2e/bootstrap.sh
3+
BASHUNIT_STRICT_MODE=true

‎.github/workflows/e2e-tests.yml‎

Lines changed: 25 additions & 2169 deletions
Large diffs are not rendered by default.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
/tests/bench/storage/local-baseline.xml
1515
tmp/.memory_limit
1616
e2e/bashunit
17+
/.bashunit
1718
/.phpbench
1819
/.amp/portals/
1920
/e2e/turbo-arena/tmp
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env bash
2+
# @tags result-cache
3+
4+
function set_up() {
5+
e2e::set_up
6+
}
7+
8+
function tear_down() {
9+
e2e::tear_down
10+
}
11+
12+
function test_invalid_ignore_errors_path() {
13+
OUTPUT=$(../../bin/phpstan analyse -c ignore.neon 2>&1) && CODE=0 || CODE=$?
14+
assert_same "1" "$CODE"
15+
echo "$OUTPUT"
16+
assert_contains 'Invalid entry in ignoreErrors' "$OUTPUT"
17+
assert_contains 'tests" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
18+
}
19+
20+
function test_invalid_ignore_errors_path_in_php_config() {
21+
OUTPUT=$(../../bin/phpstan analyse -c phpneon.php 2>&1) && CODE=0 || CODE=$?
22+
assert_same "1" "$CODE"
23+
echo "$OUTPUT"
24+
assert_contains 'Invalid entry in ignoreErrors' "$OUTPUT"
25+
assert_contains '"src/test.php" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
26+
}
27+
28+
function test_invalid_exclude_path() {
29+
OUTPUT=$(../../bin/phpstan analyse -c excludePaths.neon 2>&1) && CODE=0 || CODE=$?
30+
assert_same "1" "$CODE"
31+
echo "$OUTPUT"
32+
assert_contains 'Invalid entry in excludePaths' "$OUTPUT"
33+
assert_contains 'tests" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
34+
}
35+
36+
function test_invalid_exclude_path_in_php_config() {
37+
OUTPUT=$(../../bin/phpstan analyse -c phpneon2.php 2>&1) && CODE=0 || CODE=$?
38+
assert_same "1" "$CODE"
39+
echo "$OUTPUT"
40+
assert_contains 'Invalid entry in excludePaths' "$OUTPUT"
41+
assert_contains '"src/test.php" is neither a directory, nor a file path, nor a fnmatch pattern.' "$OUTPUT"
42+
}
43+
44+
function test_optional_exclude_path_that_does_not_exist() {
45+
OUTPUT=$(../../bin/phpstan analyse -c ignoreNonexistentExcludePath.neon)
46+
echo "$OUTPUT" && CODE=0 || CODE=$?
47+
assert_same "0" "$CODE"
48+
}
49+
50+
function test_optional_exclude_path_that_exists() {
51+
cp -r tmp-node-modules node_modules
52+
OUTPUT=$(../../bin/phpstan analyse -c ignoreNonexistentExcludePath.neon)
53+
echo "$OUTPUT" && CODE=0 || CODE=$?
54+
assert_same "0" "$CODE"
55+
}
56+
57+
function test_unmatched_ignore_path_not_reported() {
58+
OUTPUT=$(../../bin/phpstan analyse -c ignoreReportUnmatchedFalse.neon)
59+
echo "$OUTPUT" && CODE=0 || CODE=$?
60+
assert_same "0" "$CODE"
61+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
3+
function set_up() {
4+
e2e::set_up
5+
}
6+
7+
function tear_down() {
8+
e2e::tear_down
9+
}
10+
11+
function test_generated_baseline_with_debug() {
12+
../../bin/phpstan analyse --debug --configuration test-no-baseline.neon --generate-baseline test-baseline.neon
13+
../../bin/phpstan analyse --debug --configuration test.neon && CODE=0 || CODE=$?
14+
assert_same "0" "$CODE"
15+
}
16+
17+
function test_generated_baseline() {
18+
../../bin/phpstan analyse --configuration test-no-baseline.neon --generate-baseline test-baseline.neon
19+
../../bin/phpstan analyse --configuration test.neon && CODE=0 || CODE=$?
20+
assert_same "0" "$CODE"
21+
}
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
# @tags result-cache
3+
4+
function set_up() {
5+
e2e::set_up
6+
}
7+
8+
function tear_down() {
9+
e2e::tear_down
10+
}
11+
12+
function test_bootstrap_autoloader_wrong_file() {
13+
# The crash needs OPcache: forked workers switch it off, the --debug main process and
14+
# spawned workers (pcntl_fork disabled, as on Windows) keep it on.
15+
php -r 'if (!extension_loaded("Zend OPcache")) { fwrite(STDERR, "OPcache is not loaded, so this step cannot reproduce the crash.\n"); exit(1); }'
16+
../../bin/phpstan analyze --debug
17+
php -d disable_functions=pcntl_fork ../../bin/phpstan analyze && CODE=0 || CODE=$?
18+
assert_same "0" "$CODE"
19+
}

‎e2e/bootstrap.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
# Loaded by bashunit before the E2E tests, see BASHUNIT_BOOTSTRAP in .bashunitrc.
3+
# Run the tests from the repository root: e2e/bashunit test e2e
4+
5+
# A test runs in the directory of its test file, which it is free to change: the
6+
# directory is put back the way it was once the test ends, so the next test using
7+
# it - or the next run - starts from what is in the repository. The test also gets
8+
# a TMPDIR of its own, as PHPStan keeps its default tmpDir - and with it the result
9+
# cache - in sys_get_temp_dir().
10+
function e2e::set_up() {
11+
E2E_TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)
12+
E2E_TMPDIR=$(mktemp -d)
13+
cp -a "$E2E_TEST_DIR/." "$E2E_TMPDIR/snapshot"
14+
mkdir "$E2E_TMPDIR/tmp"
15+
export TMPDIR="$E2E_TMPDIR/tmp"
16+
cd "$E2E_TEST_DIR"
17+
}
18+
19+
function e2e::tear_down() {
20+
find "${E2E_TEST_DIR:?}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
21+
cp -a "${E2E_TMPDIR:?}/snapshot/." "$E2E_TEST_DIR"
22+
rm -rf "$E2E_TMPDIR"
23+
}

‎e2e/bug-10483/bug-10483_test.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
# @tags result-cache
3+
4+
function set_up() {
5+
e2e::set_up
6+
}
7+
8+
function tear_down() {
9+
e2e::tear_down
10+
}
11+
12+
function test_bug_10483() {
13+
../../bin/phpstan && CODE=0 || CODE=$?
14+
assert_same "0" "$CODE"
15+
}

‎e2e/bug-11359/bug-11359_test.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
3+
function set_up() {
4+
e2e::set_up
5+
}
6+
7+
function tear_down() {
8+
e2e::tear_down
9+
}
10+
11+
# @tag own-setup
12+
function test_mb_ucfirst_not_found_on_php_8_2() {
13+
OUTPUT=$(../../bin/phpstan analyse --no-progress --error-format raw --level=8 bug-11359.php 2>&1) && CODE=0 || CODE=$?
14+
assert_same "1" "$CODE"
15+
echo "$OUTPUT"
16+
assert_contains 'e2e/bug-11359/bug-11359.php:3:Function mb_ucfirst not found.' "$OUTPUT"
17+
}
18+
19+
# @tag own-setup
20+
function test_mb_ucfirst_found_on_php_8_4() {
21+
OUTPUT=$(../../bin/phpstan analyse --no-progress --error-format raw --level=8 bug-11359.php 2>&1) && CODE=0 || CODE=$?
22+
assert_same "0" "$CODE"
23+
echo "$OUTPUT"
24+
assert_not_contains 'Function mb_ucfirst not found.' "$OUTPUT"
25+
}

‎e2e/bug-11819/bug-11819_test.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
function set_up() {
4+
e2e::set_up
5+
}
6+
7+
function tear_down() {
8+
e2e::tear_down
9+
}
10+
11+
function test_bug_11819() {
12+
../../bin/phpstan && CODE=0 || CODE=$?
13+
assert_same "0" "$CODE"
14+
}

0 commit comments

Comments
 (0)