Skip to content

Commit cea0bbf

Browse files
Sander Mullerclaude
Sander Muller
and
claude
committed
Benchmark the OPcache file cache: timestamps, empty cache, larger projects, spawned workers, concurrency
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 562cb94 commit cea0bbf

9 files changed

Lines changed: 216 additions & 40 deletions

File tree

‎.github/workflows/tiny-run-bench.yml‎

Lines changed: 33 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,11 @@ jobs:
1313
strategy:
1414
fail-fast: false
1515
matrix:
16-
os: [ubuntu-24.04, macos-15]
16+
os: [ubuntu-24.04, macos-15, windows-2022]
1717
runs-on: ${{ matrix.os }}
18+
defaults:
19+
run:
20+
shell: bash
1821
steps:
1922
- uses: actions/checkout@v4
2023
- uses: shivammathur/setup-php@v2
@@ -25,32 +28,35 @@ jobs:
2528
- name: Environment
2629
run: |
2730
php -v
28-
php -r 'foreach (["opcache.enable_cli", "opcache.file_cache", "opcache.jit"] as $k) echo $k, "=", var_export(ini_get($k), true), "\n";'
29-
uname -a
30-
- name: Build arms
31+
php -r 'foreach (["opcache.enable_cli", "opcache.file_cache", "opcache.jit", "disable_functions"] as $k) echo $k, "=", var_export(ini_get($k), true), "\n"; echo "pcntl_fork: ", var_export(function_exists("pcntl_fork"), true), "\n";'
32+
- name: Install and build arms
3133
run: |
3234
set -e
33-
W=$RUNNER_TEMP/w; mkdir -p $W
34-
cd bench/project && composer install -q --no-interaction && cd -
35-
for arm in stock sha1 fc fc-sha1; do cp -R bench/project $W/$arm; done
36-
mkdir -p $W/fcdir-fc $W/fcdir-fc-sha1
37-
php -d phar.readonly=0 bench/make-arm.php $W/sha1/vendor/phpstan/phpstan/phpstan.phar sha1
38-
php -d phar.readonly=0 bench/make-arm.php $W/fc/vendor/phpstan/phpstan/phpstan.phar sha512 $W/fcdir-fc
39-
php -d phar.readonly=0 bench/make-arm.php $W/fc-sha1/vendor/phpstan/phpstan/phpstan.phar sha1 $W/fcdir-fc-sha1
40-
for arm in stock sha1 fc fc-sha1; do
41-
(cd $W/$arm && php vendor/bin/phpstan diagnose | grep -E 'Turbo extension|Mechanism' && php vendor/bin/phpstan analyse -c phpstan.neon --no-progress && php vendor/bin/phpstan analyse -c phpstan.neon --no-progress | tail -2)
42-
done
43-
find $W/fcdir-fc -type f | wc -l
44-
- name: Per-file open cost
35+
W="$RUNNER_TEMP/w"
36+
(cd bench/project && composer install -q --no-interaction)
37+
git clone -q --depth 1 https://github.com/nikic/PHP-Parser "$RUNNER_TEMP/php-parser"
38+
php bench/setup.php "$W" bench/project/vendor/phpstan/phpstan stock fc0 fcv fcve conc
39+
if [ "$RUNNER_OS" = Linux ]; then
40+
git clone -q --depth 1 https://github.com/tempestphp/tempest-framework "$RUNNER_TEMP/tempest"
41+
(cd "$RUNNER_TEMP/tempest" && composer install -q --no-interaction --ignore-platform-reqs --no-scripts)
42+
php bench/prepare.php "$W" "$RUNNER_TEMP/php-parser/lib" "$RUNNER_TEMP/tempest"
43+
else
44+
php bench/prepare.php "$W" "$RUNNER_TEMP/php-parser/lib"
45+
fi
46+
php bench/setup.php "$W" bench/project/vendor/phpstan/phpstan --neon-only
47+
cd "$W/projects/tiny" && php "$W/arms/fcv/phpstan.phar" diagnose | grep -E 'Turbo extension|Mechanism|Reason'
48+
- name: Tiny project
49+
run: php bench/suite.php "$RUNNER_TEMP/w" tiny 15
50+
- name: php-parser lib (271 files)
51+
run: php bench/suite.php "$RUNNER_TEMP/w" parser 5
52+
- name: php-parser lib, spawned workers
53+
run: php bench/suite.php "$RUNNER_TEMP/w" spawn 5
54+
- name: Concurrent runs sharing one empty file cache
4555
run: |
46-
for i in 1 2 3; do php bench/readmany.php bench/project/vendor 2000; done
47-
for i in 1 2 3; do php bench/readmany.php $RUNNER_TEMP/w/fcdir-fc 2000; done
48-
- name: Bench
49-
run: |
50-
W=$RUNNER_TEMP/w; A=()
51-
for arm in stock sha1 fc fc-sha1; do
52-
A+=("$arm-ver=$W/$arm|php vendor/bin/phpstan --version")
53-
A+=("$arm-warm=$W/$arm|php vendor/bin/phpstan analyse -c phpstan.neon --no-progress")
54-
A+=("$arm-cold=$W/$arm|prep:rm -rf tmp;;php vendor/bin/phpstan analyse -c phpstan.neon --no-progress")
55-
done
56-
php bench/bench.php 15 "${A[@]}" </dev/null
56+
if [ "$RUNNER_OS" != Windows ]; then php bench/conc.php "$RUNNER_TEMP/w" 3; fi
57+
php bench/conc.php "$RUNNER_TEMP/w" 3 spawn
58+
- name: Tempest (full)
59+
if: runner.os == 'Linux'
60+
run: php bench/suite.php "$RUNNER_TEMP/w" tempest 3
61+
- name: File cache size
62+
run: du -sh "$RUNNER_TEMP"/w/fcdir-* || true

‎bench/bench.php‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44
$rounds = (int) $argv[1]; $arms = [];
55
foreach (array_slice($argv, 2) as $spec) { [$label, $rest] = explode('=', $spec, 2); [$cwd, $cmd] = explode('|', $rest, 2); $prep = null; if (str_starts_with($cmd, 'prep:')) { [$prep, $cmd] = explode(';;', substr($cmd, 5), 2); } $arms[$label] = compact('cwd', 'cmd', 'prep'); }
66
$res = [];
7-
$cpu = static function (): float { $u = getrusage(1); return $u['ru_utime.tv_sec'] + $u['ru_utime.tv_usec'] / 1e6 + $u['ru_stime.tv_sec'] + $u['ru_stime.tv_usec'] / 1e6; };
7+
$null = PHP_OS_FAMILY === 'Windows' ? 'NUL' : '/dev/null';
8+
$cpu = static function (): float { $u = @getrusage(1); if (!is_array($u)) { return 0.0; } return $u['ru_utime.tv_sec'] + $u['ru_utime.tv_usec'] / 1e6 + $u['ru_stime.tv_sec'] + $u['ru_stime.tv_usec'] / 1e6; };
89
for ($r = 0; $r < $rounds; $r++) foreach ($arms as $label => $a) {
9-
if ($a['prep'] !== null) { exec('cd ' . escapeshellarg($a['cwd']) . ' && ' . $a['prep'] . ' >/dev/null 2>&1'); }
10+
if ($a['prep'] !== null) { exec('cd ' . escapeshellarg($a['cwd']) . ' && ' . $a['prep'] . ' >' . $null . ' 2>&1'); }
1011
$c0 = $cpu(); $t0 = hrtime(true);
11-
exec('cd ' . escapeshellarg($a['cwd']) . ' && ' . $a['cmd'] . ' >/dev/null 2>&1', $o, $ec);
12+
exec('cd ' . escapeshellarg($a['cwd']) . ' && ' . $a['cmd'] . ' >' . $null . ' 2>&1', $o, $ec);
1213
$res[$label]['wall'][] = (hrtime(true) - $t0) / 1e9; $res[$label]['cpu'][] = $cpu() - $c0; $res[$label]['ec'] = $ec;
1314
}
1415
$med = static function (array $v): float { sort($v); return $v[intdiv(count($v), 2)]; };

‎bench/conc.php‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php declare(strict_types = 1);
2+
// usage: php conc.php <work dir> <reps> [spawn]
3+
// Starts 4 runs at once on the parser project, all sharing one freshly emptied file cache, and compares
4+
// each run's raw errors with the stock arm's.
5+
$w = $argv[1];
6+
$reps = (int) $argv[2];
7+
$spawn = ($argv[3] ?? '') === 'spawn';
8+
$php = escapeshellarg(PHP_BINARY) . ($spawn ? ' -d disable_functions=pcntl_fork' : '');
9+
$project = "$w/projects/parser";
10+
$run = static function (string $arm, string $neon) use ($php, $w): string {
11+
return "$php " . escapeshellarg("$w/arms/$arm/phpstan.phar") . " analyse -c $neon --no-progress --error-format=raw --memory-limit=-1";
12+
};
13+
$norm = static function (string $out): array {
14+
$lines = array_filter(array_map('trim', explode("\n", $out)), static fn (string $l): bool => $l !== '');
15+
sort($lines);
16+
return $lines;
17+
};
18+
require_once __DIR__ . '/rmrf.php.inc';
19+
rmrf("$project/tmp-stock");
20+
$ref = $norm((string) shell_exec('cd ' . escapeshellarg($project) . ' && ' . $run('stock', 'phpstan-stock.neon') . ' 2>&1'));
21+
echo 'reference: ', count($ref), " lines\n";
22+
for ($r = 1; $r <= $reps; $r++) {
23+
rmrf("$w/fcdir-conc");
24+
mkdir("$w/fcdir-conc");
25+
$procs = [];
26+
for ($i = 1; $i <= 4; $i++) {
27+
rmrf("$project/tmp-conc$i");
28+
$procs[$i] = proc_open($run('conc', "phpstan-conc$i.neon"), [1 => ['pipe', 'w'], 2 => ['redirect', 1]], $pipes[$i], $project);
29+
}
30+
$same = 0;
31+
foreach ($procs as $i => $p) {
32+
$out = stream_get_contents($pipes[$i][1]);
33+
$ec = proc_close($p);
34+
$lines = $norm($out);
35+
if ($lines === $ref) {
36+
$same++;
37+
} else {
38+
echo "rep $r run $i exit $ec differs:\n", implode("\n", array_slice(array_merge(array_diff($lines, $ref), ['--- missing:'], array_diff($ref, $lines)), 0, 12)), "\n";
39+
}
40+
}
41+
echo 'rep ', $r, ($spawn ? ' spawn' : ' fork'), ": $same/4 identical to stock\n";
42+
}

‎bench/make-arm.php‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,30 @@
11
<?php declare(strict_types = 1);
2-
// usage: php -d phar.readonly=0 make-arm.php <phar> <sha1|sha512> [<file cache dir>]
3-
// Re-signs the phar, and with a directory given, makes the turbo restart use it as a persistent
4-
// OPcache file cache instead of blanking opcache.file_cache.
5-
[$_, $pharPath, $alg] = $argv;
2+
// usage: php -d phar.readonly=0 make-arm.php <phar> <sha1|sha512> [<file cache dir> [vts]]
3+
// Re-signs the phar. With a directory, the turbo restart (and the workers it spawns) use it as a
4+
// persistent OPcache file cache instead of blanking opcache.file_cache. "vts" also turns
5+
// opcache.validate_timestamps on, so edited files and a replaced phar are recompiled.
6+
$pharPath = $argv[1];
7+
$alg = $argv[2];
68
$dir = $argv[3] ?? null;
9+
$vts = ($argv[4] ?? '') === 'vts';
710
$phar = new Phar($pharPath);
811
$phar->startBuffering();
912
if ($dir !== null) {
1013
$inner = 'src/Turbo/TurboProcessRestarter.php';
1114
$src = file_get_contents('phar://' . $pharPath . '/' . $inner);
12-
$n = substr_count($src, "'opcache.file_cache=',");
13-
if ($n !== 1) {
14-
fwrite(STDERR, "file cache patch did not apply: $n\n");
15-
exit(1);
15+
$replace = ["'opcache.file_cache='," => var_export('opcache.file_cache=' . $dir, true) . ','];
16+
if ($vts) {
17+
$replace["'opcache.validate_timestamps=0',"] = "'opcache.validate_timestamps=1',";
1618
}
17-
$phar[$inner] = str_replace("'opcache.file_cache=',", var_export('opcache.file_cache=' . $dir, true) . ',', $src);
19+
foreach ($replace as $old => $new) {
20+
if (substr_count($src, $old) !== 1) {
21+
fwrite(STDERR, "patch did not apply: $old\n");
22+
exit(1);
23+
}
24+
$src = str_replace($old, $new, $src);
25+
}
26+
$phar[$inner] = $src;
1827
}
1928
$phar->setSignatureAlgorithm($alg === 'sha1' ? Phar::SHA1 : Phar::SHA512);
2029
$phar->stopBuffering();
21-
echo $pharPath, ': ', (new Phar($pharPath))->getSignature()['hash_type'], $dir !== null ? ", file cache $dir" : '', "\n";
30+
echo $pharPath, ': ', (new Phar($pharPath))->getSignature()['hash_type'], $dir !== null ? ", file cache $dir" . ($vts ? ', timestamps on' : '') : '', "\n";

‎bench/prepare.php‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?php declare(strict_types = 1);
2+
// usage: php prepare.php <work dir> <php-parser lib dir> [<tempest checkout>]
3+
// Writes the projects: tiny (one file), parser (php-parser's own lib/), and tempest (in place, when given).
4+
[$_, $w, $parserLib] = $argv;
5+
$tempest = $argv[3] ?? null;
6+
$copy = static function (string $from, string $to) use (&$copy): void {
7+
@mkdir($to, 0777, true);
8+
foreach (new DirectoryIterator($from) as $f) {
9+
if ($f->isDot()) continue;
10+
$f->isDir() ? $copy($f->getPathname(), $to . '/' . $f->getFilename()) : copy($f->getPathname(), $to . '/' . $f->getFilename());
11+
}
12+
};
13+
$copy(__DIR__ . '/project/src', "$w/projects/tiny/src");
14+
file_put_contents("$w/projects/tiny/base.neon", "includes:\n - $w/arms/stock/conf/bleedingEdge.neon\nparameters:\n level: max\n paths: [src]\n");
15+
$copy($parserLib, "$w/projects/parser/lib");
16+
file_put_contents("$w/projects/parser/base.neon", "parameters:\n level: 8\n paths: [lib]\n");
17+
if ($tempest !== null) {
18+
$dirs = glob("$tempest/packages/*/src", GLOB_ONLYDIR);
19+
$paths = implode('', array_map(static fn (string $d): string => " - $d\n", $dirs));
20+
@mkdir("$w/projects", 0777, true);
21+
symlink($tempest, "$w/projects/tempest");
22+
file_put_contents("$tempest/base.neon", "parameters:\n level: 8\n phpVersion: 80500\n paths:\n$paths");
23+
echo count($dirs), " tempest src dirs\n";
24+
}
25+
echo "projects written\n";

‎bench/rmrf.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?php declare(strict_types = 1);
2+
// usage: php rmrf.php [--recreate] <dir>... Deletes each directory tree (portable rm -rf); --recreate makes it again, empty.
3+
require_once __DIR__ . '/rmrf.php.inc';
4+
$args = array_slice($argv, 1);
5+
$recreate = ($args[0] ?? '') === '--recreate';
6+
if ($recreate) {
7+
array_shift($args);
8+
}
9+
foreach ($args as $dir) {
10+
rmrf($dir);
11+
if ($recreate) {
12+
mkdir($dir, 0777, true);
13+
}
14+
}

‎bench/rmrf.php.inc‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?php declare(strict_types = 1);
2+
function rmrf(string $dir): void
3+
{
4+
if (!is_dir($dir)) {
5+
return;
6+
}
7+
$it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::CHILD_FIRST);
8+
foreach ($it as $f) {
9+
$f->isDir() && !$f->isLink() ? rmdir($f->getPathname()) : unlink($f->getPathname());
10+
}
11+
rmdir($dir);
12+
}

‎bench/setup.php‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php declare(strict_types = 1);
2+
// usage: php setup.php <work dir> <phpstan package dir> <arm>... Copies the phpstan package once per arm,
3+
// patches each copy, and writes phpstan-<arm>.neon into every project under <work dir>/projects.
4+
[$_, $w, $pkg] = $argv;
5+
$arms = array_slice($argv, 3);
6+
$neonOnly = $arms === ['--neon-only'];
7+
if ($neonOnly) {
8+
$arms = array_map('basename', glob("$w/arms/*", GLOB_ONLYDIR));
9+
}
10+
$copy = static function (string $from, string $to) use (&$copy): void {
11+
mkdir($to, 0777, true);
12+
foreach (new DirectoryIterator($from) as $f) {
13+
if ($f->isDot()) continue;
14+
$f->isDir() ? $copy($f->getPathname(), $to . '/' . $f->getFilename()) : copy($f->getPathname(), $to . '/' . $f->getFilename());
15+
}
16+
};
17+
foreach ($neonOnly ? [] : $arms as $arm) {
18+
$dst = "$w/arms/$arm";
19+
$copy($pkg, $dst);
20+
$fc = "$w/fcdir-$arm";
21+
@mkdir($fc, 0777, true);
22+
$args = match (true) {
23+
$arm === 'stock' => ['sha512'],
24+
$arm === 'fc0' => ['sha512', $fc],
25+
default => ['sha512', $fc, 'vts'],
26+
};
27+
passthru(escapeshellarg(PHP_BINARY) . ' -d phar.readonly=0 ' . escapeshellarg(__DIR__ . '/make-arm.php') . ' ' . implode(' ', array_map('escapeshellarg', ["$dst/phpstan.phar", ...$args])), $ec);
28+
if ($ec !== 0) exit($ec);
29+
}
30+
foreach (glob("$w/projects/*", GLOB_ONLYDIR) as $proj) {
31+
$base = file_get_contents("$proj/base.neon");
32+
foreach ([...$arms, 'conc1', 'conc2', 'conc3', 'conc4'] as $arm) {
33+
file_put_contents("$proj/phpstan-$arm.neon", $base . "\n tmpDir: tmp-$arm\n");
34+
}
35+
}

‎bench/suite.php‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<?php declare(strict_types = 1);
2+
// usage: php suite.php <work dir> <tiny|parser|spawn|tempest> <rounds>
3+
// Primes every arm once (untimed), then runs bench.php over the suite's arms, interleaved.
4+
[$_, $w, $suite, $rounds] = $argv;
5+
$php = escapeshellarg(PHP_BINARY);
6+
$rm = $php . ' ' . escapeshellarg(__DIR__ . '/rmrf.php');
7+
$project = $w . '/projects/' . ($suite === 'spawn' ? 'parser' : $suite);
8+
$flags = $suite === 'spawn' ? ' -d disable_functions=pcntl_fork' : '';
9+
$arms = $suite === 'tiny' ? ['stock', 'fc0', 'fcv', 'fcve'] : ['stock', 'fcv', 'fcve'];
10+
$kinds = match ($suite) {
11+
'tiny' => ['ver', 'warm', 'cold'],
12+
'parser' => ['warm', 'cold'],
13+
default => ['cold'],
14+
};
15+
$specs = [];
16+
foreach ($arms as $arm) {
17+
$phar = escapeshellarg("$w/arms/$arm/phpstan.phar");
18+
$analyse = "$php$flags $phar analyse -c phpstan-$arm.neon --no-progress --memory-limit=-1";
19+
exec('cd ' . escapeshellarg($project) . " && $analyse > " . (PHP_OS_FAMILY === 'Windows' ? 'NUL' : '/dev/null') . ' 2>&1', $o, $ec);
20+
echo "primed $suite/$arm, exit $ec\n";
21+
$emptyFc = $arm === 'fcve' ? " && $rm --recreate " . escapeshellarg("$w/fcdir-fcve") : '';
22+
foreach ($kinds as $kind) {
23+
$cmd = match ($kind) {
24+
'ver' => "$php $phar --version",
25+
'warm' => $emptyFc === '' ? $analyse : 'prep:' . substr($emptyFc, 4) . ";;$analyse",
26+
'cold' => "prep:$rm tmp-$arm$emptyFc;;$analyse",
27+
};
28+
$specs[] = "$suite-$arm-$kind=$project|$cmd";
29+
}
30+
}
31+
passthru("$php " . escapeshellarg(__DIR__ . '/bench.php') . ' ' . (int) $rounds . ' ' . implode(' ', array_map('escapeshellarg', $specs)), $ec);
32+
exit($ec);

0 commit comments

Comments
 (0)