Skip to content

Commit 6d2b4dd

Browse files
committed
Add job size benchmark script, measures Rector wall time across parallel job sizes
1 parent 7dee71d commit 6d2b4dd

2 files changed

Lines changed: 350 additions & 0 deletions

File tree

‎phpstan.neon‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ parameters:
4040
- '*tests/*/Fixture*'
4141
- '*tests/*/Source/*'
4242
- '*tests/*/Source*'
43+
# standalone dev benchmark, shells out and parses getopt by hand
44+
- 'scripts/rector-jobsize-benchmark.php'
4345

4446
# see https://github.com/symplify/phpstan-rules
4547
symplify:
Lines changed: 348 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,348 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* Measures Rector's wall time across parallel job sizes.
7+
*
8+
* Usage, from the project root:
9+
*
10+
* php rector-jobsize-benchmark.php
11+
* php rector-jobsize-benchmark.php --config=rector.php --sizes=16,50,100,150,300
12+
* php rector-jobsize-benchmark.php --bin=vendor/bin/rector --timeout=3600
13+
*
14+
* Notes on reading the output:
15+
*
16+
* - "cached" counts the entries Rector wrote, which is files it analysed and found clean.
17+
* Files it would change are NOT cached, so "analysed" adds them back in.
18+
* - Every run is cold. A warm run is a different question and this script does not ask it.
19+
* - Job timeout is raised (default 3600 s), because a large jobSize puts more work into one
20+
* chunk and the stock 120 s limit aborts the run rather than slowing it down.
21+
* - Raising that timeout is exactly why this script only measures wall time, not safety. A
22+
* real run keeps the 120 s per-child timeout and finite memory, so a large jobSize that
23+
* looks fast here is where production hits "child process timed out" or OOM. Read the
24+
* fastest size as a curiosity, not as a recommended default - the safe band sits low,
25+
* around the stock jobSize of 20.
26+
*/
27+
final class JobSizeBenchmark
28+
{
29+
private ?string $spawnIniDirectory = null;
30+
31+
/**
32+
* @param int[] $jobSizes
33+
*/
34+
public function __construct(
35+
private readonly string $projectDirectory,
36+
private readonly string $configPath,
37+
private readonly string $rectorBinary,
38+
private readonly array $jobSizes,
39+
private readonly int $timeoutSeconds,
40+
private readonly bool $countProcesses = false,
41+
) {
42+
}
43+
44+
public function run(): int
45+
{
46+
$workingDirectory = sys_get_temp_dir() . '/rector-jobsize-' . bin2hex(random_bytes(4));
47+
48+
printf(
49+
"project: %s\nconfig: %s\ncores: %s\nphp: %s\n\n",
50+
$this->projectDirectory,
51+
$this->configPath,
52+
$this->detectCoreCount(),
53+
PHP_VERSION
54+
);
55+
56+
$rows = [];
57+
58+
foreach ($this->jobSizes as $jobSize) {
59+
$rows[] = $this->measure($jobSize, $workingDirectory);
60+
}
61+
62+
$this->removeDirectory($workingDirectory);
63+
$this->printMarkdownTable($rows);
64+
65+
return 0;
66+
}
67+
68+
/**
69+
* @return array{jobSize: int, seconds: float, exitCode: int, cached: int, changed: int, workers: int}
70+
*/
71+
private function measure(int $jobSize, string $workingDirectory): array
72+
{
73+
$runDirectory = $workingDirectory . '/job-' . $jobSize;
74+
$cacheDirectory = $runDirectory . '/cache';
75+
$containerCacheDirectory = $runDirectory . '/container';
76+
77+
$this->removeDirectory($runDirectory);
78+
79+
// Rector creates the result cache on demand but REQUIRES the container cache
80+
// directory to exist, so make both here.
81+
if (! mkdir($cacheDirectory, 0o777, true) || ! mkdir($containerCacheDirectory, 0o777, true)) {
82+
throw new RuntimeException('Could not create cache directories under ' . $runDirectory);
83+
}
84+
85+
$wrapperConfigPath = $runDirectory . '/rector-wrapper.php';
86+
file_put_contents($wrapperConfigPath, $this->buildWrapperConfig(
87+
$jobSize,
88+
$cacheDirectory,
89+
$containerCacheDirectory
90+
));
91+
92+
$spawnLogPath = $runDirectory . '/spawns.log';
93+
94+
$command = sprintf(
95+
'%s%s %s process --dry-run --no-progress-bar --output-format=json --config=%s 2>&1',
96+
$this->buildSpawnLoggingEnvironment($spawnLogPath),
97+
escapeshellarg(PHP_BINARY),
98+
escapeshellarg($this->rectorBinary),
99+
escapeshellarg($wrapperConfigPath)
100+
);
101+
102+
printf("jobSize %-6d running ... ", $jobSize);
103+
104+
$startedAt = hrtime(true);
105+
exec($command, $outputLines, $exitCode);
106+
$seconds = (hrtime(true) - $startedAt) / 1_000_000_000;
107+
108+
$changed = $this->parseChangedFiles(implode("\n", $outputLines));
109+
$cached = $this->countCacheEntries($cacheDirectory);
110+
$workers = $this->countWorkerProcesses($spawnLogPath);
111+
112+
printf("%6.1f s exit %d analysed %d\n", $seconds, $exitCode, $cached + $changed);
113+
114+
return [
115+
'jobSize' => $jobSize,
116+
'seconds' => $seconds,
117+
'exitCode' => $exitCode,
118+
'cached' => $cached,
119+
'changed' => $changed,
120+
'workers' => $workers,
121+
];
122+
}
123+
124+
private function buildWrapperConfig(
125+
int $jobSize,
126+
string $cacheDirectory,
127+
string $containerCacheDirectory
128+
): string {
129+
// The project config is imported, never modified. Calls made after the import win,
130+
// because each of them writes straight into the parameter bag.
131+
return sprintf(
132+
<<<'PHP'
133+
<?php
134+
135+
declare(strict_types=1);
136+
137+
use Rector\Config\RectorConfig;
138+
139+
return static function (RectorConfig $rectorConfig): void {
140+
$rectorConfig->import(%s);
141+
$rectorConfig->parallel(%d, %d, %d);
142+
$rectorConfig->cacheDirectory(%s);
143+
$rectorConfig->containerCacheDirectory(%s);
144+
};
145+
PHP,
146+
var_export($this->configPath, true),
147+
$this->timeoutSeconds,
148+
$this->detectCoreCount(),
149+
$jobSize,
150+
var_export($cacheDirectory, true),
151+
var_export($containerCacheDirectory, true)
152+
);
153+
}
154+
155+
/**
156+
* Counts how many PHP processes Rector starts, by prepending a one-line logger to every
157+
* one of them. Rector spawns its workers as plain `php` processes, so the only way to
158+
* reach them is through the environment they inherit - hence PHP_INI_SCAN_DIR rather
159+
* than a `-d` flag, which would apply to the parent only.
160+
*
161+
* The logger is a single `file_put_contents` per process, which is noise next to a
162+
* process start, but it is opt-in so that the default timings carry nothing extra.
163+
*/
164+
private function buildSpawnLoggingEnvironment(string $spawnLogPath): string
165+
{
166+
if (! $this->countProcesses) {
167+
return '';
168+
}
169+
170+
if ($this->spawnIniDirectory === null) {
171+
$this->spawnIniDirectory = dirname($spawnLogPath, 2) . '/spawn-ini';
172+
mkdir($this->spawnIniDirectory, 0o777, true);
173+
174+
$loggerPath = $this->spawnIniDirectory . '/spawn-logger.php';
175+
file_put_contents($loggerPath, <<<'PHP'
176+
<?php
177+
$log = getenv('RECTOR_BENCH_SPAWN_LOG');
178+
if (is_string($log) && $log !== '') {
179+
$isWorker = in_array('worker', $_SERVER['argv'] ?? [], true) ? 'worker' : 'main';
180+
file_put_contents($log, $isWorker . "\n", FILE_APPEND | LOCK_EX);
181+
}
182+
PHP);
183+
184+
file_put_contents(
185+
$this->spawnIniDirectory . '/zz-spawn.ini',
186+
'auto_prepend_file=' . $loggerPath . "\n"
187+
);
188+
}
189+
190+
return sprintf(
191+
'PHP_INI_SCAN_DIR=%s RECTOR_BENCH_SPAWN_LOG=%s ',
192+
escapeshellarg(':' . $this->spawnIniDirectory),
193+
escapeshellarg($spawnLogPath)
194+
);
195+
}
196+
197+
private function countWorkerProcesses(string $spawnLogPath): int
198+
{
199+
if (! is_file($spawnLogPath)) {
200+
return 0;
201+
}
202+
203+
$contents = file_get_contents($spawnLogPath);
204+
if (! is_string($contents)) {
205+
return 0;
206+
}
207+
208+
return substr_count($contents, "worker\n");
209+
}
210+
211+
private function parseChangedFiles(string $output): int
212+
{
213+
if (preg_match('/"changed_files"\s*:\s*(\d+)/', $output, $matches) === 1) {
214+
return (int) $matches[1];
215+
}
216+
217+
return 0;
218+
}
219+
220+
private function countCacheEntries(string $cacheDirectory): int
221+
{
222+
if (! is_dir($cacheDirectory)) {
223+
return 0;
224+
}
225+
226+
$count = 0;
227+
$iterator = new RecursiveIteratorIterator(
228+
new RecursiveDirectoryIterator($cacheDirectory, FilesystemIterator::SKIP_DOTS)
229+
);
230+
231+
foreach ($iterator as $fileInfo) {
232+
if (! $fileInfo instanceof SplFileInfo || $fileInfo->getExtension() !== 'php') {
233+
continue;
234+
}
235+
236+
// the cache also holds one configuration-hash entry, which is not a file
237+
$contents = file_get_contents($fileInfo->getPathname());
238+
if (is_string($contents) && str_contains($contents, 'file_hash')) {
239+
++$count;
240+
}
241+
}
242+
243+
return $count;
244+
}
245+
246+
/**
247+
* @param array<array{jobSize: int, seconds: float, exitCode: int, cached: int, changed: int, workers: int}> $rows
248+
*/
249+
private function printMarkdownTable(array $rows): void
250+
{
251+
$fastest = min(array_column($rows, 'seconds'));
252+
$workerColumn = $this->countProcesses ? ' worker processes |' : '';
253+
$workerDivider = $this->countProcesses ? ' --: |' : '';
254+
255+
echo "\n| `jobSize` | wall | vs fastest |" . $workerColumn . " exit | analysed | of which changed |\n";
256+
echo "| --: | --: | --: |" . $workerDivider . " --: | --: | --: |\n";
257+
258+
foreach ($rows as $row) {
259+
printf(
260+
"| %d | %.1f s | %.2fx |%s %d | %d | %d |\n",
261+
$row['jobSize'],
262+
$row['seconds'],
263+
$row['seconds'] / $fastest,
264+
$this->countProcesses ? ' ' . $row['workers'] . ' |' : '',
265+
$row['exitCode'],
266+
$row['cached'] + $row['changed'],
267+
$row['changed']
268+
);
269+
}
270+
271+
echo "\nEvery row is a cold run. `analysed` should be identical across rows - if a row is\n";
272+
echo "lower, that run silently dropped work rather than doing it faster.\n";
273+
}
274+
275+
private function detectCoreCount(): int
276+
{
277+
$shellCoreCount = @shell_exec('getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null');
278+
$coreCount = is_string($shellCoreCount) ? (int) trim($shellCoreCount) : 0;
279+
280+
return $coreCount > 0 ? $coreCount : 16;
281+
}
282+
283+
private function removeDirectory(string $directory): void
284+
{
285+
if (! is_dir($directory)) {
286+
return;
287+
}
288+
289+
$iterator = new RecursiveIteratorIterator(
290+
new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS),
291+
RecursiveIteratorIterator::CHILD_FIRST
292+
);
293+
294+
foreach ($iterator as $fileInfo) {
295+
if (! $fileInfo instanceof SplFileInfo) {
296+
continue;
297+
}
298+
299+
$fileInfo->isDir() ? rmdir($fileInfo->getPathname()) : unlink($fileInfo->getPathname());
300+
}
301+
302+
rmdir($directory);
303+
}
304+
}
305+
306+
// ---------------------------------------------------------------------------------------
307+
308+
$options = getopt('', ['config::', 'sizes::', 'bin::', 'timeout::', 'processes']);
309+
310+
$projectDirectory = getcwd();
311+
if ($projectDirectory === false) {
312+
fwrite(STDERR, "Cannot resolve the working directory.\n");
313+
exit(1);
314+
}
315+
316+
$configPath = realpath((string) ($options['config'] ?? 'rector.php'));
317+
if ($configPath === false) {
318+
fwrite(STDERR, "Config not found. Pass --config=path/to/rector.php\n");
319+
exit(1);
320+
}
321+
322+
$rectorBinary = realpath((string) ($options['bin'] ?? 'vendor/bin/rector'));
323+
if ($rectorBinary === false) {
324+
fwrite(STDERR, "Rector binary not found. Pass --bin=path/to/rector\n");
325+
exit(1);
326+
}
327+
328+
$jobSizes = array_map(
329+
static fn (string $size): int => (int) trim($size),
330+
explode(',', (string) ($options['sizes'] ?? '16,50,100,150,300'))
331+
);
332+
333+
$jobSizes = array_values(array_filter($jobSizes, static fn (int $size): bool => $size > 0));
334+
if ($jobSizes === []) {
335+
fwrite(STDERR, "No valid job sizes given.\n");
336+
exit(1);
337+
}
338+
339+
$benchmark = new JobSizeBenchmark(
340+
$projectDirectory,
341+
$configPath,
342+
$rectorBinary,
343+
$jobSizes,
344+
(int) ($options['timeout'] ?? 3600),
345+
array_key_exists('processes', $options)
346+
);
347+
348+
exit($benchmark->run());

0 commit comments

Comments
 (0)