|
| 1 | +<?php |
| 2 | + |
| 3 | +declare(strict_types=1); |
| 4 | + |
| 5 | +/** |
| 6 | + * Measures Rector's wall time across parallel job sizes. |
| 7 | + * |
| 8 | + * Usage, from the project root: |
| 9 | + * |
| 10 | + * php rector-jobsize-benchmark.php |
| 11 | + * php rector-jobsize-benchmark.php --config=rector.php --sizes=16,50,100,150,300 |
| 12 | + * php rector-jobsize-benchmark.php --bin=vendor/bin/rector --timeout=3600 |
| 13 | + * |
| 14 | + * Notes on reading the output: |
| 15 | + * |
| 16 | + * - "cached" counts the entries Rector wrote, which is files it analysed and found clean. |
| 17 | + * Files it would change are NOT cached, so "analysed" adds them back in. |
| 18 | + * - Every run is cold. A warm run is a different question and this script does not ask it. |
| 19 | + * - Job timeout is raised (default 3600 s), because a large jobSize puts more work into one |
| 20 | + * chunk and the stock 120 s limit aborts the run rather than slowing it down. |
| 21 | + * - Raising that timeout is exactly why this script only measures wall time, not safety. A |
| 22 | + * real run keeps the 120 s per-child timeout and finite memory, so a large jobSize that |
| 23 | + * looks fast here is where production hits "child process timed out" or OOM. Read the |
| 24 | + * fastest size as a curiosity, not as a recommended default - the safe band sits low, |
| 25 | + * around the stock jobSize of 20. |
| 26 | + */ |
| 27 | +final class JobSizeBenchmark |
| 28 | +{ |
| 29 | + private ?string $spawnIniDirectory = null; |
| 30 | + |
| 31 | + /** |
| 32 | + * @param int[] $jobSizes |
| 33 | + */ |
| 34 | + public function __construct( |
| 35 | + private readonly string $projectDirectory, |
| 36 | + private readonly string $configPath, |
| 37 | + private readonly string $rectorBinary, |
| 38 | + private readonly array $jobSizes, |
| 39 | + private readonly int $timeoutSeconds, |
| 40 | + private readonly bool $countProcesses = false, |
| 41 | + ) { |
| 42 | + } |
| 43 | + |
| 44 | + public function run(): int |
| 45 | + { |
| 46 | + $workingDirectory = sys_get_temp_dir() . '/rector-jobsize-' . bin2hex(random_bytes(4)); |
| 47 | + |
| 48 | + printf( |
| 49 | + "project: %s\nconfig: %s\ncores: %s\nphp: %s\n\n", |
| 50 | + $this->projectDirectory, |
| 51 | + $this->configPath, |
| 52 | + $this->detectCoreCount(), |
| 53 | + PHP_VERSION |
| 54 | + ); |
| 55 | + |
| 56 | + $rows = []; |
| 57 | + |
| 58 | + foreach ($this->jobSizes as $jobSize) { |
| 59 | + $rows[] = $this->measure($jobSize, $workingDirectory); |
| 60 | + } |
| 61 | + |
| 62 | + $this->removeDirectory($workingDirectory); |
| 63 | + $this->printMarkdownTable($rows); |
| 64 | + |
| 65 | + return 0; |
| 66 | + } |
| 67 | + |
| 68 | + /** |
| 69 | + * @return array{jobSize: int, seconds: float, exitCode: int, cached: int, changed: int, workers: int} |
| 70 | + */ |
| 71 | + private function measure(int $jobSize, string $workingDirectory): array |
| 72 | + { |
| 73 | + $runDirectory = $workingDirectory . '/job-' . $jobSize; |
| 74 | + $cacheDirectory = $runDirectory . '/cache'; |
| 75 | + $containerCacheDirectory = $runDirectory . '/container'; |
| 76 | + |
| 77 | + $this->removeDirectory($runDirectory); |
| 78 | + |
| 79 | + // Rector creates the result cache on demand but REQUIRES the container cache |
| 80 | + // directory to exist, so make both here. |
| 81 | + if (! mkdir($cacheDirectory, 0o777, true) || ! mkdir($containerCacheDirectory, 0o777, true)) { |
| 82 | + throw new RuntimeException('Could not create cache directories under ' . $runDirectory); |
| 83 | + } |
| 84 | + |
| 85 | + $wrapperConfigPath = $runDirectory . '/rector-wrapper.php'; |
| 86 | + file_put_contents($wrapperConfigPath, $this->buildWrapperConfig( |
| 87 | + $jobSize, |
| 88 | + $cacheDirectory, |
| 89 | + $containerCacheDirectory |
| 90 | + )); |
| 91 | + |
| 92 | + $spawnLogPath = $runDirectory . '/spawns.log'; |
| 93 | + |
| 94 | + $command = sprintf( |
| 95 | + '%s%s %s process --dry-run --no-progress-bar --output-format=json --config=%s 2>&1', |
| 96 | + $this->buildSpawnLoggingEnvironment($spawnLogPath), |
| 97 | + escapeshellarg(PHP_BINARY), |
| 98 | + escapeshellarg($this->rectorBinary), |
| 99 | + escapeshellarg($wrapperConfigPath) |
| 100 | + ); |
| 101 | + |
| 102 | + printf("jobSize %-6d running ... ", $jobSize); |
| 103 | + |
| 104 | + $startedAt = hrtime(true); |
| 105 | + exec($command, $outputLines, $exitCode); |
| 106 | + $seconds = (hrtime(true) - $startedAt) / 1_000_000_000; |
| 107 | + |
| 108 | + $changed = $this->parseChangedFiles(implode("\n", $outputLines)); |
| 109 | + $cached = $this->countCacheEntries($cacheDirectory); |
| 110 | + $workers = $this->countWorkerProcesses($spawnLogPath); |
| 111 | + |
| 112 | + printf("%6.1f s exit %d analysed %d\n", $seconds, $exitCode, $cached + $changed); |
| 113 | + |
| 114 | + return [ |
| 115 | + 'jobSize' => $jobSize, |
| 116 | + 'seconds' => $seconds, |
| 117 | + 'exitCode' => $exitCode, |
| 118 | + 'cached' => $cached, |
| 119 | + 'changed' => $changed, |
| 120 | + 'workers' => $workers, |
| 121 | + ]; |
| 122 | + } |
| 123 | + |
| 124 | + private function buildWrapperConfig( |
| 125 | + int $jobSize, |
| 126 | + string $cacheDirectory, |
| 127 | + string $containerCacheDirectory |
| 128 | + ): string { |
| 129 | + // The project config is imported, never modified. Calls made after the import win, |
| 130 | + // because each of them writes straight into the parameter bag. |
| 131 | + return sprintf( |
| 132 | + <<<'PHP' |
| 133 | + <?php |
| 134 | +
|
| 135 | + declare(strict_types=1); |
| 136 | +
|
| 137 | + use Rector\Config\RectorConfig; |
| 138 | +
|
| 139 | + return static function (RectorConfig $rectorConfig): void { |
| 140 | + $rectorConfig->import(%s); |
| 141 | + $rectorConfig->parallel(%d, %d, %d); |
| 142 | + $rectorConfig->cacheDirectory(%s); |
| 143 | + $rectorConfig->containerCacheDirectory(%s); |
| 144 | + }; |
| 145 | + PHP, |
| 146 | + var_export($this->configPath, true), |
| 147 | + $this->timeoutSeconds, |
| 148 | + $this->detectCoreCount(), |
| 149 | + $jobSize, |
| 150 | + var_export($cacheDirectory, true), |
| 151 | + var_export($containerCacheDirectory, true) |
| 152 | + ); |
| 153 | + } |
| 154 | + |
| 155 | + /** |
| 156 | + * Counts how many PHP processes Rector starts, by prepending a one-line logger to every |
| 157 | + * one of them. Rector spawns its workers as plain `php` processes, so the only way to |
| 158 | + * reach them is through the environment they inherit - hence PHP_INI_SCAN_DIR rather |
| 159 | + * than a `-d` flag, which would apply to the parent only. |
| 160 | + * |
| 161 | + * The logger is a single `file_put_contents` per process, which is noise next to a |
| 162 | + * process start, but it is opt-in so that the default timings carry nothing extra. |
| 163 | + */ |
| 164 | + private function buildSpawnLoggingEnvironment(string $spawnLogPath): string |
| 165 | + { |
| 166 | + if (! $this->countProcesses) { |
| 167 | + return ''; |
| 168 | + } |
| 169 | + |
| 170 | + if ($this->spawnIniDirectory === null) { |
| 171 | + $this->spawnIniDirectory = dirname($spawnLogPath, 2) . '/spawn-ini'; |
| 172 | + mkdir($this->spawnIniDirectory, 0o777, true); |
| 173 | + |
| 174 | + $loggerPath = $this->spawnIniDirectory . '/spawn-logger.php'; |
| 175 | + file_put_contents($loggerPath, <<<'PHP' |
| 176 | + <?php |
| 177 | + $log = getenv('RECTOR_BENCH_SPAWN_LOG'); |
| 178 | + if (is_string($log) && $log !== '') { |
| 179 | + $isWorker = in_array('worker', $_SERVER['argv'] ?? [], true) ? 'worker' : 'main'; |
| 180 | + file_put_contents($log, $isWorker . "\n", FILE_APPEND | LOCK_EX); |
| 181 | + } |
| 182 | + PHP); |
| 183 | + |
| 184 | + file_put_contents( |
| 185 | + $this->spawnIniDirectory . '/zz-spawn.ini', |
| 186 | + 'auto_prepend_file=' . $loggerPath . "\n" |
| 187 | + ); |
| 188 | + } |
| 189 | + |
| 190 | + return sprintf( |
| 191 | + 'PHP_INI_SCAN_DIR=%s RECTOR_BENCH_SPAWN_LOG=%s ', |
| 192 | + escapeshellarg(':' . $this->spawnIniDirectory), |
| 193 | + escapeshellarg($spawnLogPath) |
| 194 | + ); |
| 195 | + } |
| 196 | + |
| 197 | + private function countWorkerProcesses(string $spawnLogPath): int |
| 198 | + { |
| 199 | + if (! is_file($spawnLogPath)) { |
| 200 | + return 0; |
| 201 | + } |
| 202 | + |
| 203 | + $contents = file_get_contents($spawnLogPath); |
| 204 | + if (! is_string($contents)) { |
| 205 | + return 0; |
| 206 | + } |
| 207 | + |
| 208 | + return substr_count($contents, "worker\n"); |
| 209 | + } |
| 210 | + |
| 211 | + private function parseChangedFiles(string $output): int |
| 212 | + { |
| 213 | + if (preg_match('/"changed_files"\s*:\s*(\d+)/', $output, $matches) === 1) { |
| 214 | + return (int) $matches[1]; |
| 215 | + } |
| 216 | + |
| 217 | + return 0; |
| 218 | + } |
| 219 | + |
| 220 | + private function countCacheEntries(string $cacheDirectory): int |
| 221 | + { |
| 222 | + if (! is_dir($cacheDirectory)) { |
| 223 | + return 0; |
| 224 | + } |
| 225 | + |
| 226 | + $count = 0; |
| 227 | + $iterator = new RecursiveIteratorIterator( |
| 228 | + new RecursiveDirectoryIterator($cacheDirectory, FilesystemIterator::SKIP_DOTS) |
| 229 | + ); |
| 230 | + |
| 231 | + foreach ($iterator as $fileInfo) { |
| 232 | + if (! $fileInfo instanceof SplFileInfo || $fileInfo->getExtension() !== 'php') { |
| 233 | + continue; |
| 234 | + } |
| 235 | + |
| 236 | + // the cache also holds one configuration-hash entry, which is not a file |
| 237 | + $contents = file_get_contents($fileInfo->getPathname()); |
| 238 | + if (is_string($contents) && str_contains($contents, 'file_hash')) { |
| 239 | + ++$count; |
| 240 | + } |
| 241 | + } |
| 242 | + |
| 243 | + return $count; |
| 244 | + } |
| 245 | + |
| 246 | + /** |
| 247 | + * @param array<array{jobSize: int, seconds: float, exitCode: int, cached: int, changed: int, workers: int}> $rows |
| 248 | + */ |
| 249 | + private function printMarkdownTable(array $rows): void |
| 250 | + { |
| 251 | + $fastest = min(array_column($rows, 'seconds')); |
| 252 | + $workerColumn = $this->countProcesses ? ' worker processes |' : ''; |
| 253 | + $workerDivider = $this->countProcesses ? ' --: |' : ''; |
| 254 | + |
| 255 | + echo "\n| `jobSize` | wall | vs fastest |" . $workerColumn . " exit | analysed | of which changed |\n"; |
| 256 | + echo "| --: | --: | --: |" . $workerDivider . " --: | --: | --: |\n"; |
| 257 | + |
| 258 | + foreach ($rows as $row) { |
| 259 | + printf( |
| 260 | + "| %d | %.1f s | %.2fx |%s %d | %d | %d |\n", |
| 261 | + $row['jobSize'], |
| 262 | + $row['seconds'], |
| 263 | + $row['seconds'] / $fastest, |
| 264 | + $this->countProcesses ? ' ' . $row['workers'] . ' |' : '', |
| 265 | + $row['exitCode'], |
| 266 | + $row['cached'] + $row['changed'], |
| 267 | + $row['changed'] |
| 268 | + ); |
| 269 | + } |
| 270 | + |
| 271 | + echo "\nEvery row is a cold run. `analysed` should be identical across rows - if a row is\n"; |
| 272 | + echo "lower, that run silently dropped work rather than doing it faster.\n"; |
| 273 | + } |
| 274 | + |
| 275 | + private function detectCoreCount(): int |
| 276 | + { |
| 277 | + $shellCoreCount = @shell_exec('getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null'); |
| 278 | + $coreCount = is_string($shellCoreCount) ? (int) trim($shellCoreCount) : 0; |
| 279 | + |
| 280 | + return $coreCount > 0 ? $coreCount : 16; |
| 281 | + } |
| 282 | + |
| 283 | + private function removeDirectory(string $directory): void |
| 284 | + { |
| 285 | + if (! is_dir($directory)) { |
| 286 | + return; |
| 287 | + } |
| 288 | + |
| 289 | + $iterator = new RecursiveIteratorIterator( |
| 290 | + new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS), |
| 291 | + RecursiveIteratorIterator::CHILD_FIRST |
| 292 | + ); |
| 293 | + |
| 294 | + foreach ($iterator as $fileInfo) { |
| 295 | + if (! $fileInfo instanceof SplFileInfo) { |
| 296 | + continue; |
| 297 | + } |
| 298 | + |
| 299 | + $fileInfo->isDir() ? rmdir($fileInfo->getPathname()) : unlink($fileInfo->getPathname()); |
| 300 | + } |
| 301 | + |
| 302 | + rmdir($directory); |
| 303 | + } |
| 304 | +} |
| 305 | + |
| 306 | +// --------------------------------------------------------------------------------------- |
| 307 | + |
| 308 | +$options = getopt('', ['config::', 'sizes::', 'bin::', 'timeout::', 'processes']); |
| 309 | + |
| 310 | +$projectDirectory = getcwd(); |
| 311 | +if ($projectDirectory === false) { |
| 312 | + fwrite(STDERR, "Cannot resolve the working directory.\n"); |
| 313 | + exit(1); |
| 314 | +} |
| 315 | + |
| 316 | +$configPath = realpath((string) ($options['config'] ?? 'rector.php')); |
| 317 | +if ($configPath === false) { |
| 318 | + fwrite(STDERR, "Config not found. Pass --config=path/to/rector.php\n"); |
| 319 | + exit(1); |
| 320 | +} |
| 321 | + |
| 322 | +$rectorBinary = realpath((string) ($options['bin'] ?? 'vendor/bin/rector')); |
| 323 | +if ($rectorBinary === false) { |
| 324 | + fwrite(STDERR, "Rector binary not found. Pass --bin=path/to/rector\n"); |
| 325 | + exit(1); |
| 326 | +} |
| 327 | + |
| 328 | +$jobSizes = array_map( |
| 329 | + static fn (string $size): int => (int) trim($size), |
| 330 | + explode(',', (string) ($options['sizes'] ?? '16,50,100,150,300')) |
| 331 | +); |
| 332 | + |
| 333 | +$jobSizes = array_values(array_filter($jobSizes, static fn (int $size): bool => $size > 0)); |
| 334 | +if ($jobSizes === []) { |
| 335 | + fwrite(STDERR, "No valid job sizes given.\n"); |
| 336 | + exit(1); |
| 337 | +} |
| 338 | + |
| 339 | +$benchmark = new JobSizeBenchmark( |
| 340 | + $projectDirectory, |
| 341 | + $configPath, |
| 342 | + $rectorBinary, |
| 343 | + $jobSizes, |
| 344 | + (int) ($options['timeout'] ?? 3600), |
| 345 | + array_key_exists('processes', $options) |
| 346 | +); |
| 347 | + |
| 348 | +exit($benchmark->run()); |
0 commit comments