Skip to content

Commit 0b20319

Browse files
ondrejmirtesclaude
andcommitted
Share directory file hashes across workers through the arena and memoize content hashes
Every parallel worker independently walked and SHA-256-hashed the same ~46 directories at startup to compute odsl fingerprints (~5.5k files, ~44k opens per run across 8 workers). The first process to reach a directory now publishes its file-hash map to the run's shared arena behind a single-flight lock, and the rest reuse it. A new per-process FileContentHasher additionally memoizes content hashes (mtime+size revalidated), collapsing repeated hashing of the same file by OptimizedDirectorySourceLocator::getCacheKeys (once per located identifier) and FileTypeMapper. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017PkuDxrKuvvrZe2V6svxRw
1 parent 1ce1480 commit 0b20319

4 files changed

Lines changed: 127 additions & 17 deletions

File tree

‎src/File/FileContentHasher.php‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
<?php declare(strict_types = 1);
2+
3+
namespace PHPStan\File;
4+
5+
use PHPStan\DependencyInjection\AutowiredService;
6+
use function hash_file;
7+
use function stat;
8+
9+
/**
10+
* Memoizes SHA-256 content hashes per process for the lifetime of a run.
11+
*
12+
* PHPStan hashes the same file repeatedly — once per located identifier in
13+
* OptimizedDirectorySourceLocator, once per PHPDoc cache check in
14+
* FileTypeMapper, once per directory fingerprint in the source-locator
15+
* factory. Project content is already treated as immutable for the duration
16+
* of a run (the result cache and the odsl fingerprints rely on this), so a
17+
* repeated hash of an unchanged file is pure waste. An mtime+size
18+
* revalidation guards the memo against files regenerated mid-run at the
19+
* cost of one stat() per repeated ask.
20+
*/
21+
#[AutowiredService]
22+
final class FileContentHasher
23+
{
24+
25+
/** @var array<string, array{int, int, string}> path => [mtime, size, sha256] */
26+
private array $hashes = [];
27+
28+
public function hash(string $file): string|false
29+
{
30+
$stat = @stat($file);
31+
if ($stat === false) {
32+
return false;
33+
}
34+
35+
$memo = $this->hashes[$file] ?? null;
36+
if ($memo !== null && $memo[0] === $stat['mtime'] && $memo[1] === $stat['size']) {
37+
return $memo[2];
38+
}
39+
40+
$hash = hash_file('sha256', $file);
41+
if ($hash === false) {
42+
return false;
43+
}
44+
45+
$this->hashes[$file] = [$stat['mtime'], $stat['size'], $hash];
46+
47+
return $hash;
48+
}
49+
50+
}

‎src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocator.php‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use PHPStan\BetterReflection\SourceLocator\Ast\Strategy\NodeToReflection;
1616
use PHPStan\BetterReflection\SourceLocator\Type\SourceLocator;
1717
use PHPStan\Cache\ArenaCache;
18+
use PHPStan\File\FileContentHasher;
1819
use PHPStan\Cache\Cache;
1920
use PHPStan\File\CouldNotReadFileException;
2021
use PHPStan\Internal\ComposerHelper;
@@ -24,7 +25,6 @@
2425
use function array_key_exists;
2526
use function array_values;
2627
use function current;
27-
use function hash_file;
2828
use function is_array;
2929
use function is_string;
3030
use function sprintf;
@@ -58,6 +58,7 @@ public function __construct(
5858
private FileNodesFetcher $fileNodesFetcher,
5959
private Cache $cache,
6060
private PhpVersion $phpVersion,
61+
private FileContentHasher $fileContentHasher,
6162
private array $classToFile,
6263
private array $functionToFiles,
6364
private array $constantToFile,
@@ -71,7 +72,7 @@ public function __construct(
7172
*/
7273
private function getCacheKeys(string $file, Identifier $identifier): array
7374
{
74-
$fileHash = hash_file('sha256', $file);
75+
$fileHash = $this->fileContentHasher->hash($file);
7576
if ($fileHash === false) {
7677
throw new CouldNotReadFileException($file);
7778
}

‎src/Reflection/BetterReflection/SourceLocator/OptimizedDirectorySourceLocatorFactory.php‎

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
use PHPStan\Cache\Cache;
77
use PHPStan\DependencyInjection\AutowiredParameter;
88
use PHPStan\DependencyInjection\AutowiredService;
9+
use PHPStan\File\FileContentHasher;
910
use PHPStan\File\FileFinder;
1011
use PHPStan\Internal\DirectoryCreator;
1112
use PHPStan\Internal\DirectoryCreatorException;
@@ -15,8 +16,8 @@
1516
use function fclose;
1617
use function flock;
1718
use function fopen;
18-
use function hash_file;
1919
use function hrtime;
20+
use function is_array;
2021
use function ksort;
2122
use function serialize;
2223
use function sha1;
@@ -41,13 +42,22 @@ final class OptimizedDirectorySourceLocatorFactory
4142

4243
private const SCAN_LOCK_POLL_INTERVAL_MICROSECONDS = 50_000;
4344

45+
/**
46+
* The hash lock is polled much finer than the scan lock: every worker
47+
* reaches the same directories in near lockstep at startup, and most
48+
* directories hash in well under the scan lock's 50ms tick, so a coarse
49+
* poll would make lock losers sleep longer than the work they skip.
50+
*/
51+
private const HASH_LOCK_POLL_INTERVAL_MICROSECONDS = 5_000;
52+
4453
public function __construct(
4554
private FileNodesFetcher $fileNodesFetcher,
4655
#[AutowiredParameter(ref: '@fileFinderScan')]
4756
private FileFinder $fileFinder,
4857
private PhpVersion $phpVersion,
4958
private SymbolFinderInFiles $symbolFinderInFiles,
5059
private Cache $cache,
60+
private FileContentHasher $fileContentHasher,
5161
#[AutowiredParameter]
5262
private string $tmpDir,
5363
)
@@ -56,17 +66,63 @@ public function __construct(
5666

5767
public function createByDirectory(string $directory): OptimizedDirectorySourceLocator
5868
{
59-
$files = $this->fileFinder->findFiles([$directory])->getFiles();
60-
$fileHashes = [];
61-
foreach ($files as $file) {
62-
$hash = hash_file('sha256', $file);
63-
if ($hash === false) {
64-
continue;
69+
$cacheKey = sprintf('odsl-%s', $directory);
70+
$hashesRecordKey = 'odsl-filehashes-' . $directory;
71+
72+
// The walk + hash of a directory is identical in every process of a
73+
// run, and running it once per worker in parallel multiplies both the
74+
// CPU and — on hosts where concurrent open() is expensive — the wall
75+
// cost of the analysis startup. When the run has a shared arena, the
76+
// first process publishes the file-hash map and everyone else reuses
77+
// it. hasRecord() on the analysed-files record (published by the
78+
// master before workers spawn) doubles as the "is an arena active?"
79+
// probe — the seam has no explicit method for that and only grows one
80+
// together with the extension.
81+
$arenaActive = ArenaCache::hasRecord('analysed-files');
82+
$hashesLock = null;
83+
if ($arenaActive) {
84+
$shared = ArenaCache::lookup($hashesRecordKey);
85+
if (is_array($shared)) {
86+
/** @var array<string, string> $shared */
87+
return $this->createCachedDirectorySourceLocator($shared, $cacheKey);
88+
}
89+
90+
// Single-flight the walk + hash, same pattern as the cold-cache
91+
// scan below: the winner computes and publishes, losers wait and
92+
// re-read the record. A lost lock (timeout, unwritable tmp) just
93+
// means this worker hashes the directory itself.
94+
$hashesLock = $this->acquireDirectoryScanLock('hashes-' . $directory, self::HASH_LOCK_POLL_INTERVAL_MICROSECONDS);
95+
if ($hashesLock !== null) {
96+
$shared = ArenaCache::lookup($hashesRecordKey);
97+
if (is_array($shared)) {
98+
$this->releaseDirectoryScanLock($hashesLock);
99+
100+
/** @var array<string, string> $shared */
101+
return $this->createCachedDirectorySourceLocator($shared, $cacheKey);
102+
}
103+
}
104+
}
105+
106+
try {
107+
$files = $this->fileFinder->findFiles([$directory])->getFiles();
108+
$fileHashes = [];
109+
foreach ($files as $file) {
110+
$hash = $this->fileContentHasher->hash($file);
111+
if ($hash === false) {
112+
continue;
113+
}
114+
$fileHashes[$file] = $hash;
115+
}
116+
117+
if ($arenaActive) {
118+
ArenaCache::publish($hashesRecordKey, $fileHashes);
119+
}
120+
} finally {
121+
if ($hashesLock !== null) {
122+
$this->releaseDirectoryScanLock($hashesLock);
65123
}
66-
$fileHashes[$file] = $hash;
67124
}
68125

69-
$cacheKey = sprintf('odsl-%s', $directory);
70126
return $this->createCachedDirectorySourceLocator($fileHashes, $cacheKey);
71127
}
72128

@@ -96,6 +152,7 @@ private function createCachedDirectorySourceLocator(array $fileHashes, string $c
96152
$this->fileNodesFetcher,
97153
$this->cache,
98154
$this->phpVersion,
155+
$this->fileContentHasher,
99156
[],
100157
[],
101158
[],
@@ -188,6 +245,7 @@ private function createCachedDirectorySourceLocator(array $fileHashes, string $c
188245
$this->fileNodesFetcher,
189246
$this->cache,
190247
$this->phpVersion,
248+
$this->fileContentHasher,
191249
$classToFile,
192250
$functionToFiles,
193251
$constantToFile,
@@ -202,7 +260,7 @@ private function createCachedDirectorySourceLocator(array $fileHashes, string $c
202260
*
203261
* @return resource|null
204262
*/
205-
private function acquireDirectoryScanLock(string $lockKey)
263+
private function acquireDirectoryScanLock(string $lockKey, int $pollIntervalMicroseconds = self::SCAN_LOCK_POLL_INTERVAL_MICROSECONDS)
206264
{
207265
$lockDirectory = sprintf('%s/cache/locks', $this->tmpDir);
208266
try {
@@ -231,7 +289,7 @@ private function acquireDirectoryScanLock(string $lockKey)
231289
return null;
232290
}
233291

234-
usleep(self::SCAN_LOCK_POLL_INTERVAL_MICROSECONDS);
292+
usleep($pollIntervalMicroseconds);
235293
}
236294

237295
return $lockHandle;
@@ -266,7 +324,7 @@ public function createByFiles(array $files, string $uniqueCacheIdentifier): Opti
266324
{
267325
$fileHashes = [];
268326
foreach ($files as $file) {
269-
$hash = hash_file('sha256', $file);
327+
$hash = $this->fileContentHasher->hash($file);
270328
if ($hash === false) {
271329
continue;
272330
}

‎src/Type/FileTypeMapper.php‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
use PHPStan\Cache\Cache;
1313
use PHPStan\DependencyInjection\AutowiredParameter;
1414
use PHPStan\DependencyInjection\AutowiredService;
15+
use PHPStan\File\FileContentHasher;
1516
use PHPStan\File\FileHelper;
1617
use PHPStan\Internal\ComposerHelper;
1718
use PHPStan\Parser\Parser;
@@ -40,7 +41,6 @@
4041
use function array_pop;
4142
use function array_reverse;
4243
use function count;
43-
use function hash_file;
4444
use function in_array;
4545
use function is_array;
4646
use function is_file;
@@ -83,6 +83,7 @@ public function __construct(
8383
private AnonymousClassNameHelper $anonymousClassNameHelper,
8484
private FileHelper $fileHelper,
8585
private Cache $cache,
86+
private FileContentHasher $fileContentHasher,
8687
#[AutowiredParameter(ref: '%cache.resolvedPhpDocBlockCacheCountMax%')]
8788
private int $resolvedPhpDocBlockCacheCountMax,
8889
#[AutowiredParameter(ref: '%cache.nameScopeMapMemoryCacheCountMax%')]
@@ -352,7 +353,7 @@ private function getNameScopeMap(string $fileName): array
352353
[$nameScopeMap, $files] = $this->createPhpDocNodeMap($fileName, null, null, [], $fileName);
353354
$filesWithHashes = [];
354355
foreach ($files as $file) {
355-
$newHash = hash_file('sha256', $file);
356+
$newHash = $this->fileContentHasher->hash($file);
356357
$filesWithHashes[$file] = $newHash;
357358
}
358359
$this->cache->save($cacheKey, $variableCacheKey, [$nameScopeMap, $filesWithHashes]);
@@ -388,7 +389,7 @@ private function loadCachedPhpDocNodeMap(string $cacheKey, string $variableCache
388389
[$nameScopeMap, $filesWithHashes] = $cached;
389390
$useCache = true;
390391
foreach ($filesWithHashes as $file => $hash) {
391-
$newHash = @hash_file('sha256', $file);
392+
$newHash = $this->fileContentHasher->hash($file);
392393
if ($newHash === false) {
393394
$useCache = false;
394395
break;

0 commit comments

Comments
 (0)