Skip to content

Commit 67d6615

Browse files
janedbalondrejmirtes
janedbal
authored and
ondrejmirtes
committed
Read the cached collected data only when the rules on CollectedDataNode run
restore() decoded the whole collectedData section into the main process before the analysis started. The section is the largest part of the cache on a project using collectors - a dead code detector stores every member usage in it, hundreds of megabytes on a large codebase - and only the rules that run on CollectedDataNode need it, after the analysis, in the main process. Decoding it up front kept it in memory through the whole run, and the forked workers inherited it. The array frames now carry the key in its own payload, so a reader can index the entries without decoding the values. readCacheFile() walks the collectedData section recording where each file's entry is, and restore() hands that index out as LazyCollectedData together with the fresh data of the run. save() copies the entries that stay valid byte for byte from the old file into the new one and records their new positions, so the finalizer reads them back from the file that exists after the rename. The section is decoded once, in AnalyserResultFinalizer, and lives as long as the CollectedDataNode. The file is opened for each read and closed again: an open handle would be inherited by the forked workers, and a rename over the file must not find it open on Windows. A read checks that the entry it finds belongs to the file it was indexed for, so a cache file replaced by another run fails loudly instead of handing out someone else's data. AnalysisResult takes the collected data as a closure too, so a formatter that never asks for it does not decode the section a second time. Measured on phpstan-src self-analysis with a dead code detector that keeps its usages in collected data (2467 files, 88 MB cache), main process: before after restore(), warm run 200 ms, +168 MB 175 ms, +74 MB partial run, 486 files 486 MB main 440 MB main 368 MB largest worker 276 MB largest worker A warm run that copies every entry writes a byte-identical cache file. The cache version is bumped because the entry framing changed. Amp-Thread-ID: https://ampcode.com/threads/T-01a19219-6260-7460-a52b-a418f709e4bf
1 parent 091c5a7 commit 67d6615

16 files changed

Lines changed: 937 additions & 124 deletions

‎src/Analyser/Analyser.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace PHPStan\Analyser;
44

55
use Closure;
6+
use PHPStan\Analyser\ResultCache\LazyCollectedData;
67
use PHPStan\Collectors\CollectedData;
78
use PHPStan\Collectors\Registry as CollectorRegistry;
89
use PHPStan\DependencyInjection\AutowiredParameter;
@@ -142,7 +143,7 @@ public function analyse(
142143
linesToIgnore: $linesToIgnore,
143144
unmatchedLineIgnores: $unmatchedLineIgnores,
144145
internalErrors: [],
145-
collectedData: $collectedData,
146+
collectedData: LazyCollectedData::fromArray($collectedData),
146147
dependencies: $internalErrorsCount === 0 ? $dependencies : null,
147148
usedTraitDependencies: $internalErrorsCount === 0 ? $usedTraitDependencies : null,
148149
valueDependencies: $internalErrorsCount === 0 ? $valueDependencies : null,

‎src/Analyser/AnalyserResult.php‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
namespace PHPStan\Analyser;
44

5+
use PHPStan\Analyser\ResultCache\LazyCollectedData;
56
use PHPStan\Collectors\CollectedData;
67
use PHPStan\Dependency\RootExportedNode;
78
use function usort;
@@ -24,7 +25,6 @@ final class AnalyserResult
2425
* @param list<Error> $locallyIgnoredErrors
2526
* @param array<string, LinesToIgnore> $linesToIgnore
2627
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
27-
* @param CollectorData $collectedData
2828
* @param list<InternalError> $internalErrors
2929
* @param array<string, array<string>>|null $dependencies
3030
* @param array<string, array<string>>|null $usedTraitDependencies
@@ -41,7 +41,7 @@ public function __construct(
4141
private array $linesToIgnore,
4242
private array $unmatchedLineIgnores,
4343
private array $internalErrors,
44-
private array $collectedData,
44+
private LazyCollectedData $collectedData,
4545
private ?array $dependencies,
4646
private ?array $usedTraitDependencies,
4747
private ?array $valueDependencies,
@@ -145,9 +145,16 @@ public function getInternalErrors(): array
145145
}
146146

147147
/**
148+
* Decodes what came from the result cache; getLazyCollectedData() passes it on undecoded.
149+
*
148150
* @return CollectorData
149151
*/
150152
public function getCollectedData(): array
153+
{
154+
return $this->collectedData->toArray();
155+
}
156+
157+
public function getLazyCollectedData(): LazyCollectedData
151158
{
152159
return $this->collectedData;
153160
}

‎src/Analyser/AnalyserResultFinalizer.php‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
use Closure;
66
use PHPStan\AnalysedCodeException;
7+
use PHPStan\Analyser\ResultCache\LazyCollectedData;
78
use PHPStan\BetterReflection\NodeCompiler\Exception\UnableToCompileNode;
89
use PHPStan\BetterReflection\Reflection\Exception\CircularReference;
910
use PHPStan\BetterReflection\Reflector\Exception\IdentifierNotFound;
@@ -67,7 +68,7 @@ public function finalize(
6768
array $previousCollectedDataRuleResults = [],
6869
): FinalizerResult
6970
{
70-
if (count($analyserResult->getCollectedData()) === 0) {
71+
if ($analyserResult->getLazyCollectedData()->isEmpty()) {
7172
return $this->addUnmatchedIgnoredErrors($this->mergeFilteredPhpErrors($analyserResult), [], []);
7273
}
7374

@@ -76,8 +77,17 @@ public function finalize(
7677
return $this->addUnmatchedIgnoredErrors($this->mergeFilteredPhpErrors($analyserResult), [], []);
7778
}
7879

80+
try {
81+
$collectedData = $analyserResult->getCollectedData();
82+
} catch (Throwable $t) {
83+
if ($debug) {
84+
throw $t;
85+
}
86+
87+
return $this->addUnmatchedIgnoredErrors($this->mergeFilteredPhpErrors($this->withUnreadableCollectedData($analyserResult, $t)), [], []);
88+
}
89+
7990
$nodeType = CollectedDataNode::class;
80-
$collectedData = $analyserResult->getCollectedData();
8191
$node = new CollectedDataNode($collectedData, $onlyFiles);
8292

8393
// Hashed once per collector type and run, in the same order however the files were merged.
@@ -223,7 +233,7 @@ public function finalize(
223233
linesToIgnore: $allLinesToIgnore,
224234
unmatchedLineIgnores: $allUnmatchedLineIgnores,
225235
internalErrors: $internalErrors,
226-
collectedData: $analyserResult->getCollectedData(),
236+
collectedData: $analyserResult->getLazyCollectedData(),
227237
dependencies: $analyserResult->getDependencies(),
228238
usedTraitDependencies: $analyserResult->getUsedTraitDependencies(),
229239
valueDependencies: $analyserResult->getValueDependencies(),
@@ -319,6 +329,42 @@ private function addRuleErrors(array $errors, CollectedDataNode $node, Scope $sc
319329
}
320330
}
321331

332+
/**
333+
* The result cache was discarded by the reader, so the run reports the failure and the collector
334+
* rules do not run; the next run analyses everything and starts a new cache.
335+
*/
336+
private function withUnreadableCollectedData(AnalyserResult $analyserResult, Throwable $t): AnalyserResult
337+
{
338+
$internalErrors = $analyserResult->getInternalErrors();
339+
$internalErrors[] = new InternalError(
340+
$t->getMessage(),
341+
'reading the collected data from the result cache',
342+
InternalError::prepareTrace($t),
343+
$t->getTraceAsString(),
344+
shouldReportBug: false,
345+
);
346+
347+
return new AnalyserResult(
348+
unorderedErrors: $analyserResult->getUnorderedErrors(),
349+
filteredPhpErrors: $analyserResult->getFilteredPhpErrors(),
350+
allPhpErrors: $analyserResult->getAllPhpErrors(),
351+
locallyIgnoredErrors: $analyserResult->getLocallyIgnoredErrors(),
352+
linesToIgnore: $analyserResult->getLinesToIgnore(),
353+
unmatchedLineIgnores: $analyserResult->getUnmatchedLineIgnores(),
354+
internalErrors: $internalErrors,
355+
collectedData: LazyCollectedData::fromArray([]),
356+
dependencies: $analyserResult->getDependencies(),
357+
usedTraitDependencies: $analyserResult->getUsedTraitDependencies(),
358+
valueDependencies: $analyserResult->getValueDependencies(),
359+
packageDependencies: $analyserResult->getPackageDependencies(),
360+
exportedNodes: $analyserResult->getExportedNodes(),
361+
reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(),
362+
peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(),
363+
processedFiles: $analyserResult->getProcessedFiles(),
364+
workerCount: $analyserResult->getWorkerCount(),
365+
);
366+
}
367+
322368
private function mergeFilteredPhpErrors(AnalyserResult $analyserResult): AnalyserResult
323369
{
324370
return new AnalyserResult(
@@ -329,7 +375,7 @@ private function mergeFilteredPhpErrors(AnalyserResult $analyserResult): Analyse
329375
linesToIgnore: $analyserResult->getLinesToIgnore(),
330376
unmatchedLineIgnores: $analyserResult->getUnmatchedLineIgnores(),
331377
internalErrors: $analyserResult->getInternalErrors(),
332-
collectedData: $analyserResult->getCollectedData(),
378+
collectedData: $analyserResult->getLazyCollectedData(),
333379
dependencies: $analyserResult->getDependencies(),
334380
usedTraitDependencies: $analyserResult->getUsedTraitDependencies(),
335381
valueDependencies: $analyserResult->getValueDependencies(),
@@ -400,7 +446,7 @@ private function addUnmatchedIgnoredErrors(
400446
linesToIgnore: $analyserResult->getLinesToIgnore(),
401447
unmatchedLineIgnores: $analyserResult->getUnmatchedLineIgnores(),
402448
internalErrors: $analyserResult->getInternalErrors(),
403-
collectedData: $analyserResult->getCollectedData(),
449+
collectedData: $analyserResult->getLazyCollectedData(),
404450
dependencies: $analyserResult->getDependencies(),
405451
usedTraitDependencies: $analyserResult->getUsedTraitDependencies(),
406452
valueDependencies: $analyserResult->getValueDependencies(),

‎src/Analyser/ResultCache/LazyCollectedData.php‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<?php declare(strict_types = 1);
2+
3+
namespace PHPStan\Analyser\ResultCache;
4+
5+
use Closure;
6+
use PHPStan\Collectors\CollectedData;
7+
use PHPStan\ShouldNotHappenException;
8+
use function array_key_exists;
9+
use function count;
10+
11+
/**
12+
* Collected data that is read from the result cache file only when something asks for it.
13+
*
14+
* The cached part is an index of where each analysed file's entry sits in the cache file; the
15+
* fresh part is what this run's analysis produced. toArray() reads the cached entries and lays the
16+
* fresh ones over them. Until then, the main process holds a few bytes per file, which is all the
17+
* forked workers inherit.
18+
*
19+
* @phpstan-import-type CollectorData from CollectedData
20+
*/
21+
final class LazyCollectedData
22+
{
23+
24+
/**
25+
* @param array<string, array{int, int}> $cachedIndex analysed file => [offset, length] of its entry in the cache file
26+
* @param Closure(array<string, array{int, int}>): CollectorData $cachedReader
27+
* @param CollectorData $fresh
28+
*/
29+
public function __construct(
30+
private array $cachedIndex,
31+
private Closure $cachedReader,
32+
private array $fresh,
33+
)
34+
{
35+
}
36+
37+
/**
38+
* @param CollectorData $data
39+
*/
40+
public static function fromArray(array $data): self
41+
{
42+
return new self([], static fn (): array => [], $data);
43+
}
44+
45+
/**
46+
* Editor mode analyses a temporary copy of a file and reports it under the file's own path. Only
47+
* the fresh side can hold the temporary path: ResultCacheManager::mergeCollectedData() takes both
48+
* paths out of the cached index, and a cached entry is verified against its file on reading, so it
49+
* could not be renamed anyway.
50+
*/
51+
public function withRenamedFile(string $from, string $to): self
52+
{
53+
if (array_key_exists($from, $this->cachedIndex)) {
54+
throw new ShouldNotHappenException();
55+
}
56+
57+
$fresh = [];
58+
foreach ($this->fresh as $file => $data) {
59+
$fresh[$file === $from ? $to : $file] = $data;
60+
}
61+
62+
return new self($this->cachedIndex, $this->cachedReader, $fresh);
63+
}
64+
65+
/**
66+
* @return array<string, array{int, int}>
67+
*/
68+
public function getCachedIndex(): array
69+
{
70+
return $this->cachedIndex;
71+
}
72+
73+
/**
74+
* @return CollectorData
75+
*/
76+
public function getFresh(): array
77+
{
78+
return $this->fresh;
79+
}
80+
81+
public function isEmpty(): bool
82+
{
83+
return count($this->cachedIndex) === 0 && count($this->fresh) === 0;
84+
}
85+
86+
/**
87+
* Cached entries first, then the fresh ones; a fresh entry replaces a cached one for the same
88+
* file. Reads the cache file every time: nothing is kept, so the caller decides how long the
89+
* decoded data lives.
90+
*
91+
* @return CollectorData
92+
*/
93+
public function toArray(): array
94+
{
95+
if (count($this->cachedIndex) === 0) {
96+
return $this->fresh;
97+
}
98+
99+
$result = ($this->cachedReader)($this->cachedIndex);
100+
foreach ($this->fresh as $file => $data) {
101+
$result[$file] = $data;
102+
}
103+
104+
return $result;
105+
}
106+
107+
}

‎src/Analyser/ResultCache/ResultCache.php‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55
use PHPStan\Analyser\CollectedDataRuleResult;
66
use PHPStan\Analyser\Error;
77
use PHPStan\Analyser\FileAnalyserResult;
8-
use PHPStan\Collectors\CollectedData;
98
use PHPStan\Dependency\RootExportedNode;
109

1110
/**
1211
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
13-
* @phpstan-import-type CollectorData from CollectedData
1412
*/
1513
final class ResultCache
1614
{
@@ -22,7 +20,6 @@ final class ResultCache
2220
* @param array<string, list<Error>> $locallyIgnoredErrors
2321
* @param array<string, LinesToIgnore> $linesToIgnore
2422
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
25-
* @param CollectorData $collectedData
2623
* @param array<string, array<string>> $dependencies
2724
* @param array<string, array<string>> $usedTraitDependencies
2825
* @param array<string, array{string, string, string, list<string>, list<string>}> $valueDependencies id => [extension class, key, current value, analysis dependents, declarations dependents] - see ValueDependencyCollector
@@ -42,7 +39,7 @@ public function __construct(
4239
private array $locallyIgnoredErrors,
4340
private array $linesToIgnore,
4441
private array $unmatchedLineIgnores,
45-
private array $collectedData,
42+
private LazyCollectedData $collectedData,
4643
private array $dependencies,
4744
private array $usedTraitDependencies,
4845
private array $valueDependencies,
@@ -123,10 +120,7 @@ public function getUnmatchedLineIgnores(): array
123120
return $this->unmatchedLineIgnores;
124121
}
125122

126-
/**
127-
* @return CollectorData
128-
*/
129-
public function getCollectedData(): array
123+
public function getCollectedData(): LazyCollectedData
130124
{
131125
return $this->collectedData;
132126
}

0 commit comments

Comments
 (0)