Skip to content

Commit 08d8557

Browse files
ondrejmirtesclaude
andcommitted
Multi-file result cache format (bleeding edge)
Loading a var_export()ed PHP result cache file costs several times the file size in the PHP compiler - arena, interned strings, and the literal tables of the "lazy" closure sections, which materialize at compile time even when the callbacks are never invoked. For the 39.5 MB self-analysis cache that meant +192 MB retained and a 291 MB peak in the main process on every warm run. With the multiFileResultCache bleeding edge feature toggle, the heavy sections (errors, locally ignored errors, collected data, exported nodes) are stored in sibling resultCache-*.dat files as serialize()d pure data - objects are converted through jsonSerialize(), the same representation the parallel workers use to send results to the main process, and decoded back with the same decode() methods. The main file references them via errorsFilePath etc. instead of embedding errorsCallback etc. Section files are written before the main file and the whole serialized section is never built in memory at once. Some CI setups only restore resultCache.php and not the whole directory, which is why this needs the bleeding edge opt-in: a missing referenced section file fails loudly (after clearing the main file so that the next run recovers with a full analysis). Without bleeding edge, the single-file format stays exactly as before. In self-analysis the warm run restore peak drops from 291 MB to 158 MB, warm run elapsed time from ~2.5 s to ~1.5 s, and the files on disk shrink from 39.5 MB to 28.7 MB in total. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017DjgMHgfvwhD4ogdTaRtLp
1 parent ee9fe9e commit 08d8557

5 files changed

Lines changed: 222 additions & 4 deletions

File tree

‎conf/bleedingEdge.neon‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ parameters:
2222
checkDynamicConstantNameValues: true
2323
unusedLabel: true
2424
newOnNonObject: true
25+
multiFileResultCache: true

‎conf/config.neon‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ parameters:
4949
checkDynamicConstantNameValues: false
5050
unusedLabel: false
5151
newOnNonObject: false
52+
multiFileResultCache: false
5253
fileExtensions:
5354
- php
5455
checkAdvancedIsset: false

‎conf/parametersSchema.neon‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ parametersSchema:
5151
checkDynamicConstantNameValues: bool()
5252
unusedLabel: bool()
5353
newOnNonObject: bool()
54+
multiFileResultCache: bool()
5455
])
5556
fileExtensions: listOf(string())
5657
checkAdvancedIsset: bool()

‎src/Analyser/ResultCache/ResultCacheClearer.php‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
use PHPStan\DependencyInjection\AutowiredService;
77
use function dirname;
88
use function is_file;
9+
use function str_ends_with;
10+
use function substr;
911
use function unlink;
1012

1113
#[AutowiredService]
@@ -22,6 +24,20 @@ public function __construct(
2224
public function clear(): string
2325
{
2426
$dir = dirname($this->cacheFilePath);
27+
28+
$basePath = $this->cacheFilePath;
29+
if (str_ends_with($basePath, '.php')) {
30+
$basePath = substr($basePath, 0, -4);
31+
}
32+
foreach (['errors', 'locallyIgnoredErrors', 'collectedData', 'exportedNodes'] as $section) {
33+
$sectionFilePath = $basePath . '-' . $section . '.dat';
34+
if (!is_file($sectionFilePath)) {
35+
continue;
36+
}
37+
38+
@unlink($sectionFilePath);
39+
}
40+
2541
if (!is_file($this->cacheFilePath)) {
2642
return $dir;
2743
}

‎src/Analyser/ResultCache/ResultCacheManager.php‎

Lines changed: 203 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
namespace PHPStan\Analyser\ResultCache;
44

5+
use JsonSerializable;
56
use Nette\Neon\Neon;
67
use PHPStan\Analyser\AnalyserResult;
78
use PHPStan\Analyser\Error;
@@ -20,6 +21,7 @@
2021
use PHPStan\File\CouldNotWriteFileException;
2122
use PHPStan\File\FileFinder;
2223
use PHPStan\File\FileHelper;
24+
use PHPStan\File\FileReader;
2325
use PHPStan\Internal\ArrayHelper;
2426
use PHPStan\Internal\ComposerHelper;
2527
use PHPStan\PhpDoc\StubFilesProvider;
@@ -32,6 +34,7 @@
3234
use function array_filter;
3335
use function array_key_exists;
3436
use function array_keys;
37+
use function array_map;
3538
use function array_merge;
3639
use function array_unique;
3740
use function array_values;
@@ -50,12 +53,15 @@
5053
use function is_file;
5154
use function ksort;
5255
use function microtime;
56+
use function serialize;
5357
use function sort;
5458
use function sprintf;
59+
use function str_ends_with;
5560
use function str_starts_with;
5661
use function substr;
5762
use function time;
5863
use function unlink;
64+
use function unserialize;
5965
use function var_export;
6066
use const PHP_VERSION_ID;
6167

@@ -118,6 +124,8 @@ public function __construct(
118124
private array $parametersNotInvalidatingCache,
119125
#[AutowiredParameter(ref: '%resultCacheSkipIfOlderThanDays%')]
120126
private int $skipResultCacheIfOlderThanDays,
127+
#[AutowiredParameter(ref: '%featureToggles.multiFileResultCache%')]
128+
private bool $multiFileResultCache,
121129
)
122130
{
123131
}
@@ -402,12 +410,21 @@ public function restore(array $allAnalysedFiles, bool $debug, bool $onlyFiles, ?
402410
$filesToAnalyse = [];
403411
$invertedDependenciesToReturn = [];
404412
$invertedUsedTraitDependenciesToReturn = [];
405-
$errors = $data['errorsCallback']();
406-
$locallyIgnoredErrors = $data['locallyIgnoredErrorsCallback']();
413+
if (array_key_exists('errorsFilePath', $data)) {
414+
// bleeding edge multi-file format - a missing referenced file throws
415+
// (usually a CI restoring only resultCache.php and not the whole directory)
416+
$errors = $this->decodeErrorsSection($this->loadSectionFile($data['errorsFilePath']));
417+
$locallyIgnoredErrors = $this->decodeErrorsSection($this->loadSectionFile($data['locallyIgnoredErrorsFilePath']));
418+
$collectedData = $this->loadSectionFile($data['collectedDataFilePath']);
419+
$exportedNodes = $this->decodeExportedNodesSection($this->loadSectionFile($data['exportedNodesFilePath']));
420+
} else {
421+
$errors = $data['errorsCallback']();
422+
$locallyIgnoredErrors = $data['locallyIgnoredErrorsCallback']();
423+
$collectedData = $data['collectedDataCallback']();
424+
$exportedNodes = $data['exportedNodesCallback']();
425+
}
407426
$linesToIgnore = $data['linesToIgnore'];
408427
$unmatchedLineIgnores = $data['unmatchedLineIgnores'];
409-
$collectedData = $data['collectedDataCallback']();
410-
$exportedNodes = $data['exportedNodesCallback']();
411428
$filteredErrors = [];
412429
$filteredLocallyIgnoredErrors = [];
413430
$filteredLinesToIgnore = [];
@@ -1158,6 +1175,57 @@ private function save(
11581175

11591176
$file = $this->cacheFilePath;
11601177

1178+
if ($this->multiFileResultCache) {
1179+
// bleeding edge multi-file format: the heavy sections are stored as pure data
1180+
// in sibling files, decoded on restore. Loading them costs only the memory
1181+
// of the data itself, while require of a var_export()ed PHP file costs
1182+
// several times more in the PHP compiler even when lazy-loaded via closures.
1183+
// The section files are written before the main file so that the main file
1184+
// never references files that do not exist yet.
1185+
$sectionFilePaths = $this->getSectionFilePaths();
1186+
$this->saveSectionFile($sectionFilePaths['errors'], $errors);
1187+
$this->saveSectionFile($sectionFilePaths['locallyIgnoredErrors'], $locallyIgnoredErrors);
1188+
$this->saveSectionFile($sectionFilePaths['collectedData'], $collectedData);
1189+
$this->saveSectionFile($sectionFilePaths['exportedNodes'], $exportedNodes);
1190+
1191+
$handle = @fopen($file, 'w');
1192+
if ($handle === false) {
1193+
$error = error_get_last();
1194+
throw new CouldNotWriteFileException($file, $error !== null ? $error['message'] : 'unknown cause');
1195+
}
1196+
1197+
try {
1198+
$this->writeToHandle($handle, $file, "<?php declare(strict_types = 1);
1199+
1200+
return [
1201+
'lastFullAnalysisTime' => " . var_export($lastFullAnalysisTime, true) . ",
1202+
'meta' => " . var_export($meta, true) . ",
1203+
'projectExtensionFiles' => " . var_export($projectExtensionFiles, true) . ",
1204+
'errorsFilePath' => " . var_export($sectionFilePaths['errors'], true) . ",
1205+
'locallyIgnoredErrorsFilePath' => " . var_export($sectionFilePaths['locallyIgnoredErrors'], true) . ",
1206+
'linesToIgnore' => ");
1207+
$this->streamArrayVarExportToHandle($handle, $file, $linesToIgnore);
1208+
$this->writeToHandle($handle, $file, ",
1209+
'unmatchedLineIgnores' => ");
1210+
$this->streamArrayVarExportToHandle($handle, $file, $unmatchedLineIgnores);
1211+
$this->writeToHandle($handle, $file, ",
1212+
'collectedDataFilePath' => " . var_export($sectionFilePaths['collectedData'], true) . ",
1213+
'dependencies' => ");
1214+
$this->streamArrayVarExportToHandle($handle, $file, $invertedDependencies);
1215+
$this->writeToHandle($handle, $file, ",
1216+
'packageDependencies' => ");
1217+
$this->streamArrayVarExportToHandle($handle, $file, $packageDependencies);
1218+
$this->writeToHandle($handle, $file, ",
1219+
'exportedNodesFilePath' => " . var_export($sectionFilePaths['exportedNodes'], true) . ',
1220+
];
1221+
');
1222+
} finally {
1223+
fclose($handle);
1224+
}
1225+
1226+
return;
1227+
}
1228+
11611229
// streamed to the file section by section - building the whole
11621230
// var_export()ed contents in memory at once would take up roughly
11631231
// twice the size of the resulting file in the main process
@@ -1216,6 +1284,137 @@ private function writeToHandle($handle, string $file, string $contents): void
12161284
}
12171285
}
12181286

1287+
/**
1288+
* @return array{errors: string, locallyIgnoredErrors: string, collectedData: string, exportedNodes: string}
1289+
*/
1290+
private function getSectionFilePaths(): array
1291+
{
1292+
$basePath = $this->cacheFilePath;
1293+
if (str_ends_with($basePath, '.php')) {
1294+
$basePath = substr($basePath, 0, -4);
1295+
}
1296+
1297+
return [
1298+
'errors' => $basePath . '-errors.dat',
1299+
'locallyIgnoredErrors' => $basePath . '-locallyIgnoredErrors.dat',
1300+
'collectedData' => $basePath . '-collectedData.dat',
1301+
'exportedNodes' => $basePath . '-exportedNodes.dat',
1302+
];
1303+
}
1304+
1305+
/**
1306+
* Writes serialize() output of pure data (no objects). The outer array is
1307+
* hand-assembled in the serialize() format so that the whole serialized
1308+
* section never has to be built in memory at once.
1309+
*
1310+
* Objects are converted to data through jsonSerialize(), the same
1311+
* representation the parallel workers use to send their results
1312+
* to the main process.
1313+
*
1314+
* @param array<string, mixed> $dataPerFile
1315+
*/
1316+
private function saveSectionFile(string $filePath, array $dataPerFile): void
1317+
{
1318+
$handle = @fopen($filePath, 'w');
1319+
if ($handle === false) {
1320+
$error = error_get_last();
1321+
throw new CouldNotWriteFileException($filePath, $error !== null ? $error['message'] : 'unknown cause');
1322+
}
1323+
1324+
try {
1325+
$this->writeToHandle($handle, $filePath, 'a:' . count($dataPerFile) . ':{');
1326+
foreach ($dataPerFile as $analysedFile => $entries) {
1327+
$this->writeToHandle($handle, $filePath, serialize($analysedFile) . serialize($this->toPureData($entries)));
1328+
}
1329+
1330+
$this->writeToHandle($handle, $filePath, '}');
1331+
} finally {
1332+
fclose($handle);
1333+
}
1334+
}
1335+
1336+
private function toPureData(mixed $value): mixed
1337+
{
1338+
if ($value instanceof JsonSerializable) {
1339+
return $this->toPureData($value->jsonSerialize());
1340+
}
1341+
1342+
if (is_array($value)) {
1343+
$result = [];
1344+
foreach ($value as $key => $item) {
1345+
$result[$key] = $this->toPureData($item);
1346+
}
1347+
1348+
return $result;
1349+
}
1350+
1351+
return $value;
1352+
}
1353+
1354+
/**
1355+
* @return array<string, mixed>
1356+
*/
1357+
private function loadSectionFile(string $filePath): array
1358+
{
1359+
if (!is_file($filePath)) {
1360+
// clear the main cache file so that the next run does not fail the same way
1361+
@unlink($this->cacheFilePath);
1362+
1363+
throw new CouldNotReadFileException($filePath);
1364+
}
1365+
1366+
$contents = FileReader::read($filePath);
1367+
$data = @unserialize($contents, ['allowed_classes' => false]);
1368+
if (!is_array($data)) {
1369+
@unlink($this->cacheFilePath);
1370+
@unlink($filePath);
1371+
1372+
throw new ShouldNotHappenException(sprintf('Result cache file %s is corrupted.', $filePath));
1373+
}
1374+
1375+
return $data;
1376+
}
1377+
1378+
/**
1379+
* @param array<string, mixed> $section
1380+
* @return array<string, list<Error>>
1381+
*/
1382+
private function decodeErrorsSection(array $section): array
1383+
{
1384+
$result = [];
1385+
foreach ($section as $analysedFile => $errorsData) {
1386+
/** @var list<mixed[]> $errorsData */
1387+
foreach ($errorsData as $errorData) {
1388+
$result[$analysedFile][] = Error::decode($errorData);
1389+
}
1390+
}
1391+
1392+
return $result;
1393+
}
1394+
1395+
/**
1396+
* @param array<string, mixed> $section
1397+
* @return array<string, RootExportedNode[]>
1398+
*/
1399+
private function decodeExportedNodesSection(array $section): array
1400+
{
1401+
$result = [];
1402+
foreach ($section as $analysedFile => $nodesData) {
1403+
/** @var list<array{type: class-string<RootExportedNode>, data: mixed[]}> $nodesData */
1404+
$result[$analysedFile] = array_map(static function (array $node): RootExportedNode {
1405+
$class = $node['type'];
1406+
$decoded = $class::decode($node['data']);
1407+
if (!$decoded instanceof RootExportedNode) {
1408+
throw new ShouldNotHappenException();
1409+
}
1410+
1411+
return $decoded;
1412+
}, $nodesData);
1413+
}
1414+
1415+
return $result;
1416+
}
1417+
12191418
/**
12201419
* Streams the var_export() representation of an array to the file entry
12211420
* by entry, producing output byte-identical to var_export($values, true).

0 commit comments

Comments
 (0)