|
2 | 2 |
|
3 | 3 | namespace PHPStan\Analyser\ResultCache; |
4 | 4 |
|
| 5 | +use JsonSerializable; |
5 | 6 | use Nette\Neon\Neon; |
6 | 7 | use PHPStan\Analyser\AnalyserResult; |
7 | 8 | use PHPStan\Analyser\Error; |
|
20 | 21 | use PHPStan\File\CouldNotWriteFileException; |
21 | 22 | use PHPStan\File\FileFinder; |
22 | 23 | use PHPStan\File\FileHelper; |
| 24 | +use PHPStan\File\FileReader; |
23 | 25 | use PHPStan\Internal\ArrayHelper; |
24 | 26 | use PHPStan\Internal\ComposerHelper; |
25 | 27 | use PHPStan\PhpDoc\StubFilesProvider; |
|
32 | 34 | use function array_filter; |
33 | 35 | use function array_key_exists; |
34 | 36 | use function array_keys; |
| 37 | +use function array_map; |
35 | 38 | use function array_merge; |
36 | 39 | use function array_unique; |
37 | 40 | use function array_values; |
|
50 | 53 | use function is_file; |
51 | 54 | use function ksort; |
52 | 55 | use function microtime; |
| 56 | +use function serialize; |
53 | 57 | use function sort; |
54 | 58 | use function sprintf; |
| 59 | +use function str_ends_with; |
55 | 60 | use function str_starts_with; |
56 | 61 | use function substr; |
57 | 62 | use function time; |
58 | 63 | use function unlink; |
| 64 | +use function unserialize; |
59 | 65 | use function var_export; |
60 | 66 | use const PHP_VERSION_ID; |
61 | 67 |
|
@@ -118,6 +124,8 @@ public function __construct( |
118 | 124 | private array $parametersNotInvalidatingCache, |
119 | 125 | #[AutowiredParameter(ref: '%resultCacheSkipIfOlderThanDays%')] |
120 | 126 | private int $skipResultCacheIfOlderThanDays, |
| 127 | + #[AutowiredParameter(ref: '%featureToggles.multiFileResultCache%')] |
| 128 | + private bool $multiFileResultCache, |
121 | 129 | ) |
122 | 130 | { |
123 | 131 | } |
@@ -402,12 +410,21 @@ public function restore(array $allAnalysedFiles, bool $debug, bool $onlyFiles, ? |
402 | 410 | $filesToAnalyse = []; |
403 | 411 | $invertedDependenciesToReturn = []; |
404 | 412 | $invertedUsedTraitDependenciesToReturn = []; |
405 | | - $errors = $data['errorsCallback'](); |
406 | | - $locallyIgnoredErrors = $data['locallyIgnoredErrorsCallback'](); |
| 413 | + if (array_key_exists('errorsFilePath', $data)) { |
| 414 | + // bleeding edge multi-file format - a missing referenced file throws |
| 415 | + // (usually a CI restoring only resultCache.php and not the whole directory) |
| 416 | + $errors = $this->decodeErrorsSection($this->loadSectionFile($data['errorsFilePath'])); |
| 417 | + $locallyIgnoredErrors = $this->decodeErrorsSection($this->loadSectionFile($data['locallyIgnoredErrorsFilePath'])); |
| 418 | + $collectedData = $this->loadSectionFile($data['collectedDataFilePath']); |
| 419 | + $exportedNodes = $this->decodeExportedNodesSection($this->loadSectionFile($data['exportedNodesFilePath'])); |
| 420 | + } else { |
| 421 | + $errors = $data['errorsCallback'](); |
| 422 | + $locallyIgnoredErrors = $data['locallyIgnoredErrorsCallback'](); |
| 423 | + $collectedData = $data['collectedDataCallback'](); |
| 424 | + $exportedNodes = $data['exportedNodesCallback'](); |
| 425 | + } |
407 | 426 | $linesToIgnore = $data['linesToIgnore']; |
408 | 427 | $unmatchedLineIgnores = $data['unmatchedLineIgnores']; |
409 | | - $collectedData = $data['collectedDataCallback'](); |
410 | | - $exportedNodes = $data['exportedNodesCallback'](); |
411 | 428 | $filteredErrors = []; |
412 | 429 | $filteredLocallyIgnoredErrors = []; |
413 | 430 | $filteredLinesToIgnore = []; |
@@ -1158,6 +1175,57 @@ private function save( |
1158 | 1175 |
|
1159 | 1176 | $file = $this->cacheFilePath; |
1160 | 1177 |
|
| 1178 | + if ($this->multiFileResultCache) { |
| 1179 | + // bleeding edge multi-file format: the heavy sections are stored as pure data |
| 1180 | + // in sibling files, decoded on restore. Loading them costs only the memory |
| 1181 | + // of the data itself, while require of a var_export()ed PHP file costs |
| 1182 | + // several times more in the PHP compiler even when lazy-loaded via closures. |
| 1183 | + // The section files are written before the main file so that the main file |
| 1184 | + // never references files that do not exist yet. |
| 1185 | + $sectionFilePaths = $this->getSectionFilePaths(); |
| 1186 | + $this->saveSectionFile($sectionFilePaths['errors'], $errors); |
| 1187 | + $this->saveSectionFile($sectionFilePaths['locallyIgnoredErrors'], $locallyIgnoredErrors); |
| 1188 | + $this->saveSectionFile($sectionFilePaths['collectedData'], $collectedData); |
| 1189 | + $this->saveSectionFile($sectionFilePaths['exportedNodes'], $exportedNodes); |
| 1190 | + |
| 1191 | + $handle = @fopen($file, 'w'); |
| 1192 | + if ($handle === false) { |
| 1193 | + $error = error_get_last(); |
| 1194 | + throw new CouldNotWriteFileException($file, $error !== null ? $error['message'] : 'unknown cause'); |
| 1195 | + } |
| 1196 | + |
| 1197 | + try { |
| 1198 | + $this->writeToHandle($handle, $file, "<?php declare(strict_types = 1); |
| 1199 | +
|
| 1200 | +return [ |
| 1201 | + 'lastFullAnalysisTime' => " . var_export($lastFullAnalysisTime, true) . ", |
| 1202 | + 'meta' => " . var_export($meta, true) . ", |
| 1203 | + 'projectExtensionFiles' => " . var_export($projectExtensionFiles, true) . ", |
| 1204 | + 'errorsFilePath' => " . var_export($sectionFilePaths['errors'], true) . ", |
| 1205 | + 'locallyIgnoredErrorsFilePath' => " . var_export($sectionFilePaths['locallyIgnoredErrors'], true) . ", |
| 1206 | + 'linesToIgnore' => "); |
| 1207 | + $this->streamArrayVarExportToHandle($handle, $file, $linesToIgnore); |
| 1208 | + $this->writeToHandle($handle, $file, ", |
| 1209 | + 'unmatchedLineIgnores' => "); |
| 1210 | + $this->streamArrayVarExportToHandle($handle, $file, $unmatchedLineIgnores); |
| 1211 | + $this->writeToHandle($handle, $file, ", |
| 1212 | + 'collectedDataFilePath' => " . var_export($sectionFilePaths['collectedData'], true) . ", |
| 1213 | + 'dependencies' => "); |
| 1214 | + $this->streamArrayVarExportToHandle($handle, $file, $invertedDependencies); |
| 1215 | + $this->writeToHandle($handle, $file, ", |
| 1216 | + 'packageDependencies' => "); |
| 1217 | + $this->streamArrayVarExportToHandle($handle, $file, $packageDependencies); |
| 1218 | + $this->writeToHandle($handle, $file, ", |
| 1219 | + 'exportedNodesFilePath' => " . var_export($sectionFilePaths['exportedNodes'], true) . ', |
| 1220 | +]; |
| 1221 | +'); |
| 1222 | + } finally { |
| 1223 | + fclose($handle); |
| 1224 | + } |
| 1225 | + |
| 1226 | + return; |
| 1227 | + } |
| 1228 | + |
1161 | 1229 | // streamed to the file section by section - building the whole |
1162 | 1230 | // var_export()ed contents in memory at once would take up roughly |
1163 | 1231 | // twice the size of the resulting file in the main process |
@@ -1216,6 +1284,137 @@ private function writeToHandle($handle, string $file, string $contents): void |
1216 | 1284 | } |
1217 | 1285 | } |
1218 | 1286 |
|
| 1287 | + /** |
| 1288 | + * @return array{errors: string, locallyIgnoredErrors: string, collectedData: string, exportedNodes: string} |
| 1289 | + */ |
| 1290 | + private function getSectionFilePaths(): array |
| 1291 | + { |
| 1292 | + $basePath = $this->cacheFilePath; |
| 1293 | + if (str_ends_with($basePath, '.php')) { |
| 1294 | + $basePath = substr($basePath, 0, -4); |
| 1295 | + } |
| 1296 | + |
| 1297 | + return [ |
| 1298 | + 'errors' => $basePath . '-errors.dat', |
| 1299 | + 'locallyIgnoredErrors' => $basePath . '-locallyIgnoredErrors.dat', |
| 1300 | + 'collectedData' => $basePath . '-collectedData.dat', |
| 1301 | + 'exportedNodes' => $basePath . '-exportedNodes.dat', |
| 1302 | + ]; |
| 1303 | + } |
| 1304 | + |
| 1305 | + /** |
| 1306 | + * Writes serialize() output of pure data (no objects). The outer array is |
| 1307 | + * hand-assembled in the serialize() format so that the whole serialized |
| 1308 | + * section never has to be built in memory at once. |
| 1309 | + * |
| 1310 | + * Objects are converted to data through jsonSerialize(), the same |
| 1311 | + * representation the parallel workers use to send their results |
| 1312 | + * to the main process. |
| 1313 | + * |
| 1314 | + * @param array<string, mixed> $dataPerFile |
| 1315 | + */ |
| 1316 | + private function saveSectionFile(string $filePath, array $dataPerFile): void |
| 1317 | + { |
| 1318 | + $handle = @fopen($filePath, 'w'); |
| 1319 | + if ($handle === false) { |
| 1320 | + $error = error_get_last(); |
| 1321 | + throw new CouldNotWriteFileException($filePath, $error !== null ? $error['message'] : 'unknown cause'); |
| 1322 | + } |
| 1323 | + |
| 1324 | + try { |
| 1325 | + $this->writeToHandle($handle, $filePath, 'a:' . count($dataPerFile) . ':{'); |
| 1326 | + foreach ($dataPerFile as $analysedFile => $entries) { |
| 1327 | + $this->writeToHandle($handle, $filePath, serialize($analysedFile) . serialize($this->toPureData($entries))); |
| 1328 | + } |
| 1329 | + |
| 1330 | + $this->writeToHandle($handle, $filePath, '}'); |
| 1331 | + } finally { |
| 1332 | + fclose($handle); |
| 1333 | + } |
| 1334 | + } |
| 1335 | + |
| 1336 | + private function toPureData(mixed $value): mixed |
| 1337 | + { |
| 1338 | + if ($value instanceof JsonSerializable) { |
| 1339 | + return $this->toPureData($value->jsonSerialize()); |
| 1340 | + } |
| 1341 | + |
| 1342 | + if (is_array($value)) { |
| 1343 | + $result = []; |
| 1344 | + foreach ($value as $key => $item) { |
| 1345 | + $result[$key] = $this->toPureData($item); |
| 1346 | + } |
| 1347 | + |
| 1348 | + return $result; |
| 1349 | + } |
| 1350 | + |
| 1351 | + return $value; |
| 1352 | + } |
| 1353 | + |
| 1354 | + /** |
| 1355 | + * @return array<string, mixed> |
| 1356 | + */ |
| 1357 | + private function loadSectionFile(string $filePath): array |
| 1358 | + { |
| 1359 | + if (!is_file($filePath)) { |
| 1360 | + // clear the main cache file so that the next run does not fail the same way |
| 1361 | + @unlink($this->cacheFilePath); |
| 1362 | + |
| 1363 | + throw new CouldNotReadFileException($filePath); |
| 1364 | + } |
| 1365 | + |
| 1366 | + $contents = FileReader::read($filePath); |
| 1367 | + $data = @unserialize($contents, ['allowed_classes' => false]); |
| 1368 | + if (!is_array($data)) { |
| 1369 | + @unlink($this->cacheFilePath); |
| 1370 | + @unlink($filePath); |
| 1371 | + |
| 1372 | + throw new ShouldNotHappenException(sprintf('Result cache file %s is corrupted.', $filePath)); |
| 1373 | + } |
| 1374 | + |
| 1375 | + return $data; |
| 1376 | + } |
| 1377 | + |
| 1378 | + /** |
| 1379 | + * @param array<string, mixed> $section |
| 1380 | + * @return array<string, list<Error>> |
| 1381 | + */ |
| 1382 | + private function decodeErrorsSection(array $section): array |
| 1383 | + { |
| 1384 | + $result = []; |
| 1385 | + foreach ($section as $analysedFile => $errorsData) { |
| 1386 | + /** @var list<mixed[]> $errorsData */ |
| 1387 | + foreach ($errorsData as $errorData) { |
| 1388 | + $result[$analysedFile][] = Error::decode($errorData); |
| 1389 | + } |
| 1390 | + } |
| 1391 | + |
| 1392 | + return $result; |
| 1393 | + } |
| 1394 | + |
| 1395 | + /** |
| 1396 | + * @param array<string, mixed> $section |
| 1397 | + * @return array<string, RootExportedNode[]> |
| 1398 | + */ |
| 1399 | + private function decodeExportedNodesSection(array $section): array |
| 1400 | + { |
| 1401 | + $result = []; |
| 1402 | + foreach ($section as $analysedFile => $nodesData) { |
| 1403 | + /** @var list<array{type: class-string<RootExportedNode>, data: mixed[]}> $nodesData */ |
| 1404 | + $result[$analysedFile] = array_map(static function (array $node): RootExportedNode { |
| 1405 | + $class = $node['type']; |
| 1406 | + $decoded = $class::decode($node['data']); |
| 1407 | + if (!$decoded instanceof RootExportedNode) { |
| 1408 | + throw new ShouldNotHappenException(); |
| 1409 | + } |
| 1410 | + |
| 1411 | + return $decoded; |
| 1412 | + }, $nodesData); |
| 1413 | + } |
| 1414 | + |
| 1415 | + return $result; |
| 1416 | + } |
| 1417 | + |
1219 | 1418 | /** |
1220 | 1419 | * Streams the var_export() representation of an array to the file entry |
1221 | 1420 | * by entry, producing output byte-identical to var_export($values, true). |
|
0 commit comments