Skip to content

Commit e652152

Browse files
webardTomasVotruba
authored andcommitted
fix
1 parent f11caf2 commit e652152

4 files changed

Lines changed: 135 additions & 120 deletions

File tree

‎PARALLEL_LPT_POC_NOTES.md‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,12 @@ Nothing below is covered by a test yet — the user asked for a PoC first.
338338
- [ ] drop the `--lpt` flag and `RECTOR_LPT_NO_STEAL` env var, or promote them to real options
339339
- [ ] `MAX_CHUNKS_PER_WORKER` is dead in bucket mode — decide its fate (see the memory-driven
340340
recycle item above, which would give it a real job again)
341-
- [ ] fold `ExperimentalParallelFileProcessor` back into `ParallelFileProcessor` — it is a near-copy,
342-
kept separate only so the default path stays provably untouched, and it will trip the
343-
duplicate-code gate
341+
- [x] the shared result accumulation is extracted into `ParallelResultCollector` — this is what the
342+
`duplicated_code` CI gate caught (169 tokens against a 150 threshold). Default-path behaviour
343+
verified unchanged: `vendor/` before and after gives 3 338 diffs, identical sha, same 2 errors.
344+
The system-error counter stayed in the processors on purpose — three sites append an error but
345+
only two increment it, and unifying that would move when the 50-error limit trips.
346+
- [ ] fold `ExperimentalParallelFileProcessor` back into `ParallelFileProcessor` — the TCP wiring and
347+
the spawn loop are still parallel structures, kept apart only so the default path stays
348+
provably untouched
344349
- [ ] `composer check-cs`, `composer phpstan`, `vendor/bin/phpunit`

‎src/Parallel/Application/ParallelFileProcessor.php‎

Lines changed: 18 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
use Rector\Parallel\ValueObject\Schedule;
2626
use Rector\ValueObject\Error\SystemError;
2727
use Rector\ValueObject\ProcessResult;
28-
use Rector\ValueObject\Reporting\FileDiff;
2928
use Symfony\Component\Console\Command\Command;
3029
use Symfony\Component\Console\Input\InputInterface;
3130
use Throwable;
@@ -68,16 +67,7 @@ public function process(
6867
// basic properties setup
6968
$numberOfProcesses = $schedule->getNumberOfProcesses();
7069

71-
// initial counters
72-
73-
/** @var FileDiff[] $fileDiffs */
74-
$fileDiffs = [];
75-
76-
/** @var SystemError[] $systemErrors */
77-
$systemErrors = [];
78-
79-
/** @var array<string, array<string, true>> $usedSkips */
80-
$usedSkips = [];
70+
$parallelResultCollector = new ParallelResultCollector();
8171

8272
$tcpServer = new TcpServer('127.0.0.1:0', $streamSelectLoop);
8373
$this->processPool = new ProcessPool($tcpServer);
@@ -118,20 +108,19 @@ public function process(
118108

119109
$systemErrorsCount = 0;
120110
$reachedSystemErrorsCountLimit = false;
121-
$totalChanged = 0;
122111
$scheduledFilesCount = array_sum(array_map(count(...), $jobs));
123112
$processedFilesCount = 0;
124113

125114
$handleErrorCallable = function (Throwable $throwable) use (
126-
&$systemErrors,
115+
$parallelResultCollector,
127116
&$systemErrorsCount,
128117
&$reachedSystemErrorsCountLimit
129118
): void {
130-
$systemErrors[] = new SystemError(
119+
$parallelResultCollector->collectSystemError(new SystemError(
131120
$throwable->getMessage(),
132121
$throwable->getFile(),
133122
$throwable->getLine(),
134-
);
123+
));
135124

136125
++$systemErrorsCount;
137126
$reachedSystemErrorsCountLimit = true;
@@ -147,9 +136,7 @@ public function process(
147136
$fileChunksBudgetPerProcess = [];
148137

149138
$processSpawner = function () use (
150-
&$systemErrors,
151-
&$fileDiffs,
152-
&$usedSkips,
139+
$parallelResultCollector,
153140
&$jobs,
154141
$postFileCallback,
155142
&$systemErrorsCount,
@@ -162,7 +149,6 @@ public function process(
162149
$handleErrorCallable,
163150
&$fileChunksBudgetPerProcess,
164151
&$processSpawner,
165-
&$totalChanged,
166152
&$processedFilesCount
167153
): void {
168154
$processIdentifier = Random::generate();
@@ -182,17 +168,14 @@ public function process(
182168
// 1. callable on data
183169
function (array $json) use (
184170
$parallelProcess,
185-
&$systemErrors,
186-
&$fileDiffs,
187-
&$usedSkips,
171+
$parallelResultCollector,
188172
&$jobs,
189173
$postFileCallback,
190174
&$systemErrorsCount,
191175
&$reachedSystemErrorsCountLimit,
192176
$processIdentifier,
193177
&$fileChunksBudgetPerProcess,
194178
&$processSpawner,
195-
&$totalChanged,
196179
&$processedFilesCount
197180
): void {
198181
/** @var array{
@@ -203,31 +186,9 @@ function (array $json) use (
203186
* system_errors_count: int,
204187
* used_skips: array<string, string[]>
205188
* } $json */
206-
$totalChanged += $json[Bridge::TOTAL_CHANGED];
207-
208-
foreach ($json[Bridge::USED_SKIPS] as $skip => $paths) {
209-
$usedSkips[$skip] ??= [];
210-
foreach ($paths as $path) {
211-
$usedSkips[$skip][$path] = true;
212-
}
213-
}
214-
215-
// decode arrays to objects
216-
foreach ($json[Bridge::SYSTEM_ERRORS] as $jsonError) {
217-
if (is_string($jsonError)) {
218-
$systemErrors[] = new SystemError('System error: ' . $jsonError);
219-
continue;
220-
}
221-
222-
$systemErrors[] = SystemError::decode($jsonError);
223-
}
224-
225-
foreach ($json[Bridge::FILE_DIFFS] as $jsonFileDiff) {
226-
$fileDiffs[] = FileDiff::decode($jsonFileDiff);
227-
}
228-
229-
$postFileCallback($json[Bridge::FILES_COUNT]);
230-
$processedFilesCount += $json[Bridge::FILES_COUNT];
189+
$filesCount = $parallelResultCollector->collectWorkerResult($json);
190+
$postFileCallback($filesCount);
191+
$processedFilesCount += $filesCount;
231192

232193
$systemErrorsCount += $json[Bridge::SYSTEM_ERRORS_COUNT];
233194
if ($systemErrorsCount >= self::SYSTEM_ERROR_LIMIT) {
@@ -261,7 +222,7 @@ function (array $json) use (
261222
$handleErrorCallable,
262223

263224
// 3. callable on exit
264-
function ($exitCode, string $stdErr) use (&$systemErrors, $processIdentifier): void {
225+
function ($exitCode, string $stdErr) use ($parallelResultCollector, $processIdentifier): void {
265226
$this->processPool->tryQuitProcess($processIdentifier);
266227
if ($exitCode === Command::SUCCESS) {
267228
return;
@@ -271,7 +232,9 @@ function ($exitCode, string $stdErr) use (&$systemErrors, $processIdentifier): v
271232
return;
272233
}
273234

274-
$systemErrors[] = new SystemError('Child process error: ' . $stdErr);
235+
$parallelResultCollector->collectSystemError(new SystemError(
236+
'Child process error: ' . $stdErr
237+
));
275238
}
276239
);
277240

@@ -290,26 +253,21 @@ function ($exitCode, string $stdErr) use (&$systemErrors, $processIdentifier): v
290253
$streamSelectLoop->run();
291254

292255
if ($reachedSystemErrorsCountLimit) {
293-
$systemErrors[] = new SystemError(sprintf(
256+
$parallelResultCollector->collectSystemError(new SystemError(sprintf(
294257
'Reached system errors count limit of %d, exiting...',
295258
self::SYSTEM_ERROR_LIMIT
296-
));
259+
)));
297260
}
298261

299262
// a worker can end without reporting its files, e.g. when killed by the OS, so missing results must always be reported
300263
if ($processedFilesCount < $scheduledFilesCount) {
301-
$systemErrors[] = new SystemError(sprintf(
264+
$parallelResultCollector->collectSystemError(new SystemError(sprintf(
302265
'Some parallel jobs have not finished, results for %d of %d files are missing',
303266
$scheduledFilesCount - $processedFilesCount,
304267
$scheduledFilesCount
305-
));
306-
}
307-
308-
$mergedUsedSkips = [];
309-
foreach ($usedSkips as $skip => $paths) {
310-
$mergedUsedSkips[$skip] = array_keys($paths);
268+
)));
311269
}
312270

313-
return new ProcessResult($systemErrors, $fileDiffs, $totalChanged, $mergedUsedSkips);
271+
return $parallelResultCollector->createProcessResult();
314272
}
315273
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Rector\Parallel\Application;
6+
7+
use Rector\Parallel\ValueObject\Bridge;
8+
use Rector\ValueObject\Error\SystemError;
9+
use Rector\ValueObject\ProcessResult;
10+
use Rector\ValueObject\Reporting\FileDiff;
11+
12+
/**
13+
* Gathers what the workers report back over the run, so a parallel file processor only has to decide
14+
* who gets which files.
15+
*
16+
* Holds mutable per-run state - create one per process() call, never share it.
17+
*/
18+
final class ParallelResultCollector
19+
{
20+
/**
21+
* @var SystemError[]
22+
*/
23+
private array $systemErrors = [];
24+
25+
/**
26+
* @var FileDiff[]
27+
*/
28+
private array $fileDiffs = [];
29+
30+
/**
31+
* Paths are kept as keys while collecting, to deduplicate them per skip
32+
*
33+
* @var array<string, array<string, true>>
34+
*/
35+
private array $usedSkips = [];
36+
37+
private int $totalChanged = 0;
38+
39+
/**
40+
* @param array{
41+
* total_changed: int,
42+
* system_errors: mixed[],
43+
* file_diffs: array<string, mixed>,
44+
* files_count: int,
45+
* system_errors_count: int,
46+
* used_skips: array<string, string[]>
47+
* } $json
48+
* @return int files covered by this result, to advance the progress bar
49+
*/
50+
public function collectWorkerResult(array $json): int
51+
{
52+
$this->totalChanged += $json[Bridge::TOTAL_CHANGED];
53+
54+
foreach ($json[Bridge::USED_SKIPS] as $skip => $paths) {
55+
$this->usedSkips[$skip] ??= [];
56+
foreach ($paths as $path) {
57+
$this->usedSkips[$skip][$path] = true;
58+
}
59+
}
60+
61+
// decode arrays to objects
62+
foreach ($json[Bridge::SYSTEM_ERRORS] as $jsonError) {
63+
if (is_string($jsonError)) {
64+
$this->systemErrors[] = new SystemError('System error: ' . $jsonError);
65+
continue;
66+
}
67+
68+
$this->systemErrors[] = SystemError::decode($jsonError);
69+
}
70+
71+
foreach ($json[Bridge::FILE_DIFFS] as $jsonFileDiff) {
72+
$this->fileDiffs[] = FileDiff::decode($jsonFileDiff);
73+
}
74+
75+
return $json[Bridge::FILES_COUNT];
76+
}
77+
78+
public function collectSystemError(SystemError $systemError): void
79+
{
80+
$this->systemErrors[] = $systemError;
81+
}
82+
83+
public function createProcessResult(): ProcessResult
84+
{
85+
$mergedUsedSkips = [];
86+
foreach ($this->usedSkips as $skip => $paths) {
87+
$mergedUsedSkips[$skip] = array_keys($paths);
88+
}
89+
90+
return new ProcessResult($this->systemErrors, $this->fileDiffs, $this->totalChanged, $mergedUsedSkips);
91+
}
92+
}

0 commit comments

Comments
 (0)