Skip to content

Commit fcd5b08

Browse files
TomasVotrubaTomasVotruba
committed
[Utils] Add token-based duplicate code detector with CI gate
Adds utils/duplicates, a minimal phpcpd clone (token_get_all + Rabin-Karp), a plain PHP bin script, unit tests, and a Duplicated Code CI job that fails when a copy-pasted block >= 400 tokens is added.
1 parent 222bbf2 commit fcd5b08

11 files changed

Lines changed: 467 additions & 2 deletions

File tree

‎.github/workflows/duplicated_code.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: Duplicated Code
2+
3+
on:
4+
pull_request: null
5+
push:
6+
branches:
7+
- "main"
8+
9+
env:
10+
COMPOSER_ROOT_VERSION: "dev-main"
11+
12+
jobs:
13+
duplicated_code:
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 5
16+
17+
steps:
18+
- uses: actions/checkout@v5
19+
20+
-
21+
uses: shivammathur/setup-php@v2
22+
with:
23+
php-version: 8.4
24+
coverage: none
25+
26+
- run: composer install --no-progress --ansi
27+
28+
# fails when a large copy-pasted block (>= 400 tokens) is added
29+
- run: composer duplicated-code

‎composer.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@
7070
"src"
7171
],
7272
"Rector\\Utils\\": "utils",
73-
"Rector\\Utils\\PHPStan\\": "utils/phpstan/src"
73+
"Rector\\Utils\\PHPStan\\": "utils/phpstan/src",
74+
"Rector\\Utils\\Duplicates\\": "utils/duplicates/src"
7475
},
7576
"files": [
7677
"src/functions/node_helper.php"
@@ -83,6 +84,7 @@
8384
"tests"
8485
],
8586
"Rector\\Utils\\PHPStan\\Tests\\": "utils/phpstan/tests",
87+
"Rector\\Utils\\Duplicates\\Tests\\": "utils/duplicates/tests",
8688
"E2e\\Parallel\\Reflection\\Resolver\\": "e2e/parallel-reflection-resolver/src/",
8789
"Rector\\Scripts\\": "scripts/src"
8890
},
@@ -104,11 +106,12 @@
104106
"@phpstan",
105107
"@test"
106108
],
107-
"test": "vendor/bin/fastunit tests rules-tests utils/phpstan/tests",
109+
"test": "vendor/bin/fastunit tests rules-tests utils/phpstan/tests utils/duplicates/tests",
108110
"check-cs": "vendor/bin/ecs check --ansi",
109111
"fix-cs": "vendor/bin/ecs check --fix --ansi",
110112
"phpstan": "vendor/bin/phpstan analyse --ansi --memory-limit=512M",
111113
"rector": "bin/rector process --ansi",
114+
"duplicated-code": "php utils/duplicates/bin/duplicates.php --min-lines 5 --min-tokens 400 src rules",
112115
"preload": "php build/build-preload.php .",
113116
"release": "vendor/bin/rng --from-commit X --to-commit Y --remote-repository rectorphp/rector-symfony --remote-repository rectorphp/rector-doctrine --remote-repository rectorphp/rector-phpunit"
114117
},

‎utils/duplicates/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Duplicates
2+
3+
Token-based copy-paste detector, a small clone of [phpcpd](https://github.com/phpcpd-next/phpcpd).
4+
5+
It tokenizes PHP with `token_get_all()`, strips whitespace and comments, then finds exact duplicate token spans with a Rabin-Karp rolling hash.
6+
7+
## Usage
8+
9+
```bash
10+
php utils/duplicates/bin/duplicates.php src rules
11+
```
12+
13+
Options:
14+
15+
- `--min-lines` minimum lines of a clone (default `5`)
16+
- `--min-tokens` minimum tokens of a clone (default `70`)
17+
- `--fuzzy` ignore variable names when matching
18+
19+
Exit code is `1` when clones are found, `0` otherwise.

‎utils/duplicates/bin/duplicates.php‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/usr/bin/env php
2+
<?php
3+
4+
declare(strict_types=1);
5+
6+
use Rector\Utils\Duplicates\CloneDetector;
7+
use Symfony\Component\Finder\Finder;
8+
9+
$autoloadFile = __DIR__ . '/../../../vendor/autoload.php';
10+
if (! file_exists($autoloadFile)) {
11+
fwrite(STDERR, 'Composer autoload not found, run "composer install" first.' . PHP_EOL);
12+
exit(1);
13+
}
14+
15+
require_once $autoloadFile;
16+
17+
$minLines = 5;
18+
$minTokens = 70;
19+
$fuzzy = false;
20+
$paths = [];
21+
22+
$args = array_slice($argv, 1);
23+
for ($i = 0; $i < count($args); ++$i) {
24+
$arg = $args[$i];
25+
26+
if ($arg === '--fuzzy') {
27+
$fuzzy = true;
28+
} elseif ($arg === '--min-lines') {
29+
$minLines = (int) ($args[++$i] ?? $minLines);
30+
} elseif ($arg === '--min-tokens') {
31+
$minTokens = (int) ($args[++$i] ?? $minTokens);
32+
} else {
33+
$paths[] = $arg;
34+
}
35+
}
36+
37+
if ($paths === []) {
38+
fwrite(STDERR, 'Usage: php bin/duplicates.php [--min-lines N] [--min-tokens N] [--fuzzy] <path>...' . PHP_EOL);
39+
exit(1);
40+
}
41+
42+
$filePaths = [];
43+
foreach ($paths as $path) {
44+
if (is_file($path)) {
45+
$filePaths[] = $path;
46+
continue;
47+
}
48+
49+
if (! is_dir($path)) {
50+
continue;
51+
}
52+
53+
$finder = new Finder()
54+
->files()
55+
->in($path)
56+
->name('*.php');
57+
58+
foreach ($finder as $fileInfo) {
59+
$realPath = $fileInfo->getRealPath();
60+
if ($realPath === false) {
61+
continue;
62+
}
63+
64+
$filePaths[] = $realPath;
65+
}
66+
}
67+
68+
if ($filePaths === []) {
69+
echo 'No PHP files found in the given paths.' . PHP_EOL;
70+
exit(0);
71+
}
72+
73+
$cloneDetector = new CloneDetector($minLines, $minTokens, $fuzzy);
74+
$clones = $cloneDetector->detect($filePaths);
75+
76+
if ($clones === []) {
77+
echo sprintf('[OK] No duplicates found in %d files', count($filePaths)) . PHP_EOL;
78+
exit(0);
79+
}
80+
81+
$duplicatedLines = 0;
82+
foreach ($clones as $clone) {
83+
echo sprintf(
84+
' - %s:%d-%d (%d lines, %d tokens)',
85+
$clone->firstFile->filePath,
86+
$clone->firstFile->startLine,
87+
$clone->firstFile->endLine,
88+
$clone->lines,
89+
$clone->tokens
90+
) . PHP_EOL;
91+
echo sprintf(
92+
' %s:%d-%d',
93+
$clone->secondFile->filePath,
94+
$clone->secondFile->startLine,
95+
$clone->secondFile->endLine
96+
) . PHP_EOL;
97+
echo PHP_EOL;
98+
99+
$duplicatedLines += $clone->lines;
100+
}
101+
102+
fwrite(STDERR, sprintf(
103+
'[ERROR] Found %d clones with %d duplicated lines in %d scanned files',
104+
count($clones),
105+
$duplicatedLines,
106+
count($filePaths)
107+
) . PHP_EOL);
108+
109+
exit(1);

‎utils/duplicates/src/CloneDetector.php‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Rector\Utils\Duplicates;
6+
7+
use Rector\Utils\Duplicates\ValueObject\CodeClone;
8+
use Rector\Utils\Duplicates\ValueObject\CodeCloneFile;
9+
10+
/**
11+
* Token-based copy-paste detector using a Rabin-Karp rolling hash over the
12+
* normalized token stream, mirroring the classic phpcpd behaviour.
13+
* @see \Rector\Utils\Duplicates\Tests\CloneDetectorTest
14+
*/
15+
final class CloneDetector
16+
{
17+
/**
18+
* Tokens that carry no structural meaning for clone detection.
19+
*
20+
* @var array<int, true>
21+
*/
22+
private const array IGNORED_TOKENS = [
23+
T_INLINE_HTML => true,
24+
T_COMMENT => true,
25+
T_DOC_COMMENT => true,
26+
T_OPEN_TAG => true,
27+
T_OPEN_TAG_WITH_ECHO => true,
28+
T_CLOSE_TAG => true,
29+
T_WHITESPACE => true,
30+
];
31+
32+
/**
33+
* Bytes contributed by each kept token to the signature: 1 type byte + 4 CRC32 bytes.
34+
*/
35+
private const int BYTES_PER_TOKEN = 5;
36+
37+
/**
38+
* First seen location per window hash: hash => [filePath, tokenIndex].
39+
*
40+
* @var array<string, array{string, int}>
41+
*/
42+
private array $hashes = [];
43+
44+
/**
45+
* Per-file token line numbers, kept for span line lookup.
46+
*
47+
* @var array<string, int[]>
48+
*/
49+
private array $fileLines = [];
50+
51+
/**
52+
* @var CodeClone[]
53+
*/
54+
private array $clones = [];
55+
56+
public function __construct(
57+
private readonly int $minLines,
58+
private readonly int $minTokens,
59+
private readonly bool $fuzzy
60+
) {
61+
}
62+
63+
/**
64+
* @param string[] $filePaths
65+
* @return CodeClone[]
66+
*/
67+
public function detect(array $filePaths): array
68+
{
69+
foreach ($filePaths as $filePath) {
70+
$this->processFile($filePath);
71+
}
72+
73+
return $this->clones;
74+
}
75+
76+
private function processFile(string $filePath): void
77+
{
78+
$source = file_get_contents($filePath);
79+
if ($source === false) {
80+
return;
81+
}
82+
83+
[$signature, $lines] = $this->tokenize($source);
84+
$this->fileLines[$filePath] = $lines;
85+
86+
$tokenCount = count($lines);
87+
if ($tokenCount < $this->minTokens) {
88+
return;
89+
}
90+
91+
$lastWindow = $tokenCount - $this->minTokens;
92+
$windowBytes = $this->minTokens * self::BYTES_PER_TOKEN;
93+
94+
$found = false;
95+
$firstToken = 0;
96+
$originFile = '';
97+
$originToken = 0;
98+
99+
for ($i = 0; $i <= $lastWindow; ++$i) {
100+
$hash = substr(md5(substr($signature, $i * self::BYTES_PER_TOKEN, $windowBytes), true), 0, 8);
101+
102+
if (isset($this->hashes[$hash])) {
103+
if (! $found) {
104+
$found = true;
105+
$firstToken = $i;
106+
[$originFile, $originToken] = $this->hashes[$hash];
107+
}
108+
109+
continue;
110+
}
111+
112+
if ($found) {
113+
$this->recordClone($originFile, $originToken, $filePath, $firstToken, $i);
114+
$found = false;
115+
}
116+
117+
$this->hashes[$hash] = [$filePath, $i];
118+
}
119+
120+
if ($found) {
121+
$this->recordClone($originFile, $originToken, $filePath, $firstToken, $lastWindow + 1);
122+
}
123+
}
124+
125+
private function recordClone(
126+
string $originFile,
127+
int $originToken,
128+
string $currentFile,
129+
int $currentToken,
130+
int $mismatchWindow
131+
): void {
132+
$windowCount = $mismatchWindow - $currentToken;
133+
$tokenSpan = $windowCount + $this->minTokens - 1;
134+
135+
$originLines = $this->fileLines[$originFile];
136+
$currentLines = $this->fileLines[$currentFile];
137+
138+
$originStartLine = $originLines[$originToken];
139+
$originEndLine = $originLines[min($originToken + $tokenSpan - 1, count($originLines) - 1)];
140+
141+
$currentStartLine = $currentLines[$currentToken];
142+
$currentEndLine = $currentLines[min($currentToken + $tokenSpan - 1, count($currentLines) - 1)];
143+
144+
$numberOfLines = $originEndLine - $originStartLine + 1;
145+
if ($numberOfLines < $this->minLines) {
146+
return;
147+
}
148+
149+
$this->clones[] = new CodeClone(
150+
new CodeCloneFile($originFile, $originStartLine, $originEndLine),
151+
new CodeCloneFile($currentFile, $currentStartLine, $currentEndLine),
152+
$numberOfLines,
153+
$tokenSpan
154+
);
155+
}
156+
157+
/**
158+
* Builds the token signature and the parallel line map for a source string.
159+
*
160+
* @return array{string, int[]}
161+
*/
162+
private function tokenize(string $source): array
163+
{
164+
$signature = '';
165+
$lines = [];
166+
$currentLine = 1;
167+
168+
foreach (token_get_all($source) as $token) {
169+
if (is_array($token)) {
170+
$tokenId = $token[0];
171+
$tokenText = $token[1];
172+
$currentLine = $token[2];
173+
174+
if (isset(self::IGNORED_TOKENS[$tokenId])) {
175+
$currentLine += substr_count($tokenText, "\n");
176+
continue;
177+
}
178+
179+
if ($this->fuzzy && $tokenId === T_VARIABLE) {
180+
$tokenText = '$';
181+
}
182+
183+
$signature .= chr($tokenId & 255) . pack('N*', crc32($tokenText));
184+
$lines[] = $currentLine;
185+
$currentLine += substr_count($tokenText, "\n");
186+
187+
continue;
188+
}
189+
190+
$signature .= chr(0) . pack('N*', crc32($token));
191+
$lines[] = $currentLine;
192+
}
193+
194+
return [$signature, $lines];
195+
}
196+
}

0 commit comments

Comments
 (0)