Skip to content

Commit ead8022

Browse files
Map PSI tokens to original source positions
1 parent 4076eff commit ead8022

2 files changed

Lines changed: 382 additions & 0 deletions

File tree

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
/*
2+
* SonarQube Java
3+
* Copyright (C) SonarSource Sàrl
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* You can redistribute and/or modify this program under the terms of
7+
* the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.java.model;
18+
19+
import com.intellij.lang.java.lexer.JavaLexer;
20+
import com.intellij.pom.java.LanguageLevel;
21+
import com.intellij.psi.TokenType;
22+
import com.intellij.psi.impl.source.tree.java.PsiLiteralExpressionImpl;
23+
import java.util.ArrayList;
24+
import java.util.HashMap;
25+
import java.util.List;
26+
import java.util.Map;
27+
import org.sonar.plugins.java.api.tree.SyntaxTrivia;
28+
29+
/** Indexes physical UTF-16 source slices independently of PSI's decoded names and composite operator nodes. */
30+
final class PsiTokenIndex {
31+
record Lexeme(int start, int end, String kind, InternalSyntaxToken token) {
32+
}
33+
34+
private final String source;
35+
private final LineColumnConverter positions;
36+
private final List<Lexeme> tokens = new ArrayList<>();
37+
private final Map<Long, InternalSyntaxToken> slices = new HashMap<>();
38+
39+
PsiTokenIndex(String source) {
40+
this.source = source;
41+
positions = new LineColumnConverter(source);
42+
var lexer = new JavaLexer(LanguageLevel.JDK_26_PREVIEW);
43+
var input = unicodeInput(source);
44+
lexer.start(input.text());
45+
List<SyntaxTrivia> trivia = new ArrayList<>();
46+
int commentStart = 0;
47+
int commentEnd = 0;
48+
while (lexer.getTokenType() != null) {
49+
int start = input.offsets()[lexer.getTokenStart()];
50+
int end = input.offsets()[lexer.getTokenEnd()];
51+
String kind = lexer.getTokenType().toString();
52+
int restart = -1;
53+
if (kind.equals("TEXT_BLOCK_LITERAL") && !opensTextBlock(input.text(), lexer.getTokenStart())) {
54+
restart = lexer.getTokenStart() + 2;
55+
end = input.offsets()[restart];
56+
kind = "STRING_LITERAL";
57+
}
58+
String text = source.substring(start, end);
59+
if (kind.contains("COMMENT")) {
60+
int logicalEnd = lexer.getTokenEnd();
61+
if (kind.equals("END_OF_LINE_COMMENT") && logicalEnd < input.text().length()
62+
&& (input.text().charAt(logicalEnd) == '\n' || input.text().charAt(logicalEnd) == '\r')
63+
&& input.offsets()[logicalEnd + 1] - end > 1) {
64+
end = input.offsets()[logicalEnd + 1];
65+
text = source.substring(start, end);
66+
}
67+
var commentKind = text.startsWith("///") ? SyntaxTrivia.CommentKind.MARKDOWN
68+
: text.startsWith("/**") && text.length() > 4 ? SyntaxTrivia.CommentKind.JAVADOC
69+
: text.startsWith("//") ? SyntaxTrivia.CommentKind.LINE : SyntaxTrivia.CommentKind.BLOCK;
70+
if (commentKind == SyntaxTrivia.CommentKind.MARKDOWN && !trivia.isEmpty()
71+
&& trivia.getLast().commentKind() == SyntaxTrivia.CommentKind.MARKDOWN
72+
&& source.substring(commentEnd, start).matches("(?:\\r\\n?|\\n)[ \t\f]*")) {
73+
trivia.removeLast();
74+
start = commentStart;
75+
text = source.substring(start, end);
76+
}
77+
var pos = positions.toPos(start);
78+
trivia.add(new InternalSyntaxTrivia(commentKind, text, pos.line(), pos.columnOffset()));
79+
commentStart = start;
80+
commentEnd = end;
81+
} else if (lexer.getTokenType() != TokenType.WHITE_SPACE) {
82+
validateLiteral(kind, text, start);
83+
if (lexer.getTokenType() == TokenType.BAD_CHARACTER) {
84+
throw new IllegalArgumentException("Invalid Java token at UTF-16 offset " + start);
85+
}
86+
var pos = positions.toPos(start);
87+
tokens.add(new Lexeme(start, end, kind, new InternalSyntaxToken(pos.line(), pos.columnOffset(), text, List.copyOf(trivia), false)));
88+
trivia.clear();
89+
}
90+
if (restart < 0) {
91+
lexer.advance();
92+
} else {
93+
lexer.start(input.text(), restart, input.text().length(), 0);
94+
}
95+
}
96+
mergeShifts();
97+
var pos = positions.toPos(source.length());
98+
tokens.add(new Lexeme(source.length(), source.length(), "EOF",
99+
new InternalSyntaxToken(pos.line(), pos.columnOffset(), "", List.copyOf(trivia), true)));
100+
}
101+
102+
private static boolean opensTextBlock(String text, int start) {
103+
int i = start + 3;
104+
while (i < text.length() && (text.charAt(i) == ' ' || text.charAt(i) == '\t' || text.charAt(i) == '\f')) {
105+
i++;
106+
}
107+
return i < text.length() && (text.charAt(i) == '\r' || text.charAt(i) == '\n');
108+
}
109+
110+
private static void validateLiteral(String kind, String raw, int offset) {
111+
if (!kind.equals("STRING_LITERAL") && !kind.equals("CHARACTER_LITERAL") && !kind.equals("TEXT_BLOCK_LITERAL")) {
112+
return;
113+
}
114+
String text = decoded(raw);
115+
boolean valid;
116+
if (kind.equals("TEXT_BLOCK_LITERAL")) {
117+
valid = text.length() >= 6 && text.endsWith("\"\"\"");
118+
} else {
119+
char quote = kind.equals("STRING_LITERAL") ? '"' : '\'';
120+
var value = new StringBuilder();
121+
valid = text.length() >= 2 && text.charAt(text.length() - 1) == quote
122+
&& text.indexOf('\n') < 0 && text.indexOf('\r') < 0
123+
&& PsiLiteralExpressionImpl.parseStringCharacters(text.substring(1, text.length() - 1), value, null)
124+
&& (quote == '"' || value.length() == 1);
125+
}
126+
if (!valid) {
127+
throw new IllegalArgumentException("Invalid Java literal at UTF-16 offset " + offset);
128+
}
129+
}
130+
131+
record UnicodeInput(String text, int[] offsets) {
132+
}
133+
134+
static UnicodeInput unicodeInput(String raw) {
135+
var decoded = new StringBuilder(raw.length());
136+
int[] offsets = new int[raw.length() + 1];
137+
int slashes = 0;
138+
boolean escaped = false;
139+
for (int i = 0; i < raw.length();) {
140+
offsets[decoded.length()] = i;
141+
char c = raw.charAt(i++);
142+
boolean unicode = false;
143+
if (c == '\\' && (escaped || slashes % 2 == 0) && i < raw.length() && raw.charAt(i) == 'u') {
144+
while (i < raw.length() && raw.charAt(i) == 'u') {
145+
i++;
146+
}
147+
if (i + 4 > raw.length()) {
148+
throw new IllegalArgumentException("Incomplete Unicode escape at UTF-16 offset " + i);
149+
}
150+
int value = 0;
151+
for (int j = 0; j < 4; j++) {
152+
int digit = Character.digit(raw.charAt(i++), 16);
153+
if (digit < 0) {
154+
throw new IllegalArgumentException("Invalid Unicode escape at UTF-16 offset " + (i - 1));
155+
}
156+
value = value * 16 + digit;
157+
}
158+
c = (char) value;
159+
unicode = true;
160+
}
161+
decoded.append(c);
162+
slashes = c == '\\' ? slashes + 1 : 0;
163+
escaped = unicode;
164+
}
165+
offsets[decoded.length()] = raw.length();
166+
return new UnicodeInput(decoded.toString(), offsets);
167+
}
168+
169+
static String decoded(String text) {
170+
return unicodeInput(text).text();
171+
}
172+
173+
private void mergeShifts() {
174+
for (int i = 0; i < tokens.size(); i++) {
175+
var first = tokens.get(i);
176+
if (!decoded(first.token().text()).equals(">")) {
177+
continue;
178+
}
179+
int j = i + 1;
180+
int end = first.end();
181+
String operator = ">";
182+
while (j < tokens.size() && tokens.get(j).start() == end && operator.length() < 3 && decoded(tokens.get(j).token().text()).equals(">")) {
183+
end = tokens.get(j++).end();
184+
operator += ">";
185+
}
186+
if (j < tokens.size() && tokens.get(j).start() == end && decoded(tokens.get(j).token().text()).equals("=")) {
187+
end = tokens.get(j++).end();
188+
operator += "=";
189+
}
190+
if (j > i + 1) {
191+
var pos = positions.toPos(first.start());
192+
tokens.set(i, new Lexeme(first.start(), end, operator,
193+
new InternalSyntaxToken(pos.line(), pos.columnOffset(), source.substring(first.start(), end), first.token().trivias(), false)));
194+
tokens.subList(i + 1, j).clear();
195+
}
196+
}
197+
}
198+
199+
List<Lexeme> lexemes() {
200+
return java.util.Collections.unmodifiableList(tokens);
201+
}
202+
203+
InternalSyntaxToken eof() {
204+
return tokens.getLast().token();
205+
}
206+
207+
int index(int offset) {
208+
int low = 0;
209+
int high = tokens.size() - 1;
210+
while (low < high) {
211+
int middle = (low + high) >>> 1;
212+
if (tokens.get(middle).start() < offset) {
213+
low = middle + 1;
214+
} else {
215+
high = middle;
216+
}
217+
}
218+
return low;
219+
}
220+
221+
InternalSyntaxToken at(int offset) {
222+
return tokens.get(index(offset)).token();
223+
}
224+
225+
InternalSyntaxToken find(int start, int end, String text, boolean last) {
226+
InternalSyntaxToken result = null;
227+
for (int i = index(start); i < tokens.size() && tokens.get(i).start() < end; i++) {
228+
var token = tokens.get(i).token();
229+
if (decoded(token.text()).equals(text)) {
230+
result = token;
231+
if (!last) {
232+
break;
233+
}
234+
}
235+
}
236+
if (result == null) {
237+
throw new IllegalArgumentException("Missing token '" + text + "' in [" + start + ", " + end + ")");
238+
}
239+
return result;
240+
}
241+
242+
InternalSyntaxToken slice(int start, int end) {
243+
int index = index(start);
244+
if (index > 0 && tokens.get(index - 1).end() > start) {
245+
index--;
246+
}
247+
var lexeme = tokens.get(index);
248+
if (lexeme.start() == start && lexeme.end() == end) {
249+
return lexeme.token();
250+
}
251+
long key = ((long) start << 32) | end;
252+
return slices.computeIfAbsent(key, unused -> {
253+
var pos = positions.toPos(start);
254+
List<SyntaxTrivia> trivia = lexeme.start() == start ? lexeme.token().trivias() : List.of();
255+
return new InternalSyntaxToken(pos.line(), pos.columnOffset(), source.substring(start, end), trivia, false);
256+
});
257+
}
258+
}
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* SonarQube Java
3+
* Copyright (C) SonarSource Sàrl
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* You can redistribute and/or modify this program under the terms of
7+
* the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.java.model;
18+
19+
import java.nio.charset.StandardCharsets;
20+
import java.nio.file.Path;
21+
import java.util.List;
22+
import java.util.stream.Stream;
23+
import org.junit.jupiter.api.Test;
24+
import org.junit.jupiter.params.ParameterizedTest;
25+
import org.junit.jupiter.params.provider.MethodSource;
26+
import org.junit.jupiter.params.provider.ValueSource;
27+
import org.sonar.java.psi.JavaPsiConfiguration;
28+
import org.sonar.java.psi.JavaPsiEnvironment;
29+
import org.sonar.plugins.java.api.tree.SyntaxTrivia;
30+
31+
import static org.assertj.core.api.Assertions.assertThat;
32+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
33+
34+
class PsiTokenIndexTest {
35+
@ParameterizedTest
36+
@MethodSource("sources")
37+
void physical_tokens_preserve_original_utf16_slices(String source) {
38+
var index = index(source);
39+
int previousEnd = 0;
40+
for (var lexeme : index.lexemes()) {
41+
assertThat(lexeme.start()).isGreaterThanOrEqualTo(previousEnd);
42+
assertThat(lexeme.token().text()).isEqualTo(source.substring(lexeme.start(), lexeme.end()));
43+
String prefix = source.substring(0, lexeme.start());
44+
assertThat(lexeme.token().line()).isEqualTo(prefix.split("\\r\\n?|\\n", -1).length);
45+
int lineStart = Math.max(prefix.lastIndexOf('\n'), prefix.lastIndexOf('\r')) + 1;
46+
assertThat(lexeme.token().range().start().columnOffset()).isEqualTo(lexeme.start() - lineStart);
47+
previousEnd = lexeme.end();
48+
}
49+
assertThat(index.eof().isEOF()).isTrue();
50+
assertThat(previousEnd).isEqualTo(source.length());
51+
}
52+
53+
@Test
54+
void operators_keep_physical_spelling_and_combined_shifts() {
55+
assertThat(index("a>>b; a>>>b; a>>=b; a>>>=b;").lexemes().stream().map(lexeme -> lexeme.token().text()))
56+
.containsExactly("a", ">>", "b", ";", "a", ">>>", "b", ";", "a", ">>=", "b", ";", "a", ">>>=", "b", ";", "");
57+
}
58+
59+
@Test
60+
void unicode_identifiers_keep_their_raw_ranges() {
61+
String source = "cl" + "\\" + "u0061ss A {}";
62+
var token = index(source).lexemes().getFirst().token();
63+
assertThat(token.text()).isEqualTo("cl" + "\\" + "u0061ss");
64+
assertThat(token.range().start()).isEqualTo(org.sonar.plugins.java.api.location.Position.at(1, 1));
65+
assertThat(token.range().end()).isEqualTo(org.sonar.plugins.java.api.location.Position.at(1, 11));
66+
}
67+
68+
@Test
69+
void markdown_groups_keep_indentation_and_eof_comments() {
70+
var index = index("/// first\n\t/// second\nclass A {} // eof");
71+
var trivia = index.lexemes().getFirst().token().trivias();
72+
assertThat(trivia).hasSize(1);
73+
assertThat(trivia.getFirst().commentKind()).isEqualTo(SyntaxTrivia.CommentKind.MARKDOWN);
74+
assertThat(trivia.getFirst().comment()).isEqualTo("/// first\n\t/// second");
75+
assertThat(index.eof().trivias()).extracting(SyntaxTrivia::comment).containsExactly("// eof");
76+
}
77+
78+
static Stream<String> sources() {
79+
String escape = "\\" + "u";
80+
return Stream.of(
81+
"", " \t\r\n", "\"\"\"text\"\"\"", "/// first\n\n/// second", "/**/ /** doc */ /*block*/ // line\r\nclass A {} // eof",
82+
"/// first\n/// second\nclass A {}", "/// one\r\n\t/// two\r/// three\n",
83+
"class A {\r\n\tint 𐐀;\rString x=\"😀\";\n}",
84+
"class A { String x=\"\"\"\r\n abc\\\r\n def\r\n \"\"\"; }",
85+
"int x=0xCA_FE; long y=0b11L; double a=.25e+2, b=0x1.fp-2; float f=1f; char c='\\n';",
86+
"a>>b; a>>>b; a>>=b; a>>>=b; a<<b; a<=b; a!=b; a++ + --b;",
87+
"cl" + escape + "0061ss A { int " + escape + "0078 = 1; }",
88+
"// comment " + escape + "000a class A {}",
89+
"// eof", "/** doc */", "/* one */ /* two */"
90+
);
91+
}
92+
93+
@ParameterizedTest
94+
@ValueSource(strings = {"#", "`", "/* unterminated", "\"unterminated", "'x", "\"\\q\"", "'ab'", "\"\"\"\nunterminated"})
95+
void rejects_malformed_lexical_input(String source) {
96+
assertThatThrownBy(() -> index(source)).isInstanceOf(IllegalArgumentException.class);
97+
}
98+
99+
@Test
100+
void contextual_closers_share_slices_without_duplicating_trivia() {
101+
var index = index("/* c */ >>>");
102+
var first = index.slice(8, 9);
103+
assertThat(first.text()).isEqualTo(">");
104+
assertThat(first.trivias()).hasSize(1);
105+
assertThat(index.slice(8, 9)).isSameAs(first);
106+
assertThat(index.slice(9, 10).trivias()).isEmpty();
107+
assertThat(index.slice(10, 11).trivias()).isEmpty();
108+
assertThat(index.eof().isEOF()).isTrue();
109+
}
110+
111+
@Test
112+
void escaped_comment_delimiters_preserve_the_existing_rejection() {
113+
String source = "\\" + "u002f\\" + "u002a comment */ class A {}";
114+
assertThatThrownBy(() -> index(source)).isInstanceOf(IllegalArgumentException.class).hasMessageContaining("Invalid comment kind");
115+
}
116+
117+
private static PsiTokenIndex index(String source) {
118+
var config = new JavaPsiConfiguration(null, List.of(), 26, true, true);
119+
try (var environment = JavaPsiEnvironment.open(config, List.of(new JavaPsiEnvironment.SourceSnapshot(Path.of("File.java"), source, StandardCharsets.UTF_8)))) {
120+
return environment.read((project, files) -> new PsiTokenIndex(source));
121+
}
122+
}
123+
124+
}

0 commit comments

Comments
 (0)