|
| 1 | +/* |
| 2 | + * SonarQube Java |
| 3 | + * Copyright (C) SonarSource Sàrl |
| 4 | + * mailto:info AT sonarsource DOT com |
| 5 | + * |
| 6 | + * You can redistribute and/or modify this program under the terms of |
| 7 | + * the Sonar Source-Available License Version 1, as published by SonarSource Sàrl. |
| 8 | + * |
| 9 | + * This program is distributed in the hope that it will be useful, |
| 10 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| 12 | + * See the Sonar Source-Available License for more details. |
| 13 | + * |
| 14 | + * You should have received a copy of the Sonar Source-Available License |
| 15 | + * along with this program; if not, see https://sonarsource.com/license/ssal/ |
| 16 | + */ |
| 17 | +package org.sonar.java.model; |
| 18 | + |
| 19 | +import com.intellij.lang.java.lexer.JavaLexer; |
| 20 | +import com.intellij.pom.java.LanguageLevel; |
| 21 | +import com.intellij.psi.TokenType; |
| 22 | +import com.intellij.psi.impl.source.tree.java.PsiLiteralExpressionImpl; |
| 23 | +import java.util.ArrayList; |
| 24 | +import java.util.HashMap; |
| 25 | +import java.util.List; |
| 26 | +import java.util.Map; |
| 27 | +import org.sonar.plugins.java.api.tree.SyntaxTrivia; |
| 28 | + |
| 29 | +/** Indexes physical UTF-16 source slices independently of PSI's decoded names and composite operator nodes. */ |
| 30 | +final class PsiTokenIndex { |
| 31 | + record Lexeme(int start, int end, String kind, InternalSyntaxToken token) { |
| 32 | + } |
| 33 | + |
| 34 | + private final String source; |
| 35 | + private final LineColumnConverter positions; |
| 36 | + private final List<Lexeme> tokens = new ArrayList<>(); |
| 37 | + private final Map<Long, InternalSyntaxToken> slices = new HashMap<>(); |
| 38 | + |
| 39 | + PsiTokenIndex(String source) { |
| 40 | + this.source = source; |
| 41 | + positions = new LineColumnConverter(source); |
| 42 | + var lexer = new JavaLexer(LanguageLevel.JDK_26_PREVIEW); |
| 43 | + var input = unicodeInput(source); |
| 44 | + lexer.start(input.text()); |
| 45 | + List<SyntaxTrivia> trivia = new ArrayList<>(); |
| 46 | + int commentStart = 0; |
| 47 | + int commentEnd = 0; |
| 48 | + while (lexer.getTokenType() != null) { |
| 49 | + int start = input.offsets()[lexer.getTokenStart()]; |
| 50 | + int end = input.offsets()[lexer.getTokenEnd()]; |
| 51 | + String kind = lexer.getTokenType().toString(); |
| 52 | + int restart = -1; |
| 53 | + if (kind.equals("TEXT_BLOCK_LITERAL") && !opensTextBlock(input.text(), lexer.getTokenStart())) { |
| 54 | + restart = lexer.getTokenStart() + 2; |
| 55 | + end = input.offsets()[restart]; |
| 56 | + kind = "STRING_LITERAL"; |
| 57 | + } |
| 58 | + String text = source.substring(start, end); |
| 59 | + if (kind.contains("COMMENT")) { |
| 60 | + int logicalEnd = lexer.getTokenEnd(); |
| 61 | + if (kind.equals("END_OF_LINE_COMMENT") && logicalEnd < input.text().length() |
| 62 | + && (input.text().charAt(logicalEnd) == '\n' || input.text().charAt(logicalEnd) == '\r') |
| 63 | + && input.offsets()[logicalEnd + 1] - end > 1) { |
| 64 | + end = input.offsets()[logicalEnd + 1]; |
| 65 | + text = source.substring(start, end); |
| 66 | + } |
| 67 | + var commentKind = text.startsWith("///") ? SyntaxTrivia.CommentKind.MARKDOWN |
| 68 | + : text.startsWith("/**") && text.length() > 4 ? SyntaxTrivia.CommentKind.JAVADOC |
| 69 | + : text.startsWith("//") ? SyntaxTrivia.CommentKind.LINE : SyntaxTrivia.CommentKind.BLOCK; |
| 70 | + if (commentKind == SyntaxTrivia.CommentKind.MARKDOWN && !trivia.isEmpty() |
| 71 | + && trivia.getLast().commentKind() == SyntaxTrivia.CommentKind.MARKDOWN |
| 72 | + && source.substring(commentEnd, start).matches("(?:\\r\\n?|\\n)[ \t\f]*")) { |
| 73 | + trivia.removeLast(); |
| 74 | + start = commentStart; |
| 75 | + text = source.substring(start, end); |
| 76 | + } |
| 77 | + var pos = positions.toPos(start); |
| 78 | + trivia.add(new InternalSyntaxTrivia(commentKind, text, pos.line(), pos.columnOffset())); |
| 79 | + commentStart = start; |
| 80 | + commentEnd = end; |
| 81 | + } else if (lexer.getTokenType() != TokenType.WHITE_SPACE) { |
| 82 | + validateLiteral(kind, text, start); |
| 83 | + if (lexer.getTokenType() == TokenType.BAD_CHARACTER) { |
| 84 | + throw new IllegalArgumentException("Invalid Java token at UTF-16 offset " + start); |
| 85 | + } |
| 86 | + var pos = positions.toPos(start); |
| 87 | + tokens.add(new Lexeme(start, end, kind, new InternalSyntaxToken(pos.line(), pos.columnOffset(), text, List.copyOf(trivia), false))); |
| 88 | + trivia.clear(); |
| 89 | + } |
| 90 | + if (restart < 0) { |
| 91 | + lexer.advance(); |
| 92 | + } else { |
| 93 | + lexer.start(input.text(), restart, input.text().length(), 0); |
| 94 | + } |
| 95 | + } |
| 96 | + mergeShifts(); |
| 97 | + var pos = positions.toPos(source.length()); |
| 98 | + tokens.add(new Lexeme(source.length(), source.length(), "EOF", |
| 99 | + new InternalSyntaxToken(pos.line(), pos.columnOffset(), "", List.copyOf(trivia), true))); |
| 100 | + } |
| 101 | + |
| 102 | + private static boolean opensTextBlock(String text, int start) { |
| 103 | + int i = start + 3; |
| 104 | + while (i < text.length() && (text.charAt(i) == ' ' || text.charAt(i) == '\t' || text.charAt(i) == '\f')) { |
| 105 | + i++; |
| 106 | + } |
| 107 | + return i < text.length() && (text.charAt(i) == '\r' || text.charAt(i) == '\n'); |
| 108 | + } |
| 109 | + |
| 110 | + private static void validateLiteral(String kind, String raw, int offset) { |
| 111 | + if (!kind.equals("STRING_LITERAL") && !kind.equals("CHARACTER_LITERAL") && !kind.equals("TEXT_BLOCK_LITERAL")) { |
| 112 | + return; |
| 113 | + } |
| 114 | + String text = decoded(raw); |
| 115 | + boolean valid; |
| 116 | + if (kind.equals("TEXT_BLOCK_LITERAL")) { |
| 117 | + valid = text.length() >= 6 && text.endsWith("\"\"\""); |
| 118 | + } else { |
| 119 | + char quote = kind.equals("STRING_LITERAL") ? '"' : '\''; |
| 120 | + var value = new StringBuilder(); |
| 121 | + valid = text.length() >= 2 && text.charAt(text.length() - 1) == quote |
| 122 | + && text.indexOf('\n') < 0 && text.indexOf('\r') < 0 |
| 123 | + && PsiLiteralExpressionImpl.parseStringCharacters(text.substring(1, text.length() - 1), value, null) |
| 124 | + && (quote == '"' || value.length() == 1); |
| 125 | + } |
| 126 | + if (!valid) { |
| 127 | + throw new IllegalArgumentException("Invalid Java literal at UTF-16 offset " + offset); |
| 128 | + } |
| 129 | + } |
| 130 | + |
| 131 | + record UnicodeInput(String text, int[] offsets) { |
| 132 | + } |
| 133 | + |
| 134 | + static UnicodeInput unicodeInput(String raw) { |
| 135 | + var decoded = new StringBuilder(raw.length()); |
| 136 | + int[] offsets = new int[raw.length() + 1]; |
| 137 | + int slashes = 0; |
| 138 | + boolean escaped = false; |
| 139 | + for (int i = 0; i < raw.length();) { |
| 140 | + offsets[decoded.length()] = i; |
| 141 | + char c = raw.charAt(i++); |
| 142 | + boolean unicode = false; |
| 143 | + if (c == '\\' && (escaped || slashes % 2 == 0) && i < raw.length() && raw.charAt(i) == 'u') { |
| 144 | + while (i < raw.length() && raw.charAt(i) == 'u') { |
| 145 | + i++; |
| 146 | + } |
| 147 | + if (i + 4 > raw.length()) { |
| 148 | + throw new IllegalArgumentException("Incomplete Unicode escape at UTF-16 offset " + i); |
| 149 | + } |
| 150 | + int value = 0; |
| 151 | + for (int j = 0; j < 4; j++) { |
| 152 | + int digit = Character.digit(raw.charAt(i++), 16); |
| 153 | + if (digit < 0) { |
| 154 | + throw new IllegalArgumentException("Invalid Unicode escape at UTF-16 offset " + (i - 1)); |
| 155 | + } |
| 156 | + value = value * 16 + digit; |
| 157 | + } |
| 158 | + c = (char) value; |
| 159 | + unicode = true; |
| 160 | + } |
| 161 | + decoded.append(c); |
| 162 | + slashes = c == '\\' ? slashes + 1 : 0; |
| 163 | + escaped = unicode; |
| 164 | + } |
| 165 | + offsets[decoded.length()] = raw.length(); |
| 166 | + return new UnicodeInput(decoded.toString(), offsets); |
| 167 | + } |
| 168 | + |
| 169 | + static String decoded(String text) { |
| 170 | + return unicodeInput(text).text(); |
| 171 | + } |
| 172 | + |
| 173 | + private void mergeShifts() { |
| 174 | + for (int i = 0; i < tokens.size(); i++) { |
| 175 | + var first = tokens.get(i); |
| 176 | + if (!decoded(first.token().text()).equals(">")) { |
| 177 | + continue; |
| 178 | + } |
| 179 | + int j = i + 1; |
| 180 | + int end = first.end(); |
| 181 | + String operator = ">"; |
| 182 | + while (j < tokens.size() && tokens.get(j).start() == end && operator.length() < 3 && decoded(tokens.get(j).token().text()).equals(">")) { |
| 183 | + end = tokens.get(j++).end(); |
| 184 | + operator += ">"; |
| 185 | + } |
| 186 | + if (j < tokens.size() && tokens.get(j).start() == end && decoded(tokens.get(j).token().text()).equals("=")) { |
| 187 | + end = tokens.get(j++).end(); |
| 188 | + operator += "="; |
| 189 | + } |
| 190 | + if (j > i + 1) { |
| 191 | + var pos = positions.toPos(first.start()); |
| 192 | + tokens.set(i, new Lexeme(first.start(), end, operator, |
| 193 | + new InternalSyntaxToken(pos.line(), pos.columnOffset(), source.substring(first.start(), end), first.token().trivias(), false))); |
| 194 | + tokens.subList(i + 1, j).clear(); |
| 195 | + } |
| 196 | + } |
| 197 | + } |
| 198 | + |
| 199 | + List<Lexeme> lexemes() { |
| 200 | + return java.util.Collections.unmodifiableList(tokens); |
| 201 | + } |
| 202 | + |
| 203 | + InternalSyntaxToken eof() { |
| 204 | + return tokens.getLast().token(); |
| 205 | + } |
| 206 | + |
| 207 | + int index(int offset) { |
| 208 | + int low = 0; |
| 209 | + int high = tokens.size() - 1; |
| 210 | + while (low < high) { |
| 211 | + int middle = (low + high) >>> 1; |
| 212 | + if (tokens.get(middle).start() < offset) { |
| 213 | + low = middle + 1; |
| 214 | + } else { |
| 215 | + high = middle; |
| 216 | + } |
| 217 | + } |
| 218 | + return low; |
| 219 | + } |
| 220 | + |
| 221 | + InternalSyntaxToken at(int offset) { |
| 222 | + return tokens.get(index(offset)).token(); |
| 223 | + } |
| 224 | + |
| 225 | + InternalSyntaxToken find(int start, int end, String text, boolean last) { |
| 226 | + InternalSyntaxToken result = null; |
| 227 | + for (int i = index(start); i < tokens.size() && tokens.get(i).start() < end; i++) { |
| 228 | + var token = tokens.get(i).token(); |
| 229 | + if (decoded(token.text()).equals(text)) { |
| 230 | + result = token; |
| 231 | + if (!last) { |
| 232 | + break; |
| 233 | + } |
| 234 | + } |
| 235 | + } |
| 236 | + if (result == null) { |
| 237 | + throw new IllegalArgumentException("Missing token '" + text + "' in [" + start + ", " + end + ")"); |
| 238 | + } |
| 239 | + return result; |
| 240 | + } |
| 241 | + |
| 242 | + InternalSyntaxToken slice(int start, int end) { |
| 243 | + int index = index(start); |
| 244 | + if (index > 0 && tokens.get(index - 1).end() > start) { |
| 245 | + index--; |
| 246 | + } |
| 247 | + var lexeme = tokens.get(index); |
| 248 | + if (lexeme.start() == start && lexeme.end() == end) { |
| 249 | + return lexeme.token(); |
| 250 | + } |
| 251 | + long key = ((long) start << 32) | end; |
| 252 | + return slices.computeIfAbsent(key, unused -> { |
| 253 | + var pos = positions.toPos(start); |
| 254 | + List<SyntaxTrivia> trivia = lexeme.start() == start ? lexeme.token().trivias() : List.of(); |
| 255 | + return new InternalSyntaxToken(pos.line(), pos.columnOffset(), source.substring(start, end), trivia, false); |
| 256 | + }); |
| 257 | + } |
| 258 | +} |
0 commit comments