From 770a416042566662c088de543c6b70329e46b328 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Fri, 25 Sep 2026 12:28:51 -0300 Subject: [PATCH 1/5] Bound f-strings and match-sequence delimiters by parser coordinates A PEP 701 replacement field may span lines, which the textual string pattern cannot match, so JoinedStr raised MismatchedTokenError; its region now comes from the node's own coordinates, implicit concatenation included. A MatchSequence took every character before its first pattern as its opening delimiter, so a grouped first pattern (`((a | b), c)`) produced `((`; the delimiters are now the bracket pair that encloses the whole node, found by tokenizing its segment. Co-Authored-By: Claude Opus 5.5 --- rope/refactor/patchedast.py | 53 +++++++++++++++++------------ ropetest/refactor/patchedasttest.py | 38 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 2a52fc924..c5d5a13d1 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -1,6 +1,8 @@ import collections +import io import numbers import re +import tokenize import warnings from itertools import chain @@ -395,11 +397,10 @@ def end_quote_char(): return self.source[end - len(quote_char) : end] QUOTE_CHARS = ['"""', "'''", '"', "'"] - offset = self.source.offset - start, end = self.source.consume_string( - end=self._find_next_statement_start(), - ) - self.source.offset = offset + # The parser's coordinates bound the whole literal, implicit + # concatenation included. A textual string pattern cannot: since + # PEP 701 a replacement field may span lines (``f"{\n x\n}"``). + start, end = self.ast_adapter[node] children = [] children.append(start_quote_char()) @@ -817,24 +818,32 @@ def _MatchSequence(self, node): self._handle(node, children) def _get_surrounding_parens(self, node: ast.MatchSequence): + """The sequence's own ``[...]``/``(...)`` delimiters, if it has them. + + The text between the node's start and its first pattern can also + hold grouping parentheses of that pattern -- ``case (("a" | b), c)`` + -- which the AST does not represent. The sequence is delimited only + when the bracket at its first character is the one closed at its + last character; ``case (1), (2)`` starts and ends with parentheses + that belong to its elements. + """ node_start, node_end = self.ast_adapter[node] - first_pattern_start, _ = self.ast_adapter[node.patterns[0]] - _, last_pattern_end = self.ast_adapter[node.patterns[-1]] - opening_paren = self.source[node_start:first_pattern_start].strip() - closing_paren = self.source[last_pattern_end:node_end].strip() - - if opening_paren not in ["[", "(", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!", - RuntimeWarning, - ) - - if closing_paren not in ["]", ")", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!", - RuntimeWarning, - ) - return opening_paren, closing_paren + segment = self.source[node_start:node_end] + pairs = {"[": "]", "(": ")"} + if segment[:1] not in pairs or segment[-1:] != pairs[segment[0]]: + return "", "" + depth = 0 + for token in tokenize.generate_tokens(io.StringIO(segment).readline): + if token.type != tokenize.OP or token.string not in "()[]{}": + continue + depth += 1 if token.string in "([{" else -1 + if depth == 0: + end_row = segment.count("\n") + 1 + end_col = len(segment) - (segment.rfind("\n") + 1) + if token.end != (end_row, end_col): + return "", "" + return segment[0], segment[-1] + return "", "" def _MatchStar(self, node): self._handle(node, ["*", node.name or "_"]) diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index c014f6ed1..bfcfb0641 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -326,6 +326,15 @@ def test_handling_format_strings_basic(self): checker.check_children("JoinedStr", ['f"', "abc", "FormattedValue", "", '"']) checker.check_children("FormattedValue", ["{", "", "Name", "", "}"]) + @testutils.only_for_versions_higher("3.12") + def test_handling_format_strings_with_multiline_replacement_field(self): + source = 'x = (f"{a}{\n b\n}{c}").upper()\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index('f"') + checker.check_region("JoinedStr", start, source.index(").")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.6") def test_handling_format_strings_with_implicit_join(self): source = '''"1" + rf'abc{a}' f"""xxx{b} """\n''' @@ -1557,6 +1566,35 @@ def test_match_node_with_match_sequence_with_internal_parens(self): "MatchSequence", "", ",", " ", "MatchSequence" ]) + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_with_grouped_first_pattern(self): + source = dedent("""\ + match x: + case (("a" | None), None): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("MatchSequence", [ + "(", "(", "MatchOr", ")", ",", " ", "MatchSingleton", "", ")", + ]) + start = source.index("((") + checker.check_region("MatchSequence", start, source.index(":\n", start)) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_of_parenthesized_patterns(self): + source = dedent("""\ + match x: + case (1), (2): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index("(1)") + checker.check_region("MatchSequence", start, start + len("(1), (2)")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.10") def test_match_node_with_match_sequence_empty_round_parens(self): source = dedent("""\ From d8b776f01cedf45d98068dbbbd0848e913c9e7f1 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Fri, 25 Sep 2026 12:33:41 -0300 Subject: [PATCH 2/5] Start replacement fields at their parser coordinate In an implicit concatenation mixing f-strings and plain strings, the field's opening brace was searched for as text and matched a "{" inside a plain part (`"({})," f"{a}"`), shifting the field's region and duplicating source on write. Since Python 3.12 the parser locates each FormattedValue at its own "{"; the walker now starts there when the coordinate points at one. test_extract_method_f_string_false_format_value _in_regular_string, marked as an expected failure for this bug, passes. Co-Authored-By: Claude Opus 5.5 --- rope/refactor/patchedast.py | 8 ++++++++ ropetest/refactor/extracttest.py | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index c5d5a13d1..e20c40609 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -411,6 +411,14 @@ def end_quote_char(): self._handle(node, children) def _FormattedValue(self, node): + # Since Python 3.12 the parser locates each replacement field at its + # own "{". Starting there keeps a "{" inside a plain string of the + # same implicit concatenation -- ``"({})," f"{a}"`` -- from being + # taken as the field's opening brace. Older parsers give the + # enclosing literal's position, which does not point at a "{". + field_start, _ = self.ast_adapter[node] + if field_start >= self.source.offset and self.source[field_start] == "{": + self.source.offset = field_start children = [] children.append("{") children.append(node.value) diff --git a/ropetest/refactor/extracttest.py b/ropetest/refactor/extracttest.py index 4b34d58d3..4c063adbc 100644 --- a/ropetest/refactor/extracttest.py +++ b/ropetest/refactor/extracttest.py @@ -1284,7 +1284,6 @@ def new_func(a_var): """) self.assertEqual(expected, refactored) - @unittest.expectedFailure @testutils.only_for_versions_higher("3.6") def test_extract_method_f_string_false_format_value_in_regular_string(self): code = dedent("""\ From 94603cbdd5167a0b9e1d63f436c953db7d613705 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Fri, 25 Sep 2026 12:35:01 -0300 Subject: [PATCH 3/5] Add CHANGELOG entry for parser-coordinate patchedast fixes Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1650aa77a..c2a284e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan) - #870 Add default implementation for is_dir() (@lieryan) - #872 Fix unicode handling in patchedast (@lieryan) +- Locate multi-line f-strings, replacement fields and match-sequence delimiters in patchedast by parser coordinates # Release 1.14.0 From 3fc3d1ca2966dea9c3b7ad38f3b70ae59b391f76 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Sun, 18 Oct 2026 10:48:09 -0300 Subject: [PATCH 4/5] Match MatchSequence delimiters by tokenizing the node segment _get_surrounding_parens stripped the text around the first and last patterns and emitted a please-report RuntimeWarning whenever that text was not a single bracket, which happens for grouping parentheses of an element (case (("a" | b), c)) and for sequences whose elements are separately parenthesized (case (1), (2)). Tokenize the segment instead: the sequence has its own delimiters exactly when the bracket opening its first character closes at its last character. (cherry picked from commit 431061b90ba18a1022c47e310aa161b71ba1fe23) --- rope/refactor/patchedast.py | 44 ++++++++++++++++++----------- ropetest/refactor/patchedasttest.py | 38 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 70b7143bc..06632abfe 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -1,6 +1,8 @@ import collections +import io import numbers import re +import tokenize import warnings from itertools import chain @@ -775,24 +777,32 @@ def _MatchSequence(self, node): self._handle(node, children) def _get_surrounding_parens(self, node: ast.MatchSequence): + """The sequence's own ``[...]``/``(...)`` delimiters, if it has them. + + The text between the node's start and its first pattern can also + hold grouping parentheses of that pattern -- ``case (("a" | b), c)`` + -- which the AST does not represent. The sequence is delimited only + when the bracket at its first character is the one closed at its + last character; ``case (1), (2)`` starts and ends with parentheses + that belong to its elements. + """ node_start, node_end = self.ast_adapter[node] - first_pattern_start, _ = self.ast_adapter[node.patterns[0]] - _, last_pattern_end = self.ast_adapter[node.patterns[-1]] - opening_paren = self.source[node_start:first_pattern_start].strip() - closing_paren = self.source[last_pattern_end:node_end].strip() - - if opening_paren not in ["[", "(", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!", - RuntimeWarning, - ) - - if closing_paren not in ["]", ")", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!", - RuntimeWarning, - ) - return opening_paren, closing_paren + segment = self.source[node_start:node_end] + pairs = {"[": "]", "(": ")"} + if segment[:1] not in pairs or segment[-1:] != pairs[segment[0]]: + return "", "" + depth = 0 + for token in tokenize.generate_tokens(io.StringIO(segment).readline): + if token.type != tokenize.OP or token.string not in "()[]{}": + continue + depth += 1 if token.string in "([{" else -1 + if depth == 0: + end_row = segment.count("\n") + 1 + end_col = len(segment) - (segment.rfind("\n") + 1) + if token.end != (end_row, end_col): + return "", "" + return segment[0], segment[-1] + return "", "" def _MatchStar(self, node): self._handle(node, ["*", node.name or "_"]) diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index f3320f450..c754aae93 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -414,6 +414,15 @@ def test_handling_format_strings_with_implicit_join(self): ) checker.check_children("FormattedValue", ["{", "", "Name", "", "}"]) + @testutils.only_for_versions_higher("3.12") + def test_handling_format_strings_with_multiline_replacement_field(self): + source = 'x = (f"{a}{\n b\n}{c}").upper()\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index('f"') + checker.check_region("JoinedStr", start, source.index(").")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.6") def test_handling_format_strings_with_format_spec(self): source = 'f"abc{a:01}"\n' @@ -1597,6 +1606,35 @@ def test_match_node_with_match_sequence_with_no_parens(self): "MatchValue", "", ",", " ", "MatchValue", ]) + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_with_grouped_first_pattern(self): + source = dedent("""\ + match x: + case (("a" | None), None): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("MatchSequence", [ + "(", "(", "MatchOr", ")", ",", " ", "MatchSingleton", "", ")", + ]) + start = source.index("((") + checker.check_region("MatchSequence", start, source.index(":\n", start)) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_of_parenthesized_patterns(self): + source = dedent("""\ + match x: + case (1), (2): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index("(1)") + checker.check_region("MatchSequence", start, start + len("(1), (2)")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.10") def test_match_node_with_match_sequence_with_square_parens(self): source = dedent("""\ From 80958befafd0631027c55f9482f2c01be98d30be Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Sat, 31 Oct 2026 16:04:01 -0300 Subject: [PATCH 5/5] Keep the pull request to the MatchSequence delimiters The multiline replacement field case already passes on master after #896. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + ropetest/refactor/patchedasttest.py | 9 --------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee7ddf653..090ab8525 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - #895 patchedast Starred and keyword now consumes their syntactically expected and ** (@lieryan) - #896 patchedast cleanup and refactoring (@lieryan) - #905 Ignore `node_modules` directories by default when scanning project files and source folders. (@marcoeangeli) +- #882 Match MatchSequence delimiters by tokenizing the node segment (@marlon-costa-dc) # Release 1.15.0 diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index c754aae93..8add1aa7f 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -414,15 +414,6 @@ def test_handling_format_strings_with_implicit_join(self): ) checker.check_children("FormattedValue", ["{", "", "Name", "", "}"]) - @testutils.only_for_versions_higher("3.12") - def test_handling_format_strings_with_multiline_replacement_field(self): - source = 'x = (f"{a}{\n b\n}{c}").upper()\n' - ast_frag = patchedast.get_patched_ast(source, True) - checker = _ResultChecker(self, ast_frag) - start = source.index('f"') - checker.check_region("JoinedStr", start, source.index(").")) - self.assertEqual(source, patchedast.write_ast(ast_frag)) - @testutils.only_for_versions_higher("3.6") def test_handling_format_strings_with_format_spec(self): source = 'f"abc{a:01}"\n'