Fix patchedast region for implicitly-concatenated f-strings

`_JoinedStr` builds its children as [start_quote, *FormattedValues,
end_quote] and lets `_handle` consume each by forward search. The
end-quote child therefore matches the first closing quote after the last
interpolation -- which, in an implicit concatenation, closes only the
part that held that interpolation, not the whole concatenation.

The node's region was then understated and, worse, the scanner was left
parked inside source text that had not been recognised as a string
literal. A `#` in a trailing part -- `f"{a} one " f"#52 two"` -- then
reads to `_good_token` as the start of a real comment, so the following
`)` is skipped as commented-out and `consume` raises
MismatchedTokenError (issue #879).

The truncation itself predates the `#`: a hash-free concatenation such
as `f"{a} one " f"52 two"` also produced a short region, it just failed
silently because nothing downstream needed to match a token past the
truncation point.

`consume_string` already measures the full concatenation before
`_JoinedStr` rewinds the scanner, so close the gap from its end offset
rather than re-scanning. Plain (non-f) strings were never affected
because `_Str` consumes via that same pattern.

Fixes #879

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: stevemadere

CHANGELOG.md

@@ -8,6 +8,7 @@
 - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan)
 - #870 Add default implementation for is_dir() (@lieryan)
 - #872 Fix unicode handling in patchedast (@lieryan)
+- #879 Fix patchedast region for implicitly-concatenated f-strings whose trailing parts hold no interpolation (@stevemadere)
 
 # Release 1.14.0
 

rope/refactor/patchedast.py

@@ -408,6 +408,27 @@ def end_quote_char():
                 children.append(part)
         children.append(end_quote_char())
         self._handle(node, children)
+        self._extend_joined_string_to(node, end)
+
+    def _extend_joined_string_to(self, node, string_end):
+        """Cover concatenation parts that follow the last interpolation.
+
+        `_handle` stops at the first closing quote it finds after the last
+        `FormattedValue`, so in an implicit concatenation whose trailing parts
+        are pure literal text -- ``f"{a} one " f"#52 two"`` -- it stops at the
+        end of the part holding the interpolation.  That both understates the
+        node's region and parks the scanner inside the literal, where an
+        unescaped ``#`` reads as the start of a comment and swallows the rest
+        of the line.  `consume_string` already measured the whole
+        concatenation, so close the gap from its end offset.
+        """
+        if self.source.offset >= string_end:
+            return
+        trailing_parts = self.source[self.source.offset : string_end]
+        if self.children:
+            node.sorted_children.append(trailing_parts)
+        self.source.offset = string_end
+        node.region = (node.region[0], string_end)
 
     def _FormattedValue(self, node):
         children = []

ropetest/refactor/patchedasttest.py

@@ -337,6 +337,48 @@ def test_handling_format_strings_with_implicit_join(self):
         )
         checker.check_children("FormattedValue", ["{", "", "Name", "", "}"])
 
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_with_implicit_join_trailing_literal_part(self):
+        source = 'a = f"one {b}" f"two"\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children(
+            "JoinedStr", ['f"', "one ", "FormattedValue", "", '"', ' f"two"']
+        )
+        checker.check_region("JoinedStr", 4, len('a = f"one {b}" f"two"'))
+
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_with_hash_in_joined_trailing_part(self):
+        source = dedent("""\
+            E(
+                f"{a} one "
+                f"#52 two")
+        """)
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children(
+            "JoinedStr", ['f"', "", "FormattedValue", " one ", '"', '\n    f"#52 two"']
+        )
+        checker.check_children("Call", ["Name", "", "(", "\n    ", "JoinedStr", "", ")"])
+
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_with_hash_in_joined_trailing_part_assignment(self):
+        source = 'a = 1\nb = (f"{a} one "\n     f"#52 two")\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children(
+            "JoinedStr",
+            ['f"', "", "FormattedValue", " one ", '"', '\n     f"#52 two"'],
+        )
+
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_followed_by_a_real_comment(self):
+        source = 'a = f"one {b}"  # trailing comment\nc = 1\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children("JoinedStr", ['f"', "one ", "FormattedValue", "", '"'])
+        checker.check_region("JoinedStr", 4, len('a = f"one {b}"'))
+
     @testutils.only_for_versions_higher("3.6")
     def test_handling_format_strings_with_format_spec(self):
         source = 'f"abc{a:01}"\n'
@@ -2001,6 +2043,12 @@ def __call__(self, node):
         ast.call_for_nodes(self.ast, search)
         return search.result
 
+    def check_region(self, text, start, end):
+        node = self._find_node(text)
+        if node is None:
+            self.test_case.fail("Node <%s> cannot be found" % text)
+        self.test_case.assertEqual((start, end), node.region)
+
     def check_children(self, text, children):
         node = self._find_node(text)
         if node is None: