Merge pull request #14903 from RonnyPfannschmidt/fix-10745-cheaper-recursion-tracebacks

Make rendering deep and recursive tracebacks cheap

Author: RonnyPfannschmidt

changelog/10745.improvement.1.rst

@@ -0,0 +1,3 @@
+The line numbers of a module's statements are now computed once per traceback
+instead of once per traceback entry, which noticeably speeds up rendering
+failures raised from large test modules.

changelog/10745.improvement.2.rst

@@ -0,0 +1,5 @@
+A ``RecursionError`` whose recursion origin cannot be located now shows only the
+first and last 10 stack frames, matching what pytest already did when locating
+the origin failed outright. Previously the full traceback -- often around a
+thousand near-identical frames -- was rendered, which was both slow and
+unreadable. Pass ``--full-trace`` to see every frame.

src/_pytest/_code/code.py

@@ -1167,10 +1167,10 @@ def _truncate_recursive_traceback(
         TypeError), in which case we do our best to warn the user of the
         error and show a limited traceback.
         """
+        max_frames = 10
         try:
             recursionindex = traceback.recursionindex()
         except Exception as e:
-            max_frames = 10
             extraline: str | None = (
                 "!!! Recursion error detected, but an error occurred locating the origin of recursion.\n"
                 "  The following exception happened when comparing locals in the stack frame:\n"
@@ -1184,6 +1184,18 @@ def _truncate_recursive_traceback(
             if recursionindex is not None:
                 extraline = "!!! Recursion detected (same locals & position)"
                 traceback = traceback[: recursionindex + 1]
+            elif self.tbfilter is not False and len(traceback) > 2 * max_frames:
+                # The origin of the recursion could not be pinned down (the frames
+                # differ), but rendering hundreds of near-identical entries is both
+                # slow and unreadable -- show the ends only, as above.
+                extraline = (
+                    "!!! Recursion error detected, but the origin of the recursion "
+                    "could not be located.\n"
+                    f"  Displaying first and last {max_frames} stack frames out of {len(traceback)}.\n"
+                    "  Pass `--full-trace` to see all frames."
+                )
+                # Type ignored for the same reason as above.
+                traceback = traceback[:max_frames] + traceback[-max_frames:]  # type: ignore
             else:
                 extraline = None
 

src/_pytest/_code/source.py

@@ -166,10 +166,19 @@ def deindent(lines: Iterable[str]) -> list[str]:
     return textwrap.dedent("\n".join(lines)).splitlines()
 
 
-def get_statement_startend2(lineno: int, node: ast.AST) -> tuple[int, int | None]:
+def _statement_linenos(node: ast.AST) -> list[int]:
+    """Sorted 0-based line numbers of all statements below ``node``.
+
+    Walking the tree is proportional to the size of the whole module, while a
+    traceback asks for many line numbers of the same module -- so the result is
+    memoized on the node, which shares the lifetime of the caller's ast cache.
+    """
+    values: list[int] | None = getattr(node, "_pytest_statement_linenos", None)
+    if values is not None:
+        return values
     # Flatten all statements and except handlers into one lineno-list.
     # AST's line numbers start indexing at 1.
-    values: list[int] = []
+    values = []
     for x in ast.walk(node):
         if isinstance(x, ast.stmt | ast.ExceptHandler):
             # The lineno points to the class/def, so need to include the decorators.
@@ -183,6 +192,15 @@ def get_statement_startend2(lineno: int, node: ast.AST) -> tuple[int, int | None
                     # Treat the finally/orelse part as its own statement.
                     values.append(val[0].lineno - 1 - 1)
     values.sort()
+    try:
+        node._pytest_statement_linenos = values  # type: ignore[attr-defined]
+    except AttributeError:  # pragma: no cover - defensive
+        pass
+    return values
+
+
+def get_statement_startend2(lineno: int, node: ast.AST) -> tuple[int, int | None]:
+    values = _statement_linenos(node)
     insert_index = bisect_right(values, lineno)
     if insert_index == 0:
         return 0, None

testing/code/test_excinfo.py

@@ -1876,6 +1876,47 @@ def b(x):
     )
 
 
+@pytest.mark.usefixtures("limited_recursion_depth")
+def test_undetectable_recursion_is_truncated() -> None:
+    """A recursion whose origin cannot be located is shown ends-only (#10745)."""
+
+    def f(x):
+        # The locals differ in every frame, so recursionindex() finds nothing.
+        f(x + 1)
+
+    with pytest.raises(RecursionError) as excinfo:
+        f(0)
+
+    p = ExceptionInfoFormatter(style="long", tbfilter=True)
+    traceback, extraline = p._truncate_recursive_traceback(excinfo.traceback)
+    assert len(traceback) == 20
+    assert extraline is not None
+    matcher = LineMatcher(extraline.splitlines())
+    matcher.fnmatch_lines(
+        [
+            "!!! Recursion error detected, but the origin of the recursion could not be located.",
+            "*Displaying first and last 10 stack frames out of *",
+            "*--full-trace*",
+        ]
+    )
+
+
+@pytest.mark.usefixtures("limited_recursion_depth")
+def test_undetectable_recursion_kept_with_full_trace() -> None:
+    """``--full-trace`` (tbfilter=False) still renders every frame (#10745)."""
+
+    def f(x):
+        f(x + 1)
+
+    with pytest.raises(RecursionError) as excinfo:
+        f(0)
+
+    p = ExceptionInfoFormatter(style="long", tbfilter=False)
+    traceback, extraline = p._truncate_recursive_traceback(excinfo.traceback)
+    assert len(traceback) == len(excinfo.traceback)
+    assert extraline is None
+
+
 @pytest.mark.usefixtures("limited_recursion_depth")
 def test_no_recursion_index_on_recursion_error():
     """

testing/code/test_source.py

@@ -1,6 +1,7 @@
 # mypy: allow-untyped-defs
 from __future__ import annotations
 
+import ast
 import inspect
 import linecache
 from pathlib import Path
@@ -706,3 +707,20 @@ def patched_compile2(_, *args, **kwargs):
 
     with patch("builtins.compile", new=patched_compile2):
         Source(patched_compile2).getstatement(1)
+
+
+def test_statement_linenos_are_memoized_per_node() -> None:
+    """The statement index of a module is reused across traceback entries (#10745)."""
+    from _pytest._code.source import _statement_linenos
+    from _pytest._code.source import get_statement_startend2
+
+    node = ast.parse("x = 1\nif x:\n    y = 2\n")
+    values = _statement_linenos(node)
+    assert values == [0, 1, 2]
+    # A second call reuses the very same list instead of walking the tree again.
+    assert _statement_linenos(node) is values
+    assert get_statement_startend2(1, node) == (1, 2)
+
+    other = ast.parse("z = 3\n")
+    assert _statement_linenos(other) == [0]
+    assert _statement_linenos(node) is values