Merge remote-tracking branch 'origin/main' into gh-159117-profiler-dealloc

Author: Cha-Imaa

Doc/library/mailbox.rst

@@ -15,6 +15,12 @@ accessing and manipulating on-disk mailboxes and the messages they contain.
 :class:`~email.message.Message` class with format-specific state and behavior.
 Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF.
 
+.. warning::
+
+   The :mod:`!mailbox` module is designed for use in trusted environments,
+   where data from untrusted sources is never passed to the module's API
+   calls and only trusted users have access to the filesystems accessible
+   to the module.
 
 .. seealso::
 

Include/errcode.h

@@ -38,6 +38,7 @@ extern "C" {
 #define E_BADSINGLE      27      /* Ill-formed single statement input */
 #define E_INTERACT_STOP  28      /* Interactive mode stopped tokenization */
 #define E_COLUMNOVERFLOW 29      /* Column offset overflow */
+#define E_INCOMPLETE     30      /* An unfinished token in partial input */
 
 #ifdef __cplusplus
 }

Include/internal/pycore_parser.h

@@ -27,13 +27,17 @@ _Py_DECLARE_STR(empty, "")
         }, \
     }
 
+/* The caller must initialize *incomplete to NULL if supplied. A returned AST
+   can also return an owned exception there for a missing line terminator.
+   Validate the AST before reporting that exception. */
 extern struct _mod* _PyParser_ASTFromString(
     const char *str,
     PyObject* filename,
     int mode,
     PyCompilerFlags *flags,
     PyArena *arena,
-    PyObject *module);
+    PyObject *module,
+    PyObject **incomplete);
 
 extern struct _mod* _PyParser_ASTFromFile(
     FILE *fp,

Lib/codeop.py

@@ -62,15 +62,10 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
         warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
         try:
             compiler(source, filename, symbol, flags=flags)
+        except _IncompleteInputError:
+            return None
         except SyntaxError:  # Let other compile() errors propagate.
-            try:
-                compiler(source + "\n", filename, symbol, flags=flags)
-                return None
-            except _IncompleteInputError:
-                return None
-            except SyntaxError:
-                pass
-                # fallthrough
+            pass
 
     return compiler(source, filename, symbol, incomplete_input=False)
 

Lib/test/support/__init__.py

@@ -348,16 +348,17 @@ def requires(resource, msg=None):
     if resource == 'gui' and not _is_gui_available():
         raise ResourceDenied(_is_gui_available.reason)
 
-def _requires_unix_version(sysname, min_version):
-    """Decorator raising SkipTest if the OS is `sysname` and the version is less
+
+def _requires_platform_version(name, sys_platform, get_version_text, min_version):
+    """
+    Decorator raising SkipTest if the OS is `sysname` and the version is less
     than `min_version`.
 
-    For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if
-    the FreeBSD version is less than 7.2.
+    For example, @_requires_platform_version('FreeBSD', 'freebsd', (7, 2))
+    raises SkipTest if the FreeBSD version is less than 7.2.
     """
-    min_version_txt = '.'.join(map(str, min_version))
-    version_txt = platform.release().split('-', 1)[0]
-    if platform.system() == sysname:
+    version_txt = get_version_text()
+    if sys.platform == sys_platform:
         try:
             version = tuple(map(int, version_txt.split('.')))
         except ValueError:
@@ -367,21 +368,31 @@ def _requires_unix_version(sysname, min_version):
     else:
         skip = False
 
+    min_version_txt = '.'.join(map(str, min_version))
     return unittest.skipIf(
         skip,
-        f"{sysname} version {min_version_txt} or higher required, not "
+        f"{name} version {min_version_txt} or higher required, not "
         f"{version_txt}"
     )
 
 
+def _get_platform_release():
+    # Replace '7.2.8-200.fc44.x86_64' with '7.2.8'
+    return platform.release().partition('-')[0]
+
+
 def requires_freebsd_version(*min_version):
     """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is
     less than `min_version`.
 
     For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD
     version is less than 7.2.
     """
-    return _requires_unix_version('FreeBSD', min_version)
+    return _requires_platform_version(
+        'FreeBSD', 'freebsd',
+        get_version_text=_get_platform_release,
+        min_version=min_version)
+
 
 def requires_linux_version(*min_version):
     """Decorator raising SkipTest if the OS is Linux and the Linux version is
@@ -390,28 +401,10 @@ def requires_linux_version(*min_version):
     For example, @requires_linux_version(2, 6, 32) raises SkipTest if the Linux
     version is less than 2.6.32.
     """
-    return _requires_unix_version('Linux', min_version)
-
-def _requires_apple_os_version(name, sys_platform, get_version_text, min_version):
-    def decorator(func):
-        @functools.wraps(func)
-        def wrapper(*args, **kw):
-            if sys.platform == sys_platform:
-                version_txt = get_version_text()
-                try:
-                    version = tuple(map(int, version_txt.split('.')))
-                except ValueError:
-                    pass
-                else:
-                    if version < min_version:
-                        min_version_txt = '.'.join(map(str, min_version))
-                        raise unittest.SkipTest(
-                            f"{name} {min_version_txt} or higher required, "
-                            + f"not {version_txt}")
-            return func(*args, **kw)
-        wrapper.min_version = min_version
-        return wrapper
-    return decorator
+    return _requires_platform_version(
+        "Linux", "linux",
+        get_version_text=_get_platform_release,
+        min_version=min_version)
 
 
 def requires_mac_ver(*min_version):
@@ -421,7 +414,7 @@ def requires_mac_ver(*min_version):
     For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version
     is lesser than 10.5.
     """
-    return _requires_apple_os_version(
+    return _requires_platform_version(
         "macOS", "darwin",
         get_version_text=lambda: platform.mac_ver()[0],
         min_version=min_version)
@@ -434,7 +427,7 @@ def requires_ios_ver(*min_version):
     For example, requires_ios_ver(16, 0) raises SkipTest if the iOS version
     is lesser than 16.0.
     """
-    return _requires_apple_os_version(
+    return _requires_platform_version(
         "iOS", "ios",
         get_version_text=lambda: platform.ios_ver().release,
         min_version=min_version)

Lib/test/test_codeop.py

@@ -124,6 +124,99 @@ def test_incomplete_string_diagnostics(self, symbol, prefix):
         self.assertEqual(cm.exception.args, (
             'incomplete input', ('<input>', 2, 9, text, 2, -1)))
 
+    @subTests('compiler', COMPILERS)
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_unterminated_trailing_line(self, compiler, newline):
+        for head, symbol in (('1', 'eval'), ('if 1: pass', 'single'),
+                             ('if 1:\n pass', 'single')):
+            for tail in (' ', '\t', '\f ', '\n '):
+                source = head + newline + tail
+                with self.subTest(source=source, symbol=symbol):
+                    if compiler in RAW_COMPILERS:
+                        with self.assertRaises(_IncompleteInputError):
+                            compiler(source, '<input>', symbol)
+                    else:
+                        self.assertIsNone(compiler(source, '<input>', symbol))
+                    self.assertValid(source + '\n', symbol, compiler=compiler)
+            for tail in ('\f', ' \f', '\n\f'):
+                with self.subTest(head=head, tail=tail, symbol=symbol):
+                    self.assertValid(head + newline + tail, symbol,
+                                     compiler=compiler)
+
+        for tail in ('# comment', ' # comment'):
+            self.assertValid('if 1: pass' + newline + tail, compiler=compiler)
+            self.assertValid('1' + newline + tail, 'eval', compiler=compiler)
+
+        for head in ('1', 'if 1: pass', 'if 1:\n pass'):
+            for tail in (' ', '\t', '\f', '# comment'):
+                self.assertValid(head + newline + tail, 'exec',
+                                 compiler=compiler)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    def test_unfinished_token_in_invalid_position(self, symbol):
+        for source in ('a b', '(a b', '1 """', '(1 """',
+                       'f"{lambda:\\', 't"{lambda:\\'):
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaises(SyntaxError) as cm:
+                        compiler(source, '<input>', symbol)
+                    self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    def test_invalid_partial_string_diagnostic(self, symbol):
+        with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+            Compile()('a"""', '<input>', symbol)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('', 'r', 'b', 'f', 't', 'rf', 'rt'))
+    def test_string_continuation(self, symbol, prefix):
+        source = prefix + '"first\\'
+        compile(source + '\nsecond"\n', '<input>', symbol)
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_continued_final_line(self, symbol, newline):
+        source = '1\\' + newline
+        compile(source + '+ 2\n', '<input>', symbol)
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+        with self.assertRaises(SyntaxError) as cm:
+            Compile()('1 + \\' + newline, '<input>', symbol)
+        # Appending LF to CR completes the CRLF line terminator.
+        if newline != '\r':
+            self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec'))
+    def test_eof_before_nested_body(self, symbol):
+        for source in ('if True:\n    if True:',
+                       'if True:\n    def f():',
+                       'match 1:\n    case 1:'):
+            for tail in ('', '\n', '\n\n'):
+                with self.subTest(source=source, tail=tail):
+                    compile(source + tail + '\n        pass\n', '<input>', symbol)
+                    self.assertIsNone(compile_command(source + tail, symbol=symbol))
+                    with self.assertRaises(_IncompleteInputError):
+                        Compile()(source + tail, '<input>', symbol)
+
+    def test_incomplete_suite_semantic_errors(self):
+        cases = (
+            ('if True: break', "'break' outside loop"),
+            ('if True: return 1', "'return' outside function"),
+            ('def f(x, x):\n    pass', 'duplicate parameter'),
+        )
+        for source, message in cases:
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaises(SyntaxError) as cm:
+                        compiler(source, '<input>', 'single')
+                    self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+                    if compiler in RAW_COMPILERS:
+                        self.assertIn(message, str(cm.exception))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Lib/test/test_curses.py

@@ -14,6 +14,7 @@
                           check_disallow_instantiation, MISSING_C_DOCSTRINGS,
                           gc_collect, SHORT_TIMEOUT)
 from test.support.import_helper import import_module
+from test.support import os_helper
 
 # Optionally test curses module.  This currently requires that the
 # 'curses' resource be given on the regrtest command line using the -u
@@ -129,6 +130,9 @@ def _broken_variation_selector_width():
 
 BROKEN_VARIATION_SELECTOR_WIDTH = _broken_variation_selector_width()
 
+class CustomError(Exception):
+    pass
+
 # newterm() is used when available (it reports errors instead of exiting), but
 # initscr() is still the fallback, and an unusable $TERM has no terminal to
 # drive either way.
@@ -3353,6 +3357,50 @@ def test_set_term(self):
         self.assertIs(curses.set_term(a), b)  # returns the previous one
         self.assertIs(curses.set_term(b), a)
 
+    def make_screens_of_different_terms(self):
+        # Two screens of different sizes and numbers of colors.  Return them
+        # with the values of LINES, COLS, COLORS and COLOR_PAIRS for each.
+        screens = []
+        with os_helper.EnvironmentVarGuard() as env:
+            for term, lines, cols in [('xterm', 25, 80),
+                                      ('xterm-256color', 30, 100)]:
+                env['LINES'] = str(lines)
+                env['COLUMNS'] = str(cols)
+                s = self.make_pty()
+                try:
+                    screen = curses.newterm(term, s, s)
+                except curses.error:
+                    self.skipTest(f'no {term} terminfo entry')
+                curses.start_color()
+                self.assertEqual(screen.stdscr.getmaxyx(), (lines, cols))
+                screens.append((screen, self.get_screen_values()))
+        self.assertNotEqual(screens[0][1], screens[1][1])
+        return screens
+
+    def get_screen_values(self):
+        return curses.LINES, curses.COLS, curses.COLORS, curses.COLOR_PAIRS
+
+    def test_set_term_refreshes_lines_cols_colors(self):
+        (a, avalues), (b, bvalues) = self.make_screens_of_different_terms()
+        self.assertIs(curses.set_term(a), b)
+        self.assertEqual(self.get_screen_values(), avalues)
+        self.assertIs(curses.set_term(b), a)
+        self.assertEqual(self.get_screen_values(), bvalues)
+
+    @unittest.skipUnless(hasattr(curses.screen, 'use'),
+                         'requires curses.screen.use()')
+    def test_use_screen_refreshes_lines_cols_colors(self):
+        (a, avalues), (b, bvalues) = self.make_screens_of_different_terms()
+        self.assertEqual(a.use(lambda scr: self.get_screen_values()), avalues)
+        self.assertEqual(self.get_screen_values(), bvalues)
+
+        def raise_in_use(scr):
+            self.assertEqual(self.get_screen_values(), avalues)
+            raise CustomError
+        with self.assertRaises(CustomError):
+            a.use(raise_in_use)
+        self.assertEqual(self.get_screen_values(), bvalues)
+
     def test_window_keeps_screen_alive(self):
         # The standard window keeps its screen alive; dropping every other
         # reference and collecting must not invalidate the window.

Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst

@@ -0,0 +1,2 @@
+Improve incomplete input detection in :mod:`codeop` without an extra parser
+pass, and preserve syntax and compiler errors.

Misc/NEWS.d/next/Library/2026-10-18-08-30-10.gh-issue-155976.feSum4_j.rst

@@ -0,0 +1,3 @@
+:func:`curses.set_term` and :meth:`curses.screen.use` now update
+:data:`curses.LINES`, :data:`curses.COLS`, :data:`curses.COLORS`
+and :data:`curses.COLOR_PAIRS` for the current screen.