Add basic implementation for mypy: ignore comments (#22180)

Fixes #12358

Implementation is mostly straightforward, but there are few things:
* This requires mypyc/ast_serialize#95, so tests
will fail until we merge that.
* We forgot to bump cache version to `6`. This is trivial however,
functions now have a flag indicating if they have a type comment.
* I found a pretty bad bug in native parser: parse errors cannot be
type-ignored even if they are non-blocking. Again, it is a one line
change, so not making a separate PR for this.

Finally, there is one thing that bothers me a bit: we have few places
where we mention words `type: ignore` in errors and notes. However, it
would take a significant effort to reliably change them to `mypy:
ignore` when needed. Essentially we would need to record provenance for
each error code, and thread it through to many places (including to
cache metas, for some technical reasons). So we have three options:
1. Do nothing about it. `mypy: ignore` really means mypy-specific type
ignore, so it is fine to use `type: ignore` in error notes, if we
actually have `mypy: ignore` written in code.
2. Actually implement all the work needed (not a fan of this option
TBH).
3. Change the wording in those few errors/notes so that it is more
neutral: i.e. say `Unused "ignore" comment` instead of `Unused "type:
ignore" comment` etc.

I am leaning towards option 3, but wanted to double-check before
actually doing this.

Author: ilevkivskyi

docs/source/error_codes.rst

@@ -53,6 +53,43 @@ codes prefixed with ``mypy:`` will be validated:
    # OK: error code may be used by other type checker.
    y = 2  # type: ignore[something-else]
 
+An alternative way to specify mypy-specific type ignores is to use
+``# mypy: ignore[one-code, other-code]``. Such form is equivalent to writing
+``# type: ignore[mypy:one-code, mypy:other-code]``. You can have both
+``# type: ignore[...]`` and ``# mypy: ignore[...]`` on the same line, in this
+case the error code lists will be merged.
+
+Note that to allow better interoperability with other tools, mypy recognizes
+``# type: ignore`` and ``mypy: ignore`` in any position in a comment, including
+after another "magic" comments used by other tools. To temporarily disable
+an ignore, place it after an empty (i.e. whitespace only) comment. For example:
+
+.. code-block:: python
+
+   from typing import Any
+
+   # No error, type ignore is active.
+   x: str = 1  # noqa  # pragma: no cover  # type: ignore[assignment]
+
+   # Error: Incompatible types in assignment (expression has type "int", variable has type "str")
+   y: str = 1  ## type: ignore[assignment]
+
+   # Error: Incompatible types in assignment (expression has type "int", variable has type "str")
+   z: str = 1  # # type: ignore[assignment]
+
+The only exception to this logic, is if there is no other code on a line
+except for the comment. In this case type ignores preceded by another comment
+are disabled. This is done to avoid spurious errors about unused ignores in
+a commented out block of code:
+
+.. code-block:: python
+
+   # mypy: warn-unused-ignores
+
+   x: int = 1
+   # OK, no unused-ignore error below.
+   # x = "no way"  # type: ignore[assignment]
+
 Enabling/disabling specific error codes globally
 ------------------------------------------------
 

mypy/message_registry.py

@@ -311,6 +311,9 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
     codes.SYNTAX,
 )
 INVALID_TYPE_IGNORE: Final = ErrorMessage('Invalid "type: ignore" comment', codes.SYNTAX)
+NO_PREFIX_IN_MYPY_IGNORE: Final = (
+    'Error codes in "mypy: ignore" should not be prefixed with "mypy:"'
+)
 TYPE_COMMENT_SYNTAX_ERROR_VALUE: Final = ErrorMessage(
     'Syntax error in type comment "{}"', codes.SYNTAX
 )

mypy/nativeparse.py

@@ -48,6 +48,7 @@
 )
 from mypy.errorcodes import error_codes
 from mypy.errors import CompileError
+from mypy.message_registry import NO_PREFIX_IN_MYPY_IGNORE
 from mypy.nodes import (
     ARG_KINDS,
     ARG_POS,
@@ -235,14 +236,25 @@ def native_parse(
         b,
         errors,
         ignores,
+        mypy_ignores,
         import_bytes,
         is_partial_package,
         uses_template_strings,
         source_hash,
         mypy_comments,
     ) = parse_to_binary_ast(filename, options, source, skip_function_bodies)
+
     ignores, extra_errors = normalize_error_codes(ignores)
     errors.extend(extra_errors)
+    mypy_ignores, extra_errors = normalize_error_codes(
+        mypy_ignores, verify_all=True, prohibit_prefix=True
+    )
+    errors.extend(extra_errors)
+
+    ignores = merge_type_ignores(ignores, mypy_ignores)
+    ignores, extra_errors = remove_duplicate_codes(ignores)
+    errors.extend(extra_errors)
+
     node = MypyFile([], [])
     node.path = filename
     node.raw_data = FileRawData(
@@ -258,29 +270,89 @@ def native_parse(
     return node, errors, ignores
 
 
-def normalize_error_codes(ignores: TypeIgnores) -> tuple[TypeIgnores, list[ParseError]]:
+def normalize_error_codes(
+    ignores: TypeIgnores, verify_all: bool = False, prohibit_prefix: bool = False
+) -> tuple[TypeIgnores, list[ParseError]]:
     """Remove mypy: prefix from error codes (if present)."""
     normalized = []
     extra_errors: list[ParseError] = []
     for line, codes in ignores:
         new_codes = []
         for c in codes:
-            if not c.startswith("mypy:"):
-                new_codes.append(c)
-                continue
+            prefixed = c.startswith("mypy:")
             c = c.removeprefix("mypy:")
-            if c in error_codes:
-                new_codes.append(c)
+            # Always record unknown codes, so that type: ignore[unknown]
+            # will not become blanket ignore.
+            new_codes.append(c)
+            if (prefixed or verify_all) and c not in error_codes:
+                extra_errors.append(
+                    {
+                        "line": line,
+                        "column": -1,
+                        "message": f'Unrecognized error code "{c}"',
+                        "blocker": False,
+                        "code": "misc",
+                    }
+                )
+            if prefixed and prohibit_prefix:
+                extra_errors.append(
+                    {
+                        "line": line,
+                        "column": -1,
+                        "message": NO_PREFIX_IN_MYPY_IGNORE,
+                        "blocker": False,
+                        "code": "misc",
+                    }
+                )
+        normalized.append((line, new_codes))
+    return normalized, extra_errors
+
+
+def merge_type_ignores(ignores: TypeIgnores, mypy_ignores: TypeIgnores) -> TypeIgnores:
+    """Merge error codes specified in # type: ignore[...] and in # mypy: ignore[...]."""
+    merged = []
+    ignores_by_line = dict(ignores)
+    mypy_ignores_by_line = dict(mypy_ignores)
+    for line in sorted(ignores_by_line.keys() | mypy_ignores_by_line.keys()):
+        codes = ignores_by_line.get(line)
+        mypy_codes = mypy_ignores_by_line.get(line)
+        if codes is not None and mypy_codes is None:
+            new_codes = codes
+        elif codes is None and mypy_codes is not None:
+            new_codes = mypy_codes
+        else:
+            assert codes is not None and mypy_codes is not None
+            if not codes or not mypy_codes:
+                # If at least one ignore is blanket, make the merged ignore blanket.
+                # We are not overly strict here, in case someone wants to quickly
+                # silence everything.
+                new_codes = []
+            else:
+                new_codes = codes + mypy_codes
+        merged.append((line, new_codes))
+    return merged
+
+
+def remove_duplicate_codes(ignores: TypeIgnores) -> tuple[TypeIgnores, list[ParseError]]:
+    normalized = []
+    extra_errors: list[ParseError] = []
+    for line, codes in ignores:
+        seen = set()
+        new_codes = []
+        for c in codes:
+            if c in seen:
+                extra_errors.append(
+                    {
+                        "line": line,
+                        "column": -1,
+                        "message": f'Duplicate error code "{c}"',
+                        "blocker": False,
+                        "code": "misc",
+                    }
+                )
                 continue
-            extra_errors.append(
-                {
-                    "line": line,
-                    "column": -1,
-                    "message": f'Unrecognized error code "{c}"',
-                    "blocker": False,
-                    "code": "misc",
-                }
-            )
+            seen.add(c)
+            new_codes.append(c)
         normalized.append((line, new_codes))
     return normalized, extra_errors
 
@@ -327,7 +399,17 @@ def parse_to_binary_ast(
     options: Options,
     source: str | bytes | None = None,
     skip_function_bodies: bool = False,
-) -> tuple[bytes, list[ParseError], TypeIgnores, bytes, bool, bool, str, list[tuple[int, str]]]:
+) -> tuple[
+    bytes,
+    list[ParseError],
+    TypeIgnores,
+    TypeIgnores,
+    bytes,
+    bool,
+    bool,
+    str,
+    list[tuple[int, str]],
+]:
     try:
         ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse(
             filename,
@@ -337,7 +419,7 @@ def parse_to_binary_ast(
             platform=options.platform,
             always_true=options.always_true,
             always_false=options.always_false,
-            cache_version=5,
+            cache_version=7,
         )
     except ValueError as exc:
         if source is None:
@@ -366,6 +448,7 @@ def parse_to_binary_ast(
         ast_bytes,
         errors,
         ignores,
+        ast_data["mypy_ignores"],
         import_bytes,
         ast_data["is_partial_package"],
         ast_data["uses_template_strings"],
@@ -796,6 +879,7 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef:
     func_def = FuncDef(name, arguments, body, typ=typ, type_args=type_params)
     if is_async:
         func_def.is_coroutine = True
+    func_def.has_type_comments = read_bool(data)
     read_loc(data, func_def)
     if type_params:
         state.check_min_version(

mypy/nodes.py

@@ -1114,6 +1114,7 @@ class FuncDef(FuncItem, SymbolNode, Statement):
         "docstring",
         "deprecated",
         "original_first_arg",
+        "has_type_comments",
     )
 
     __match_args__ = ("name", "arguments", "type", "body")
@@ -1156,6 +1157,9 @@ def __init__(
         # We record this status to avoid multiple (similar but different) errors in case
         # of partial types etc.
         self.is_invalid_redefinition = False
+        # Record if the function definition has any type comments (including per-argument).
+        # This is only available with the native parser (and always False otherwise).
+        self.has_type_comments = False
 
     @property
     def name(self) -> str:

mypy/parse.py

@@ -95,6 +95,7 @@ def load_from_raw(
     # Report parse errors, this replicates the logic in parse().
     all_errors = raw_data.raw_errors + state.errors
     errors.set_file(fnam, module, options=options)
+    errors.set_file_ignored_lines(fnam, tree.ignored_lines, options.ignore_errors)
     for error in all_errors:
         # Note we never raise in this function, so it should not be called in coordinator.
         report_parse_error(error, errors)

mypy/test/test_nativeparse.py

@@ -268,15 +268,15 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) ->
 
     def test_trivial_binary_data_from_file(self) -> None:
         with temp_source("print('hello')") as fnam:
-            b, _, _, _, _, _, _, _ = parse_to_binary_ast(fnam, Options())
+            b, _, _, _, _, _, _, _, _ = parse_to_binary_ast(fnam, Options())
             self._assert_trivial_binary_data(b)
 
     def test_trivial_binary_data_from_string_source(self) -> None:
-        b, _, _, _, _, _, _, _ = parse_to_binary_ast("", Options(), "print('hello')")
+        b, _, _, _, _, _, _, _, _ = parse_to_binary_ast("", Options(), "print('hello')")
         self._assert_trivial_binary_data(b)
 
     def test_trivial_binary_data_from_bytes_source(self) -> None:
-        b, _, _, _, _, _, _, _ = parse_to_binary_ast("", Options(), b"print('hello')")
+        b, _, _, _, _, _, _, _, _ = parse_to_binary_ast("", Options(), b"print('hello')")
         self._assert_trivial_binary_data(b)
 
     def test_invalid_bytes_raises(self) -> None:

test-data/unit/check-deprecated.test

@@ -49,7 +49,7 @@ from typing_extensions import deprecated
 @deprecated("use f2 instead")
 def f() -> None: ...
 
-f  # E: function __main__.f is deprecated: use f2 instead  # type: ignore[deprecated]
+f  # E: function __main__.f is deprecated: use f2 instead
 f(1)  # E: function __main__.f is deprecated: use f2 instead \
       # E: Too many arguments for "f"
 f[1]  # E: function __main__.f is deprecated: use f2 instead \

test-data/unit/check-fastparse.test

@@ -77,7 +77,8 @@ for i in (1, 2, 3, 100):  # type: str # type: ignore
     pass
 [builtins fixtures/tuple.pyi]
 
-[case testFastParseVariableCommentThenIgnore]
+-- New parser has a different stance on this.
+[case testFastParseVariableCommentThenIgnore_old_parser]
 a="test" # type: int #comment # type: ignore  # E: Incompatible types in assignment (expression has type "str", variable has type "int")
 
 [case testFastParseProperty]