66import re
77import sys
88import tomllib
9+ from collections .abc import Container , Sequence
10+ from enum import Enum
911from pathlib import Path
1012from time import time
11- from typing import Sequence
1213
1314import tomlkit
1415
@@ -63,17 +64,24 @@ def run_tests(
6364 update_type_checker_info (type_checker , root_dir )
6465
6566
67+ class ErrorMultiplicity (Enum ):
68+ """How many lines with the same tag can have an error."""
69+
70+ SINGLE = 1 # exactly one line must have an error
71+ MULTI = 2 # at least one line must have an error
72+ REQUIRE_SUCCESS = 3 # at least one line must not have an error
73+
74+
6675def get_expected_errors (test_case : Path ) -> tuple [
6776 dict [int , tuple [int , int ]],
68- dict [str , tuple [list [int ], bool ]],
77+ dict [str , tuple [list [int ], ErrorMultiplicity ]],
6978]:
7079 """Return the line numbers where type checkers are expected to produce an error.
7180
7281 The return value is a tuple of two dictionaries:
7382 - The format of the first is {line number: (number of required errors, number of optional errors)}.
74- - The format of the second is {error tag: ([lines where the error may appear], allow multiple}.
75- If allow multiple is True, the error may appear on multiple lines; otherwise, it must
76- appear exactly once.
83+ - The format of the second is {error tag: ([lines where the error may appear], error multiplicity)}.
84+ See the ErrorMultiplicy enum for the second argument.
7785
7886 For example, the following test case:
7987
@@ -92,7 +100,7 @@ def f(): pass # E[final]
92100 with open (test_case , "r" , encoding = "utf-8" ) as f :
93101 lines = f .readlines ()
94102 output : dict [int , tuple [int , int ]] = {}
95- groups : dict [str , tuple [list [int ], bool ]] = {}
103+ groups : dict [str , tuple [list [int ], ErrorMultiplicity ]] = {}
96104 for i , line in enumerate (lines , start = 1 ):
97105 line_without_comment , * _ = line .split ("#" )
98106 # Ignore lines with no non-comment content. This allows commenting out test cases.
@@ -110,15 +118,18 @@ def f(): pass # E[final]
110118 for match in re .finditer (r"# E\[([^\]]+)\]" , line ):
111119 tag = match .group (1 )
112120 if tag .endswith ("+" ):
113- allow_multiple = True
121+ multiplicity = ErrorMultiplicity .MULTI
122+ tag = tag [:- 1 ]
123+ elif tag .endswith ("!" ):
124+ multiplicity = ErrorMultiplicity .REQUIRE_SUCCESS
114125 tag = tag [:- 1 ]
115126 else :
116- allow_multiple = False
127+ multiplicity = ErrorMultiplicity . SINGLE
117128 if tag not in groups :
118- groups [tag ] = ([i ], allow_multiple )
129+ groups [tag ] = ([i ], multiplicity )
119130 else :
120- if groups [tag ][1 ] != allow_multiple :
121- raise ValueError (f"Error group { tag } has inconsistent allow_multiple value in { test_case } " )
131+ if groups [tag ][1 ] != multiplicity :
132+ raise ValueError (f"Error group { tag } has inconsistent multiplicity value in { test_case } " )
122133 groups [tag ][0 ].append (i )
123134 for group , linenos in groups .items ():
124135 if len (linenos ) == 1 :
@@ -151,21 +162,47 @@ def diff_expected_errors(
151162 differences .append (f"Line { expected_lineno } : Expected { expected_count } errors" )
152163 # We don't report an issue if the count differs, because type checkers may produce
153164 # multiple error messages for a single line.
154- linenos_used_by_groups : set [int ] = set ()
155- for group , (linenos , allow_multiple ) in error_groups .items ():
156- num_errors = sum (1 for lineno in linenos if lineno in errors )
157- if num_errors == 0 :
158- differences .append (f"Lines { ', ' .join (map (str , linenos ))} : Expected error (tag { group !r} )" )
159- elif num_errors == 1 or allow_multiple :
160- linenos_used_by_groups .update (linenos )
161- else :
162- differences .append (f"Lines { ', ' .join (map (str , linenos ))} : Expected exactly one error (tag { group !r} )" )
165+
166+ for group , (linenos , multiplicity ) in error_groups .items ():
167+ error = determine_group_error (group , linenos , multiplicity , errors )
168+ if error is not None :
169+ differences .append (error )
170+
171+ linenos_used_by_groups = {ln for linenos , _ in error_groups .values () for ln in linenos }
163172 for actual_lineno , actual_errors in errors .items ():
164- if actual_lineno not in expected_errors and actual_lineno not in linenos_used_by_groups :
165- differences .append (f"Line { actual_lineno } : Unexpected errors { actual_errors } " )
173+ if (
174+ actual_lineno not in expected_errors
175+ and actual_lineno not in linenos_used_by_groups
176+ ):
177+ differences .append (
178+ f"Line { actual_lineno } : Unexpected errors { actual_errors } "
179+ )
166180 return "" .join (f"{ diff } \n " for diff in differences )
167181
168182
183+ def determine_group_error (
184+ group : str ,
185+ group_linenos : Sequence [int ],
186+ multiplicity : ErrorMultiplicity ,
187+ error_linenos : Container [int ],
188+ ) -> str | None :
189+ """Return the error message for the given group or None."""
190+ num_errors = sum (1 for lineno in group_linenos if lineno in error_linenos )
191+ match multiplicity :
192+ case ErrorMultiplicity .SINGLE :
193+ if num_errors == 0 :
194+ return f"Lines { ', ' .join (map (str , group_linenos ))} : Expected error (tag { group !r} )"
195+ elif num_errors > 1 :
196+ return f"Lines { ', ' .join (map (str , group_linenos ))} : Expected exactly one error (tag { group !r} )"
197+ case ErrorMultiplicity .MULTI :
198+ if num_errors == 0 :
199+ return f"Lines { ', ' .join (map (str , group_linenos ))} : Expected error (tag { group !r} )"
200+ case ErrorMultiplicity .REQUIRE_SUCCESS :
201+ if num_errors == len (group_linenos ):
202+ return f"Lines { ', ' .join (map (str , group_linenos ))} : Expected at least one success (tag { group !r} )"
203+ return None
204+
205+
169206def update_output_for_test (
170207 type_checker : TypeChecker ,
171208 results_dir : Path ,
0 commit comments