Skip to content

Commit 1f8d6f8

Browse files
authored
New conformance tag '# E[tag!]' to require at least one success (#2354)
1 parent 784a348 commit 1f8d6f8

2 files changed

Lines changed: 61 additions & 22 deletions

File tree

‎conformance/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ Test cases support the following special comments for declaring where errors sho
6363
* `# E[tag]`, where `tag` is an arbitrary string: must appear multiple times in a file with the same tag.
6464
Exactly one line with this tag must raise an error.
6565
* `# E[tag+]`: like `# E[tag]`, but errors may be raised on multiple lines.
66+
* `# E[tag!]`: like `# E[tag]`, but zero or more lines with this tag may raise
67+
an error while at least line must not raise an error.
6668

6769
Each comment may be followed by a colon plus an explanation of the error; the explanation is ignored
6870
by the scoring system.

‎conformance/src/main.py‎

Lines changed: 59 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,10 @@
66
import re
77
import sys
88
import tomllib
9+
from collections.abc import Container, Sequence
10+
from enum import Enum
911
from pathlib import Path
1012
from time import time
11-
from typing import Sequence
1213

1314
import tomlkit
1415

@@ -63,17 +64,24 @@ def run_tests(
6364
update_type_checker_info(type_checker, root_dir)
6465

6566

67+
class ErrorMultiplicity(Enum):
68+
"""How many lines with the same tag can have an error."""
69+
70+
SINGLE = 1 # exactly one line must have an error
71+
MULTI = 2 # at least one line must have an error
72+
REQUIRE_SUCCESS = 3 # at least one line must not have an error
73+
74+
6675
def get_expected_errors(test_case: Path) -> tuple[
6776
dict[int, tuple[int, int]],
68-
dict[str, tuple[list[int], bool]],
77+
dict[str, tuple[list[int], ErrorMultiplicity]],
6978
]:
7079
"""Return the line numbers where type checkers are expected to produce an error.
7180
7281
The return value is a tuple of two dictionaries:
7382
- The format of the first is {line number: (number of required errors, number of optional errors)}.
74-
- The format of the second is {error tag: ([lines where the error may appear], allow multiple}.
75-
If allow multiple is True, the error may appear on multiple lines; otherwise, it must
76-
appear exactly once.
83+
- The format of the second is {error tag: ([lines where the error may appear], error multiplicity)}.
84+
See the ErrorMultiplicy enum for the second argument.
7785
7886
For example, the following test case:
7987
@@ -92,7 +100,7 @@ def f(): pass # E[final]
92100
with open(test_case, "r", encoding="utf-8") as f:
93101
lines = f.readlines()
94102
output: dict[int, tuple[int, int]] = {}
95-
groups: dict[str, tuple[list[int], bool]] = {}
103+
groups: dict[str, tuple[list[int], ErrorMultiplicity]] = {}
96104
for i, line in enumerate(lines, start=1):
97105
line_without_comment, *_ = line.split("#")
98106
# Ignore lines with no non-comment content. This allows commenting out test cases.
@@ -110,15 +118,18 @@ def f(): pass # E[final]
110118
for match in re.finditer(r"# E\[([^\]]+)\]", line):
111119
tag = match.group(1)
112120
if tag.endswith("+"):
113-
allow_multiple = True
121+
multiplicity = ErrorMultiplicity.MULTI
122+
tag = tag[:-1]
123+
elif tag.endswith("!"):
124+
multiplicity = ErrorMultiplicity.REQUIRE_SUCCESS
114125
tag = tag[:-1]
115126
else:
116-
allow_multiple = False
127+
multiplicity = ErrorMultiplicity.SINGLE
117128
if tag not in groups:
118-
groups[tag] = ([i], allow_multiple)
129+
groups[tag] = ([i], multiplicity)
119130
else:
120-
if groups[tag][1] != allow_multiple:
121-
raise ValueError(f"Error group {tag} has inconsistent allow_multiple value in {test_case}")
131+
if groups[tag][1] != multiplicity:
132+
raise ValueError(f"Error group {tag} has inconsistent multiplicity value in {test_case}")
122133
groups[tag][0].append(i)
123134
for group, linenos in groups.items():
124135
if len(linenos) == 1:
@@ -151,21 +162,47 @@ def diff_expected_errors(
151162
differences.append(f"Line {expected_lineno}: Expected {expected_count} errors")
152163
# We don't report an issue if the count differs, because type checkers may produce
153164
# multiple error messages for a single line.
154-
linenos_used_by_groups: set[int] = set()
155-
for group, (linenos, allow_multiple) in error_groups.items():
156-
num_errors = sum(1 for lineno in linenos if lineno in errors)
157-
if num_errors == 0:
158-
differences.append(f"Lines {', '.join(map(str, linenos))}: Expected error (tag {group!r})")
159-
elif num_errors == 1 or allow_multiple:
160-
linenos_used_by_groups.update(linenos)
161-
else:
162-
differences.append(f"Lines {', '.join(map(str, linenos))}: Expected exactly one error (tag {group!r})")
165+
166+
for group, (linenos, multiplicity) in error_groups.items():
167+
error = determine_group_error(group, linenos, multiplicity, errors)
168+
if error is not None:
169+
differences.append(error)
170+
171+
linenos_used_by_groups = {ln for linenos, _ in error_groups.values() for ln in linenos}
163172
for actual_lineno, actual_errors in errors.items():
164-
if actual_lineno not in expected_errors and actual_lineno not in linenos_used_by_groups:
165-
differences.append(f"Line {actual_lineno}: Unexpected errors {actual_errors}")
173+
if (
174+
actual_lineno not in expected_errors
175+
and actual_lineno not in linenos_used_by_groups
176+
):
177+
differences.append(
178+
f"Line {actual_lineno}: Unexpected errors {actual_errors}"
179+
)
166180
return "".join(f"{diff}\n" for diff in differences)
167181

168182

183+
def determine_group_error(
184+
group: str,
185+
group_linenos: Sequence[int],
186+
multiplicity: ErrorMultiplicity,
187+
error_linenos: Container[int],
188+
) -> str | None:
189+
"""Return the error message for the given group or None."""
190+
num_errors = sum(1 for lineno in group_linenos if lineno in error_linenos)
191+
match multiplicity:
192+
case ErrorMultiplicity.SINGLE:
193+
if num_errors == 0:
194+
return f"Lines {', '.join(map(str, group_linenos))}: Expected error (tag {group!r})"
195+
elif num_errors > 1:
196+
return f"Lines {', '.join(map(str, group_linenos))}: Expected exactly one error (tag {group!r})"
197+
case ErrorMultiplicity.MULTI:
198+
if num_errors == 0:
199+
return f"Lines {', '.join(map(str, group_linenos))}: Expected error (tag {group!r})"
200+
case ErrorMultiplicity.REQUIRE_SUCCESS:
201+
if num_errors == len(group_linenos):
202+
return f"Lines {', '.join(map(str, group_linenos))}: Expected at least one success (tag {group!r})"
203+
return None
204+
205+
169206
def update_output_for_test(
170207
type_checker: TypeChecker,
171208
results_dir: Path,

0 commit comments

Comments
 (0)