Skip to content

Commit d05f214

Browse files
Implement expectation inheritance []
1 parent 0d49bfb commit d05f214

6 files changed

Lines changed: 385 additions & 101 deletions

File tree

‎dev_tools/bin/edg-rematerialize-test-run‎

Lines changed: 104 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@ import tempfile
1717

1818
import edgtest
1919
import edgtest.exec
20+
import edgtest.expectations
2021
import edgutil
2122

2223
from datetime import datetime
23-
from typing import Dict, List, Optional, Tuple
24+
from typing import Dict, Iterable, List, Optional, Set, Tuple
2425
from pathlib import Path, PurePosixPath
2526

2627
from edgutil import eprint
@@ -64,7 +65,7 @@ TOOL_PATH_PATCH = edgutil.RequiredToolPath('patch')
6465

6566
def build_diff_files(diff_file: Path,
6667
tmp_diffs_dir: Path) -> List[DiffFileDescr]:
67-
diff_files = []
68+
diff_files: List[DiffFileDescr] = []
6869

6970
with contextlib.ExitStack() as exit_stack:
7071
diff_file_handle = exit_stack.enter_context(open(diff_file))
@@ -112,8 +113,8 @@ def build_diff_files(diff_file: Path,
112113

113114
class TestChangeState:
114115
def __init__(self) -> None:
115-
self.tests = set()
116-
self.configs = set()
116+
self.tests: Set[Path] = set()
117+
self.configs: Set[str] = set()
117118

118119
def collect_affected_tests(test_home: Path,
119120
diff_files: List[DiffFileDescr]) -> TestChangeState:
@@ -159,9 +160,9 @@ def collect_affected_tests(test_home: Path,
159160
test_file_found = True
160161
else:
161162
# Look for all mft files that might be associated with this file.
162-
for possible_test in posix_path.iterdir():
163+
for possible_test in file_path.iterdir():
163164
if possible_test.suffixes[0] == '.mft':
164-
result.tests.add(candidate_test_file)
165+
result.tests.add(possible_test)
165166
test_file_found = True
166167
# File not found, move on to the parent.
167168
file_path = file_path.parent
@@ -354,11 +355,8 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
354355

355356
while len(test_options) < num_cases:
356357
# This should only happen in the case of reduction of test cases.
357-
# Use 's' (SKIP_TEST) as a placeholder mode.
358-
test_options.append((
359-
edgtest.runtest.test_type_to_string(edgtest.runtest.TestType.SKIP_TEST),
360-
[]
361-
))
358+
# Use SKIP_TEST as a placeholder mode.
359+
test_options.append((edgtest.runtest.TestType.SKIP_TEST, []))
362360

363361
with tempfile.TemporaryDirectory() as tmp_dir_name:
364362
tmp_dir = Path(tmp_dir_name)
@@ -411,7 +409,7 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
411409
await edgtest.exec.check_output_mismatch(
412410
old_rec,
413411
body_for_diff,
414-
os.environ,
412+
dict(os.environ),
415413
change_diff_path
416414
)
417415
)
@@ -500,15 +498,12 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
500498
case_entry['matches'] = matches_metadata
501499
rt_metadata_cases.append(case_entry)
502500

503-
# Make test status lines.
504-
if old_status != 'PASS':
505-
test_status_lines.append(
506-
f"- {relative_test_path.as_posix()} {test_details}{old_status}:\n"
507-
)
508-
test_status_lines.append(
509-
f"+ {relative_test_path.as_posix()} {test_details}{new_status}:\n"
510-
)
511-
elif any_mismatch:
501+
# Make test status lines (for the tests whose status changed).
502+
if old_status != new_status:
503+
if old_status != 'PASS':
504+
test_status_lines.append(
505+
f"- {relative_test_path.as_posix()} {test_details}{old_status}:\n"
506+
)
512507
test_status_lines.append(
513508
f"+ {relative_test_path.as_posix()} {test_details}{new_status}:\n"
514509
)
@@ -527,53 +522,108 @@ async def simulate_test(test_home: Path, config_run_dir: Path,
527522

528523
return test_status_lines
529524

530-
def load_test_suite_expectations(all_expectations: Dict[str, List[str]],
531-
suite_file: Path) -> None:
525+
def read_suite_expectation_lines(suite_file: Path) -> List[str]:
526+
if not suite_file.exists():
527+
return []
532528
with open(suite_file, 'r') as file_handle:
533-
for line in file_handle:
534-
test_name = f"{suite_file.stem}/{line.split(':')[0]}"
535-
if test_name not in all_expectations:
536-
all_expectations[test_name] = []
537-
all_expectations[test_name].append(line.strip())
529+
return file_handle.readlines()
530+
531+
def load_test_suite_expectations(all_expectations: Dict[str, List[str]],
532+
suite_name: str, expectations_dir: Path,
533+
parent_expectations_dir: Optional[Path]
534+
) -> None:
535+
'''Load the aggregate expectations of a suite.
536+
537+
A configuration that inherits expectations has the parent configuration's
538+
expectations with its own applied on top, so its file's lines are resolved
539+
into concrete expectations here.
540+
'''
541+
suite_file_name = f"{suite_name}.log"
542+
lines = read_suite_expectation_lines(expectations_dir / suite_file_name)
543+
if parent_expectations_dir is not None:
544+
lines = edgtest.expectations.overlay_expectations(
545+
read_suite_expectation_lines(parent_expectations_dir / suite_file_name),
546+
lines
547+
)
548+
549+
for expectation_line in lines:
550+
line = expectation_line.strip()
551+
test_name = f"{suite_name}/{line.split(':')[0]}"
552+
if test_name not in all_expectations:
553+
all_expectations[test_name] = []
554+
all_expectations[test_name].append(line)
538555

539556
async def simulate_config_testing(
540557
loop: asyncio.AbstractEventLoop,
541558
test_home: Path, new_run_dir: Path,
542559
patched_tests_dir: Path, config: str,
543560
diff_files: List[DiffFileDescr],
544-
tests: List[Path]) -> None:
561+
tests: Iterable[Path]) -> None:
545562
subtasks = []
546563

547564
config_run_dir = new_run_dir / config
548565
config_run_dir.mkdir()
549566

550-
old_expectations = {}
551-
new_expectations = {}
552-
with tempfile.TemporaryDirectory() as tmp_dir:
553-
src_expectations = test_home / 'expectations' / config
554-
dst_expectations = Path(tmp_dir, 'expectations', config)
555-
dst_expectations.mkdir(parents = True)
556-
557-
# Load the old expectations and copy the files to the tmp directory
558-
# (important so that expectations files that weren't modified are still
559-
# loaded and files that were modified are present to be patched).
560-
for suite_file in src_expectations.iterdir():
561-
load_test_suite_expectations(old_expectations, suite_file)
567+
# Configurations other than the default configuration inherit the default
568+
# configuration's expectations, so the default configuration's files (and the
569+
# changes to them) are part of what the configuration's expectations are.
570+
parent_config = edgtest.runtest.get_parent_config_name(config)
571+
if (parent_config is not None and
572+
not (test_home / 'expectations' / parent_config).is_dir()):
573+
parent_config = None
562574

563-
shutil.copy2(suite_file, dst_expectations / suite_file.name)
564-
565-
# Overwrite/patch any modified expectations files.
566-
await patch_files_with_prefix(
567-
loop,
568-
dst_expectations,
569-
src_expectations,
570-
f"tests/expectations/{config}/",
571-
diff_files
572-
)
575+
old_expectations: Dict[str, List[str]] = {}
576+
new_expectations: Dict[str, List[str]] = {}
577+
with tempfile.TemporaryDirectory() as tmp_dir:
578+
configs_to_load = [config]
579+
if parent_config is not None:
580+
configs_to_load.append(parent_config)
581+
582+
suite_names = set()
583+
for config_to_load in configs_to_load:
584+
src_expectations = test_home / 'expectations' / config_to_load
585+
dst_expectations = Path(tmp_dir, 'expectations', config_to_load)
586+
dst_expectations.mkdir(parents = True)
587+
588+
# Copy the files to the tmp directory (important so that expectations
589+
# files that weren't modified are still loaded and files that were
590+
# modified are present to be patched).
591+
for suite_file in src_expectations.iterdir():
592+
suite_names.add(suite_file.stem)
593+
shutil.copy2(suite_file, dst_expectations / suite_file.name)
594+
595+
# Overwrite/patch any modified expectations files.
596+
await patch_files_with_prefix(
597+
loop,
598+
dst_expectations,
599+
src_expectations,
600+
f"tests/expectations/{config_to_load}/",
601+
diff_files
602+
)
573603

574-
# Load the new expectations.
575-
for suite_file in dst_expectations.iterdir():
576-
load_test_suite_expectations(new_expectations, suite_file)
604+
# Load the old expectations (from the repository) and the new expectations
605+
# (from the patched copies).
606+
old_expectations_dir = test_home / 'expectations'
607+
new_expectations_dir = Path(tmp_dir, 'expectations')
608+
old_parent_dir = None
609+
new_parent_dir = None
610+
if parent_config is not None:
611+
old_parent_dir = old_expectations_dir / parent_config
612+
new_parent_dir = new_expectations_dir / parent_config
613+
614+
for suite_name in suite_names:
615+
load_test_suite_expectations(
616+
old_expectations,
617+
suite_name,
618+
old_expectations_dir / config,
619+
old_parent_dir
620+
)
621+
load_test_suite_expectations(
622+
new_expectations,
623+
suite_name,
624+
new_expectations_dir / config,
625+
new_parent_dir
626+
)
577627

578628
for test_path in tests:
579629
rel_test_path_str = str(

0 commit comments

Comments
 (0)