From 81c07ac139cf7610b6da2a01e3b194dba86d99e4 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 7 Nov 2026 01:29:26 -0800 Subject: [PATCH 1/5] test: cover video keys in change detection and recipe rendering --- .../infx/tests/matrix/test_changed.py | 68 +++++++++++++------ inferencex-e2e/infx/tests/srt_recipes.py | 42 +++++++++++- .../infx/tests/srt_slurm/test_generate.py | 52 +++++++++++--- 3 files changed, 129 insertions(+), 33 deletions(-) diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index 14a09273ed..0c69a72087 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -9,7 +9,14 @@ import yaml from infx.matrix import changed -from infx.tests.srt_recipes import single_node_fragment, write_shared_blocks +from infx.tests.srt_recipes import ( + VIDEO_MODEL, + VIDEO_SPEC, + single_node_fragment, + video_fragment, + write_shared_blocks, + write_video_specs, +) ROOT = Path(__file__).resolve().parents[3] RECIPES = "benchmarks/single_node/srt-slurm-recipes/fixture" @@ -26,9 +33,21 @@ def entry(recipe: str, concurrencies: list[int]) -> dict: } # fmt: skip -def write(root: Path, master: dict, edited: dict) -> None: - (root / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) - (root / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited)) +def video_entry() -> dict: + return { + "image": "example/video:1", "model": VIDEO_MODEL, "model-prefix": "h3", + "precision": "bf16", "framework": "sglang", "runner": "cluster:fixture", + "multinode": False, "srt-recipe-dir": "fixture", + "scenarios": {"video-generation": [{"spec": VIDEO_SPEC, "set": "release-v1", "search-space": [ + {"gpus": 2, "conc-list": [1], "srt-recipe": "video.yaml"}, + ]}]}, + } # fmt: skip + + +def write(project: Path, master: dict, edited: dict, video: dict) -> None: + (project / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) + (project / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited)) + (project / RECIPES / "video.yaml").write_text(yaml.safe_dump(video)) def git(root: Path, *args: str) -> str: @@ -38,47 +57,54 @@ def git(root: Path, *args: str) -> str: def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, capsys): - shutil.copytree(ROOT / "infx", tmp_path / "infx", ignore=shutil.ignore_patterns("__pycache__")) - (tmp_path / RECIPES).mkdir(parents=True) - write_shared_blocks(tmp_path) - (tmp_path / "configs/amd-master.yaml").write_text("{}\n") - (tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump({ + project = tmp_path / "inferencex-e2e" + shutil.copytree(ROOT / "infx", project / "infx", ignore=shutil.ignore_patterns("__pycache__")) + (project / RECIPES).mkdir(parents=True) + write_shared_blocks(project) + write_video_specs(tmp_path) + (project / "configs/amd-master.yaml").write_text("{}\n") + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ "labels": {"cluster:fixture": ["node-a"]}, "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", "slurm": {"partition": "batch", "exclusive": True}}}, })) # fmt: skip - (tmp_path / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) - write(tmp_path, { + (project / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) + write(project, { "edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2]), - "same": entry("stable.yaml", [8]), "gone": entry("stable.yaml", [2]), - }, single_node_fragment(4)) # fmt: skip + "same": entry("stable.yaml", [8]), "video": video_entry(), "gone": entry("stable.yaml", [2]), + }, single_node_fragment(4), video_fragment()) # fmt: skip + (tmp_path / ".gitignore").write_text("__pycache__/\n") git(tmp_path, "init", "-q") git(tmp_path, "config", "user.name", "Test") git(tmp_path, "config", "user.email", "test@example.com") git(tmp_path, "add", ".") git(tmp_path, "commit", "-qm", "base") git(tmp_path, "tag", "base") - write(tmp_path, { + write(project, { "edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2, 4]), - "same": entry("stable.yaml", [8]), - }, single_node_fragment(4, **{"max-running-requests": 16})) # fmt: skip - monkeypatch.chdir(tmp_path) + "same": entry("stable.yaml", [8]), "video": video_entry(), + }, single_node_fragment(4, **{"max-running-requests": 16}), video_fragment(**{"max-running-requests": "7"})) # fmt: skip + monkeypatch.chdir(project) changed.main(["--base", "base", "--json"]) report = json.loads(capsys.readouterr().out) - assert (report["config-keys"], report["errors"]) == (4, []) + assert (report["config-keys"], report["errors"]) == (5, []) assert [ (change["config-key"], change["base-points"], change["head-points"], [point["conc"] for point in change["added"]], [point["conc"] for point in change["removed"]]) for change in report["changed"] - ] == [("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), ("gone", 1, 0, [], [2])] # fmt: skip + ] == [ + ("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), + ("video", 1, 1, [1], [1]), ("gone", 1, 0, [], [2]), + ] # fmt: skip assert git(tmp_path, "status", "--porcelain").splitlines() == [ - f" M {RECIPES}/edited.yaml", " M configs/nvidia-master.yaml", + f" M inferencex-e2e/{RECIPES}/edited.yaml", f" M inferencex-e2e/{RECIPES}/video.yaml", + " M inferencex-e2e/configs/nvidia-master.yaml", ] changed.main(["--base", "base"]) - assert capsys.readouterr().out.splitlines()[-1] == "3 of 4 config keys changed since base." + assert capsys.readouterr().out.splitlines()[-1] == "4 of 5 config keys changed since base." def test_rows_without_fingerprints_are_compared_individually(tmp_path): diff --git a/inferencex-e2e/infx/tests/srt_recipes.py b/inferencex-e2e/infx/tests/srt_recipes.py index c01f9cc60c..5a7c902278 100644 --- a/inferencex-e2e/infx/tests/srt_recipes.py +++ b/inferencex-e2e/infx/tests/srt_recipes.py @@ -1,11 +1,18 @@ """Launchable srt-slurm recipes for tests that plan or fingerprint matrix rows.""" +import shutil from pathlib import Path import yaml +from infx.matrix.validation import VIDEO_SPEC_DIR from infx.srt_slurm.workload import NATIVE_TELEMETRY_BLOCK, SHARED_BLOCKS, TELEMETRY_BLOCK, VIDEO_BLOCK +ROOT = Path(__file__).resolve().parents[2] +VIDEO_MODEL = "MiniMaxAI/MiniMax-H3" +VIDEO_SPEC = "h3-v2.yaml" +VIDEO_PORT = "30283" + def shared_blocks() -> dict[str, str]: """Each lane's shared block (its benchmark client) and the DCGM telemetry block, by path.""" @@ -21,9 +28,13 @@ def shared_blocks() -> dict[str, str]: blocks[NATIVE_TELEMETRY_BLOCK.as_posix()] = yaml.safe_dump( {"telemetry": {"enabled": True, "required": False}} ) - blocks[VIDEO_BLOCK.as_posix()] = yaml.safe_dump( - {"frontend": {"type": "none"}, "benchmark": {"type": "custom", "command": "python3 -m infx.bench video client"}} - ) + blocks[VIDEO_BLOCK.as_posix()] = yaml.safe_dump({ + "frontend": {"type": "none"}, + "benchmark": { + "type": "custom", "command": "python3 -m infx.bench video client --request-seconds 60", + "env": {"VIDEO_SERVER_SERVICE": "video-server", "VIDEO_SERVER_PORT": VIDEO_PORT}, + }, + }) # fmt: skip return blocks @@ -33,8 +44,33 @@ def write_shared_blocks(project: Path) -> None: (project / path).write_text(text) +def write_video_specs(repository: Path) -> None: + """Copy the workload spec the video binder loads beside ``repository``'s project.""" + specs = repository / VIDEO_SPEC_DIR + specs.mkdir(parents=True, exist_ok=True) + for name in ("video_spec.py", VIDEO_SPEC): + shutil.copy(ROOT.parent / VIDEO_SPEC_DIR / name, specs / name) + + def single_node_fragment(tp: int, **args) -> dict: """A single-node sglang fragment serving TP ``tp`` points without speculation.""" return {"schema": 2, "engine": "sglang", "roles": {"agg": { "nodes": 1, "workers": 1, "gpus": tp, "args": {"tensor-parallel-size": tp, **args}, }}} # fmt: skip + + +def video_fragment(**args: str) -> dict: + """A native sglang video fragment the binder accepts, with extra engine ``args`` appended.""" + return {"schema": 2, "name": "video-fixture", "resources": {"gpu_type": "fixture"}, "services": [{ + "name": "video-server", "type": "generic", "nodes": 1, "critical": True, + "command": ["python3", "-m", "infx.bench", "video", "serve"], + "args": [ + "--model-view", "snapshot", "--", "python3", "-m", "evaluator.srt_selected", "serve", + "--model-path", "{model_path}", "--model-id", "{model_id}", + "--revision", "{model_revision}", "--num-gpus", "{gpus_per_node}", + "--disable-conditioning-cache", "--host", "0.0.0.0", "--port", VIDEO_PORT, + *(f"--{flag}={value}" for flag, value in args.items()), + ], + "env": {"VIDEO_SERVER_PORT": VIDEO_PORT, "VIDEO_RUNTIME_REVISION": "b" * 40}, + "readiness": {"http": {"port": int(VIDEO_PORT), "path": "/health"}}, + }]} # fmt: skip diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index d5751eecba..3c7baea3a9 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -9,6 +9,13 @@ from infx.cli import main from infx.launch.drivers.srt import lanes from infx.launch.policy import LaunchPath +from infx.tests.srt_recipes import ( + VIDEO_MODEL, + VIDEO_SPEC, + video_fragment, + write_shared_blocks, + write_video_specs, +) ROOT = Path(__file__).resolve().parents[3] FRAGMENT = { @@ -51,6 +58,14 @@ "power": True, }]}]}, }, + "video": { + "image": "org/video:1", "model": VIDEO_MODEL, "model-prefix": "h3", "precision": "bf16", + "framework": "sglang", "runner": "cluster:fixture", "multinode": False, + "srt-recipe-dir": "fixture", "scenarios": {"video-generation": [{ + "spec": VIDEO_SPEC, "set": "release-v1", + "search-space": [{"gpus": 2, "conc-list": [1], "srt-recipe": "video.yaml"}], + }]}, + }, } # fmt: skip SHARED = { "telemetry-native.yaml": {"telemetry": {"enabled": True, "required": False}}, @@ -64,11 +79,14 @@ @pytest.fixture def project(tmp_path): + """The end-to-end project beside the video harness, as the repository lays them out.""" + project = tmp_path / "inferencex-e2e" + write_shared_blocks(project) + write_video_specs(tmp_path) for name, block in SHARED.items(): - (tmp_path / "configs/srt-recipes" / name).parent.mkdir(parents=True, exist_ok=True) - (tmp_path / "configs/srt-recipes" / name).write_text(yaml.safe_dump(block)) - (tmp_path / "configs/master.yaml").write_text(yaml.safe_dump(MASTER)) - (tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump({ + (project / "configs/srt-recipes" / name).write_text(yaml.safe_dump(block)) + (project / "configs/master.yaml").write_text(yaml.safe_dump(MASTER)) + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ "labels": {"fixture": ["fixture_0"], "cluster:fixture": ["fixture_0"]}, "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", "slurm": { "partition": "p", "exclusive": True, @@ -88,6 +106,7 @@ def project(tmp_path): **RECIPE, "frontend": {"type": "sglang", "enable_multiple_frontends": False}, "roles": {"agg": {"nodes": 1, "workers": 1, "gpus": 4, "args": {"tensor-parallel-size": 4}}}, }, + "single_node/srt-slurm-recipes/fixture/video.yaml": video_fragment(), "multi_node/srt-slurm-recipes/fixture/agentic.yaml": { **RECIPE, "frontend": {"type": "sglang-router"}, "roles": { role: {"nodes": 1, "workers": 1, "gpus": 8, "args": {"tensor-parallel-size": 8}} @@ -96,11 +115,11 @@ def project(tmp_path): }, } # fmt: skip for path, data in recipes.items(): - (tmp_path / "benchmarks" / path).parent.mkdir(parents=True, exist_ok=True) - (tmp_path / "benchmarks" / path).write_text(yaml.safe_dump(data)) - (tmp_path / "utils/srt-slurm").mkdir(parents=True) - (tmp_path / "utils/srt-slurm/src").symlink_to(ROOT / "utils/srt-slurm/src") - return tmp_path + (project / "benchmarks" / path).parent.mkdir(parents=True, exist_ok=True) + (project / "benchmarks" / path).write_text(yaml.safe_dump(data)) + (project / "utils/srt-slurm").mkdir(parents=True) + (project / "utils/srt-slurm/src").symlink_to(ROOT / "utils/srt-slurm/src") + return project @pytest.fixture @@ -314,6 +333,21 @@ def test_output_must_be_new_or_empty(project, tmp_path, capsys): assert "Output directory must be empty" in capsys.readouterr().err +def test_a_video_point_renders_its_bound_server_recipe(project, tmp_path): + output = tmp_path / "out" + assert generate(project, output, "video") == 0 + + [(record, recipe)] = written(output) + assert (record["variant"], record["cluster"]) == (None, "fixture") + assert (record["matrix"]["scenario-type"], record["matrix"]["conc-list"]) == ("video-generation", [1]) + assert recipe["model"] == {"path": f"hf:{VIDEO_MODEL}", "container": "org/video:1", "precision": "bf16"} + assert recipe["resources"]["gpus_per_node"] == 2 + [server] = recipe["services"] + assert server["args"][server["args"].index("--num-gpus") + 1] == "2" + assert server["env"]["VIDEO_SPEC_PATH"] == f"/video-package/specs/{VIDEO_SPEC}" + assert recipe["benchmark"]["env"]["CONC_LIST"] == "1" + + def test_a_recipe_srtctl_rejects_fails_without_writing(project, tmp_path, capsys): recipe = project / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" recipe.write_text(yaml.safe_dump({**FRAGMENT, "base": {**FRAGMENT["base"], "no_such_field": 1}})) From 397254f9a6127d0861f5ec48775d44fd237b4f00 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 7 Nov 2026 01:29:26 -0800 Subject: [PATCH 2/5] fix: include video scenarios in change detection and recipe rendering --- inferencex-e2e/infx/matrix/changed.py | 22 +++++---- inferencex-e2e/infx/matrix/generate.py | 5 +- inferencex-e2e/infx/matrix/plan.py | 2 +- inferencex-e2e/infx/srt_slurm/generate.py | 6 ++- .../infx/workflows/signoff_facts.py | 47 ++++++------------- 5 files changed, 36 insertions(+), 46 deletions(-) diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index 329b9ccb4e..c8adf98863 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -14,7 +14,7 @@ from infx.config import MASTER_CONFIGS, RUNNER_CONFIG from .fingerprint import recipe_fingerprint, row_fingerprint -from .generate import generate_config_matrix +from .generate import SCENARIO_TYPES, generate_config_matrix from .revision import snapshot from .validation import config_root, load_config_files, load_runner_file @@ -43,17 +43,20 @@ def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: return points, errors +def config_points(key: str, master: dict, runners: dict, root: Path) -> tuple[dict, list[str]]: + """Every point of config ``key``, across the scenario types a changelog entry can select.""" + rows = generate_config_matrix( + [key], master, runners, scenario_types=SCENARIO_TYPES, eval_mode="none", root=root + ) + return _points(rows, root) + + def head_matrix(config_files: list[str], runner_config: str) -> Matrix: """Generate the working tree's matrix for each config key, as points and fingerprint errors.""" master = load_config_files(config_files) runners = load_runner_file(runner_config) root = config_root(config_files) - return { - key: _points( - generate_config_matrix([key], master, runners, eval_mode="none", root=root), root - ) - for key in master - } + return {key: config_points(key, master, runners, root) for key in master} def base_matrix(ref: str, config_files: list[str]) -> Matrix: @@ -68,9 +71,8 @@ def base_matrix(ref: str, config_files: list[str]) -> Matrix: single.write_text(yaml.safe_dump({key: entry}, sort_keys=False)) singles.append(str(single)) - requests = [ - ([key], ["--no-evals"], [single]) for key, single in zip(master, singles, strict=True) - ] + flags = ["--no-evals", "--scenario-type", *SCENARIO_TYPES] + requests = [([key], flags, [single]) for key, single in zip(master, singles, strict=True)] rows = revision.generate_each(requests, os.cpu_count() or 1) return { key: _points(generated, revision.root) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 52f4aaad53..8883945c37 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -30,6 +30,9 @@ seq_len_stoi = {"1k1k": (1024, 1024), "8k1k": (8192, 1024)} +# Every scenario type a changelog entry can select; unscoped generation leaves video out. +SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding", "video-generation") + MIN_EVAL_CONC = 16 DEFAULT_EVAL_FRAMEWORK = "lm-eval" AUTOMATIC_AGENTIC_VENDOR_EVALS = { @@ -1797,7 +1800,7 @@ def main() -> list[dict]: parent_parser.add_argument( "--scenario-type", nargs="+", - choices=["fixed-seq-len", "agentic-coding", "video-generation"], + choices=list(SCENARIO_TYPES), required=False, help="Scenario type(s) to include. If not specified, fixed-seq-len and agentic-coding are generated; video-generation runs only when named.", ) diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index f03b731a45..23ace5ed8a 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -11,6 +11,7 @@ from .fingerprint import recipe_fingerprint from .generate import ( + SCENARIO_TYPES, EvalMode, drop_app_colliding_evals, freeze_config_value, @@ -27,7 +28,6 @@ load_runner_file, ) -SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding", "video-generation") # An entry without scenario-type reruns these; video runs only for entries that name it. DEFAULT_SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding") diff --git a/inferencex-e2e/infx/srt_slurm/generate.py b/inferencex-e2e/infx/srt_slurm/generate.py index b82de73a20..05d9c61df8 100644 --- a/inferencex-e2e/infx/srt_slurm/generate.py +++ b/inferencex-e2e/infx/srt_slurm/generate.py @@ -19,7 +19,7 @@ from infx.launch.drivers.srt import config, lanes, power from infx.launch.policy import launch_path from infx.launch.request import MultiNodeRequest -from infx.matrix.generate import expand_config_keys, generate_config_matrix +from infx.matrix.generate import SCENARIO_TYPES, expand_config_keys, generate_config_matrix from infx.matrix.validation import config_root, load_config_files, load_runner_file from infx.srt_slurm.single_node import select_recipe from infx.srt_slurm.synthetic_acceptance import build_overrides, selected_recipes @@ -292,7 +292,9 @@ def generate_recipes( recipes: dict[str, dict[str, Any]] = {} records = [] for key in expand_config_keys(config_keys, master): - points = generate_config_matrix([key], master, runners, eval_mode="none", root=root) + points = generate_config_matrix( + [key], master, runners, scenario_types=SCENARIO_TYPES, eval_mode="none", root=root + ) points = [point for point in points if point.get("srt-recipe")] if not points: raise ValueError(f"{key} has no srt-slurm points") diff --git a/inferencex-e2e/infx/workflows/signoff_facts.py b/inferencex-e2e/infx/workflows/signoff_facts.py index 22c9ba31a1..5df6f01896 100644 --- a/inferencex-e2e/infx/workflows/signoff_facts.py +++ b/inferencex-e2e/infx/workflows/signoff_facts.py @@ -7,14 +7,12 @@ from __future__ import annotations import argparse -import io import itertools import json import math import re import subprocess import sys -import tarfile import tempfile import urllib.request from collections.abc import Callable @@ -47,11 +45,6 @@ EVAL_JOB = re.compile(r"^(?:(?:single|multi)-node )?(?:agentic )?eval /") RUN_ID = re.compile(r"/actions/runs/([0-9]+)") IMAGE_REF = re.compile(r"""\b(?:image|container):\s*["']?([A-Za-z0-9][\w./#:@+-]*)""") -DATA_PATHS = ( - "inferencex-e2e/configs", - "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes", - "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes", -) SIGMAS = 2 @@ -362,32 +355,22 @@ def merge_base(base_ref: str, sha: str) -> str: return _git("merge-base", base_ref, sha).stdout.decode().strip() -def _extract(rev: str, target: Path) -> Path: - with tarfile.open(fileobj=io.BytesIO(_git("archive", rev, "--", *DATA_PATHS).stdout)) as tar: - tar.extractall(target, filter="data") - return target / "inferencex-e2e" - - -def matrix_at(rev: str, workdir: Path) -> tuple[dict, dict[str, str]]: +def matrix_at(rev: str) -> tuple[dict, dict[str, str]]: """Generate every config key's points from a revision's config data with trusted code.""" - from infx.matrix.changed import _points - from infx.matrix.generate import generate_config_matrix + from infx.matrix.changed import config_points + from infx.matrix.revision import snapshot from infx.matrix.validation import load_config_files, load_runner_file - project = _extract(rev, workdir) - configs = [ - str(project / "configs" / name) for name in ("amd-master.yaml", "nvidia-master.yaml") - ] - master = load_config_files([path for path in configs if Path(path).is_file()], validate=False) - runners = load_runner_file(str(project / "configs" / "runners.yaml"), validate=False) - matrix, errors = {}, {} - for key in master: - try: - rows = generate_config_matrix([key], master, runners, eval_mode="none", root=project) - matrix[key] = _points(rows, project) - except Exception as error: # noqa: BLE001 - one malformed key must not hide the others - errors[key] = str(error)[:300] - return matrix, errors + with snapshot(rev) as revision: + master = load_config_files(revision.master_configs, validate=False) + runners = load_runner_file(revision.runner_config, validate=False) + matrix, errors = {}, {} + for key in master: + try: + matrix[key] = config_points(key, master, runners, revision.root) + except Exception as error: # noqa: BLE001 - one malformed key must not hide the others + errors[key] = str(error)[:300] + return matrix, errors def public_model_name(point: dict) -> str: @@ -733,8 +716,8 @@ def gaps() -> dict[str, Any]: report["check_2_eval_gaps"] = _section(gaps) def diff() -> dict[str, Any]: - before, before_errors = matrix_at(base, workdir / "base") - after, after_errors = matrix_at(sha, workdir / "head") + before, before_errors = matrix_at(base) + after, after_errors = matrix_at(sha) ingested, unavailable = [], {} for prefix, checkpoint in changed_models(before, after): try: From 1ca51dc0fc4e572b722b2904ea9074f838ccfa9a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 7 Nov 2026 01:54:06 -0800 Subject: [PATCH 3/5] test: write a fixture video spec instead of copying h3-v2.yaml --- .../infx/tests/matrix/test_changed.py | 14 +------- inferencex-e2e/infx/tests/srt_recipes.py | 33 ++++++++++++++++--- .../infx/tests/srt_slurm/test_generate.py | 12 ++----- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index 0c69a72087..e8453fe0e7 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -10,9 +10,8 @@ from infx.matrix import changed from infx.tests.srt_recipes import ( - VIDEO_MODEL, - VIDEO_SPEC, single_node_fragment, + video_entry, video_fragment, write_shared_blocks, write_video_specs, @@ -33,17 +32,6 @@ def entry(recipe: str, concurrencies: list[int]) -> dict: } # fmt: skip -def video_entry() -> dict: - return { - "image": "example/video:1", "model": VIDEO_MODEL, "model-prefix": "h3", - "precision": "bf16", "framework": "sglang", "runner": "cluster:fixture", - "multinode": False, "srt-recipe-dir": "fixture", - "scenarios": {"video-generation": [{"spec": VIDEO_SPEC, "set": "release-v1", "search-space": [ - {"gpus": 2, "conc-list": [1], "srt-recipe": "video.yaml"}, - ]}]}, - } # fmt: skip - - def write(project: Path, master: dict, edited: dict, video: dict) -> None: (project / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) (project / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited)) diff --git a/inferencex-e2e/infx/tests/srt_recipes.py b/inferencex-e2e/infx/tests/srt_recipes.py index 5a7c902278..f2f25b5e3e 100644 --- a/inferencex-e2e/infx/tests/srt_recipes.py +++ b/inferencex-e2e/infx/tests/srt_recipes.py @@ -9,8 +9,9 @@ from infx.srt_slurm.workload import NATIVE_TELEMETRY_BLOCK, SHARED_BLOCKS, TELEMETRY_BLOCK, VIDEO_BLOCK ROOT = Path(__file__).resolve().parents[2] -VIDEO_MODEL = "MiniMaxAI/MiniMax-H3" -VIDEO_SPEC = "h3-v2.yaml" +VIDEO_MODEL = "example/video-model" +VIDEO_SPEC = "fixture.yaml" +VIDEO_SET = "fixture-set" VIDEO_PORT = "30283" @@ -45,11 +46,33 @@ def write_shared_blocks(project: Path) -> None: def write_video_specs(repository: Path) -> None: - """Copy the workload spec the video binder loads beside ``repository``'s project.""" + """Put the workload helper and a one-case spec serving ``VIDEO_MODEL`` beside the project.""" specs = repository / VIDEO_SPEC_DIR specs.mkdir(parents=True, exist_ok=True) - for name in ("video_spec.py", VIDEO_SPEC): - shutil.copy(ROOT.parent / VIDEO_SPEC_DIR / name, specs / name) + shutil.copy(ROOT.parent / VIDEO_SPEC_DIR / "video_spec.py", specs / "video_spec.py") + (specs / VIDEO_SPEC).write_text(yaml.safe_dump({ + "schema_version": 1, + "model": {"id": VIDEO_MODEL, "revision": "a" * 40}, + "generation": {"num_inference_steps": 1}, + "prompts": {"fixture": {"text": "a fixture prompt", "requires_motion": False, "requires_sound": False}}, + "sets": {VIDEO_SET: { + "plan_id": "fixture-plan", "repetitions": 1, "warmup_runs": 0, + "cases": [{"case_id": "fixture", "prompt": "fixture", "seed": 1}], + }}, + "reference": {}, + })) # fmt: skip + + +def video_entry(recipe: str = "video.yaml") -> dict: + """A master entry with one 2-GPU video point of ``VIDEO_SPEC``'s set served by ``recipe``.""" + return { + "image": "example/video:1", "model": VIDEO_MODEL, "model-prefix": "h3", + "precision": "bf16", "framework": "sglang", "runner": "cluster:fixture", + "multinode": False, "srt-recipe-dir": "fixture", + "scenarios": {"video-generation": [{"spec": VIDEO_SPEC, "set": VIDEO_SET, "search-space": [ + {"gpus": 2, "conc-list": [1], "srt-recipe": recipe}, + ]}]}, + } # fmt: skip def single_node_fragment(tp: int, **args) -> dict: diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index 3c7baea3a9..d301b08b40 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -12,6 +12,7 @@ from infx.tests.srt_recipes import ( VIDEO_MODEL, VIDEO_SPEC, + video_entry, video_fragment, write_shared_blocks, write_video_specs, @@ -58,14 +59,7 @@ "power": True, }]}]}, }, - "video": { - "image": "org/video:1", "model": VIDEO_MODEL, "model-prefix": "h3", "precision": "bf16", - "framework": "sglang", "runner": "cluster:fixture", "multinode": False, - "srt-recipe-dir": "fixture", "scenarios": {"video-generation": [{ - "spec": VIDEO_SPEC, "set": "release-v1", - "search-space": [{"gpus": 2, "conc-list": [1], "srt-recipe": "video.yaml"}], - }]}, - }, + "video": video_entry(), } # fmt: skip SHARED = { "telemetry-native.yaml": {"telemetry": {"enabled": True, "required": False}}, @@ -340,7 +334,7 @@ def test_a_video_point_renders_its_bound_server_recipe(project, tmp_path): [(record, recipe)] = written(output) assert (record["variant"], record["cluster"]) == (None, "fixture") assert (record["matrix"]["scenario-type"], record["matrix"]["conc-list"]) == ("video-generation", [1]) - assert recipe["model"] == {"path": f"hf:{VIDEO_MODEL}", "container": "org/video:1", "precision": "bf16"} + assert recipe["model"] == {"path": f"hf:{VIDEO_MODEL}", "container": "example/video:1", "precision": "bf16"} assert recipe["resources"]["gpus_per_node"] == 2 [server] = recipe["services"] assert server["args"][server["args"].index("--num-gpus") + 1] == "2" From 0ffab9a9cafd103d38e2cf7c337ca941ea8ef338 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 7 Nov 2026 01:56:27 -0800 Subject: [PATCH 4/5] fix: keep sign-off matrix generation LLM-only --- .../infx/tests/matrix/test_changed.py | 25 ++++------ inferencex-e2e/infx/tests/srt_recipes.py | 12 +++++ .../tests/workflows/test_signoff_facts.py | 49 +++++++++++++++++++ .../infx/workflows/signoff_facts.py | 47 ++++++++++++------ 4 files changed, 103 insertions(+), 30 deletions(-) diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index e8453fe0e7..7d6b133633 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -10,6 +10,7 @@ from infx.matrix import changed from infx.tests.srt_recipes import ( + fixed_seq_entry, single_node_fragment, video_entry, video_fragment, @@ -21,17 +22,6 @@ RECIPES = "benchmarks/single_node/srt-slurm-recipes/fixture" -def entry(recipe: str, concurrencies: list[int]) -> dict: - return { - "image": "example/image:1", "model": "org/model", "model-prefix": "dsr1", - "precision": "fp8", "framework": "sglang", "runner": "cluster:fixture", - "multinode": False, "srt-recipe-dir": "fixture", - "scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [ - {"tp": 4, "conc-list": concurrencies, "srt-recipe": recipe}, - ]}]}, - } # fmt: skip - - def write(project: Path, master: dict, edited: dict, video: dict) -> None: (project / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) (project / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited)) @@ -58,8 +48,11 @@ def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, c })) # fmt: skip (project / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) write(project, { - "edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2]), - "same": entry("stable.yaml", [8]), "video": video_entry(), "gone": entry("stable.yaml", [2]), + "edited": fixed_seq_entry("edited.yaml", [2, 4]), + "grown": fixed_seq_entry("stable.yaml", [2]), + "same": fixed_seq_entry("stable.yaml", [8]), + "video": video_entry(), + "gone": fixed_seq_entry("stable.yaml", [2]), }, single_node_fragment(4), video_fragment()) # fmt: skip (tmp_path / ".gitignore").write_text("__pycache__/\n") git(tmp_path, "init", "-q") @@ -69,8 +62,10 @@ def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, c git(tmp_path, "commit", "-qm", "base") git(tmp_path, "tag", "base") write(project, { - "edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2, 4]), - "same": entry("stable.yaml", [8]), "video": video_entry(), + "edited": fixed_seq_entry("edited.yaml", [2, 4]), + "grown": fixed_seq_entry("stable.yaml", [2, 4]), + "same": fixed_seq_entry("stable.yaml", [8]), + "video": video_entry(), }, single_node_fragment(4, **{"max-running-requests": 16}), video_fragment(**{"max-running-requests": "7"})) # fmt: skip monkeypatch.chdir(project) diff --git a/inferencex-e2e/infx/tests/srt_recipes.py b/inferencex-e2e/infx/tests/srt_recipes.py index f2f25b5e3e..fa954a8e6d 100644 --- a/inferencex-e2e/infx/tests/srt_recipes.py +++ b/inferencex-e2e/infx/tests/srt_recipes.py @@ -63,6 +63,18 @@ def write_video_specs(repository: Path) -> None: })) # fmt: skip +def fixed_seq_entry(recipe: str, concurrencies: list[int]) -> dict: + """A master entry with one TP4 fixed-sequence point per concurrency, served by ``recipe``.""" + return { + "image": "example/image:1", "model": "org/model", "model-prefix": "dsr1", + "precision": "fp8", "framework": "sglang", "runner": "cluster:fixture", + "multinode": False, "srt-recipe-dir": "fixture", + "scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [ + {"tp": 4, "conc-list": concurrencies, "srt-recipe": recipe}, + ]}]}, + } # fmt: skip + + def video_entry(recipe: str = "video.yaml") -> dict: """A master entry with one 2-GPU video point of ``VIDEO_SPEC``'s set served by ``recipe``.""" return { diff --git a/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py b/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py index 21c154d75b..33826373c3 100644 --- a/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py +++ b/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py @@ -1,10 +1,21 @@ """Exercise the verifier's deterministic facts against controlled inputs.""" import json +import subprocess import pytest +import yaml from infx.klaud.models import Feed +from infx.matrix.validation import VIDEO_SPEC_DIR +from infx.tests.srt_recipes import ( + VIDEO_SPEC, + fixed_seq_entry, + single_node_fragment, + video_entry, + video_fragment, + write_shared_blocks, +) from infx.workflows import signoff_facts SERIES = ["--model", "m", "--precision", "fp4", "--framework", "sglang", "--hardware", "b200"] @@ -268,3 +279,41 @@ def test_eval_gap_estimates_zero_standard_errors_from_sample_size(): def test_digest_pins_are_inspected_by_digest(monkeypatch, image, inspected): monkeypatch.setattr(signoff_facts, "_inspect", lambda reference: {"inspected": reference}) assert signoff_facts.inspect_image(image)["inspected"] == inspected + + +def test_matrix_at_generates_llm_points_only_and_runs_no_revision_code(tmp_path, monkeypatch): + project = tmp_path / "inferencex-e2e" + recipes = project / "benchmarks/single_node/srt-slurm-recipes/fixture" + write_shared_blocks(project) + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ + "labels": {"cluster:fixture": ["node-a"]}, + "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", + "slurm": {"partition": "batch", "exclusive": True}}}, + })) # fmt: skip + (project / "configs/nvidia-master.yaml").write_text( + yaml.safe_dump({"llm": fixed_seq_entry("stable.yaml", [2]), "video": video_entry()}) + ) + (project / "configs/amd-master.yaml").write_text("{}\n") + recipes.mkdir(parents=True) + (recipes / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) + (recipes / "video.yaml").write_text(yaml.safe_dump(video_fragment())) + # git archive rejects a DATA_PATHS entry that the tree lacks. + (project / "benchmarks/multi_node/srt-slurm-recipes").mkdir(parents=True) + (project / "benchmarks/multi_node/srt-slurm-recipes/.keep").touch() + specs = tmp_path / VIDEO_SPEC_DIR + specs.mkdir(parents=True) + (specs / "video_spec.py").write_text('raise RuntimeError("video_spec.py from the revision tree ran")\n') + (specs / VIDEO_SPEC).write_text("{}\n") + for args in (["init", "-q"], ["add", "."], ["commit", "-qm", "base"]): + subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], + cwd=tmp_path, check=True, capture_output=True, + ) # fmt: skip + monkeypatch.chdir(tmp_path) + + matrix, errors = signoff_facts.matrix_at("HEAD", tmp_path / "work") + + assert errors == {} + assert { + key: ([conc for _, conc in points], messages) for key, (points, messages) in matrix.items() + } == {"llm": ([2], []), "video": ([], [])} diff --git a/inferencex-e2e/infx/workflows/signoff_facts.py b/inferencex-e2e/infx/workflows/signoff_facts.py index 5df6f01896..22c9ba31a1 100644 --- a/inferencex-e2e/infx/workflows/signoff_facts.py +++ b/inferencex-e2e/infx/workflows/signoff_facts.py @@ -7,12 +7,14 @@ from __future__ import annotations import argparse +import io import itertools import json import math import re import subprocess import sys +import tarfile import tempfile import urllib.request from collections.abc import Callable @@ -45,6 +47,11 @@ EVAL_JOB = re.compile(r"^(?:(?:single|multi)-node )?(?:agentic )?eval /") RUN_ID = re.compile(r"/actions/runs/([0-9]+)") IMAGE_REF = re.compile(r"""\b(?:image|container):\s*["']?([A-Za-z0-9][\w./#:@+-]*)""") +DATA_PATHS = ( + "inferencex-e2e/configs", + "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes", + "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes", +) SIGMAS = 2 @@ -355,22 +362,32 @@ def merge_base(base_ref: str, sha: str) -> str: return _git("merge-base", base_ref, sha).stdout.decode().strip() -def matrix_at(rev: str) -> tuple[dict, dict[str, str]]: +def _extract(rev: str, target: Path) -> Path: + with tarfile.open(fileobj=io.BytesIO(_git("archive", rev, "--", *DATA_PATHS).stdout)) as tar: + tar.extractall(target, filter="data") + return target / "inferencex-e2e" + + +def matrix_at(rev: str, workdir: Path) -> tuple[dict, dict[str, str]]: """Generate every config key's points from a revision's config data with trusted code.""" - from infx.matrix.changed import config_points - from infx.matrix.revision import snapshot + from infx.matrix.changed import _points + from infx.matrix.generate import generate_config_matrix from infx.matrix.validation import load_config_files, load_runner_file - with snapshot(rev) as revision: - master = load_config_files(revision.master_configs, validate=False) - runners = load_runner_file(revision.runner_config, validate=False) - matrix, errors = {}, {} - for key in master: - try: - matrix[key] = config_points(key, master, runners, revision.root) - except Exception as error: # noqa: BLE001 - one malformed key must not hide the others - errors[key] = str(error)[:300] - return matrix, errors + project = _extract(rev, workdir) + configs = [ + str(project / "configs" / name) for name in ("amd-master.yaml", "nvidia-master.yaml") + ] + master = load_config_files([path for path in configs if Path(path).is_file()], validate=False) + runners = load_runner_file(str(project / "configs" / "runners.yaml"), validate=False) + matrix, errors = {}, {} + for key in master: + try: + rows = generate_config_matrix([key], master, runners, eval_mode="none", root=project) + matrix[key] = _points(rows, project) + except Exception as error: # noqa: BLE001 - one malformed key must not hide the others + errors[key] = str(error)[:300] + return matrix, errors def public_model_name(point: dict) -> str: @@ -716,8 +733,8 @@ def gaps() -> dict[str, Any]: report["check_2_eval_gaps"] = _section(gaps) def diff() -> dict[str, Any]: - before, before_errors = matrix_at(base) - after, after_errors = matrix_at(sha) + before, before_errors = matrix_at(base, workdir / "base") + after, after_errors = matrix_at(sha, workdir / "head") ingested, unavailable = [], {} for prefix, checkpoint in changed_models(before, after): try: From eb50d8bf4483527a25a5fabfdc8e7a1929a8a168 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 7 Nov 2026 01:59:18 -0800 Subject: [PATCH 5/5] fix: record a base video bind failure per key in changed --- inferencex-e2e/infx/matrix/changed.py | 33 +++++++++++++++---- .../infx/tests/matrix/test_changed.py | 21 +++++++++--- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index c8adf98863..bcb971b536 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -15,7 +15,7 @@ from .fingerprint import recipe_fingerprint, row_fingerprint from .generate import SCENARIO_TYPES, generate_config_matrix -from .revision import snapshot +from .revision import Revision, snapshot from .validation import config_root, load_config_files, load_runner_file Matrix = dict[str, tuple[dict[tuple[str, int], dict[str, Any]], list[str]]] @@ -59,6 +59,19 @@ def head_matrix(config_files: list[str], runner_config: str) -> Matrix: return {key: config_points(key, master, runners, root) for key in master} +def _base_video_points(revision: Revision, key: str, single: str) -> tuple[dict, list[str]]: + """The video points of config ``key`` at ``revision``, or its generator's failure as an error. + + That generator exits when a video recipe cannot be bound, where head records the error per key. + """ + flags = ["--no-evals", "--scenario-type", "video-generation"] + try: + rows = revision.generate([key], flags, [single]) + except subprocess.CalledProcessError as error: + return {}, [error.stderr.strip().rsplit("\n", 1)[-1]] + return _points(rows, revision.root) + + def base_matrix(ref: str, config_files: list[str]) -> Matrix: """Generate the matrix of each config key at git ``ref``, as points and fingerprint errors.""" with snapshot(ref) as revision: @@ -71,13 +84,19 @@ def base_matrix(ref: str, config_files: list[str]) -> Matrix: single.write_text(yaml.safe_dump({key: entry}, sort_keys=False)) singles.append(str(single)) - flags = ["--no-evals", "--scenario-type", *SCENARIO_TYPES] - requests = [([key], flags, [single]) for key, single in zip(master, singles, strict=True)] + requests = [ + ([key], ["--no-evals"], [single]) for key, single in zip(master, singles, strict=True) + ] rows = revision.generate_each(requests, os.cpu_count() or 1) - return { - key: _points(generated, revision.root) - for key, generated in zip(master, rows, strict=True) - } + matrix: Matrix = {} + for (key, entry), single, generated in zip(master.items(), singles, rows, strict=True): + points, errors = _points(generated, revision.root) + if "video-generation" in entry.get("scenarios", {}): + video_points, video_errors = _base_video_points(revision, key, single) + points.update(video_points) + errors.extend(video_errors) + matrix[key] = (points, errors) + return matrix def _sorted(points: Iterable[dict]) -> list[dict]: diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index 7d6b133633..62501403da 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -47,11 +47,15 @@ def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, c "slurm": {"partition": "batch", "exclusive": True}}}, })) # fmt: skip (project / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) + unbound = video_fragment() + del unbound["services"][0]["env"]["VIDEO_RUNTIME_REVISION"] + (project / RECIPES / "unbound.yaml").write_text(yaml.safe_dump(unbound)) write(project, { "edited": fixed_seq_entry("edited.yaml", [2, 4]), "grown": fixed_seq_entry("stable.yaml", [2]), "same": fixed_seq_entry("stable.yaml", [8]), "video": video_entry(), + "unbound": video_entry("unbound.yaml"), "gone": fixed_seq_entry("stable.yaml", [2]), }, single_node_fragment(4), video_fragment()) # fmt: skip (tmp_path / ".gitignore").write_text("__pycache__/\n") @@ -66,28 +70,37 @@ def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, c "grown": fixed_seq_entry("stable.yaml", [2, 4]), "same": fixed_seq_entry("stable.yaml", [8]), "video": video_entry(), + "unbound": video_entry("unbound.yaml"), }, single_node_fragment(4, **{"max-running-requests": 16}), video_fragment(**{"max-running-requests": "7"})) # fmt: skip + (project / RECIPES / "unbound.yaml").write_text(yaml.safe_dump(video_fragment())) monkeypatch.chdir(project) changed.main(["--base", "base", "--json"]) report = json.loads(capsys.readouterr().out) - assert (report["config-keys"], report["errors"]) == (5, []) + assert report["config-keys"] == 6 + [error] = report["errors"] + assert (error["config-key"], error["revision"]) == ("unbound", "base") + assert error["error"].endswith("video service requires a pinned VIDEO_RUNTIME_REVISION") assert [ (change["config-key"], change["base-points"], change["head-points"], [point["conc"] for point in change["added"]], [point["conc"] for point in change["removed"]]) for change in report["changed"] ] == [ - ("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), + ("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), ("unbound", 0, 1, [1], []), ("video", 1, 1, [1], [1]), ("gone", 1, 0, [], [2]), ] # fmt: skip assert git(tmp_path, "status", "--porcelain").splitlines() == [ - f" M inferencex-e2e/{RECIPES}/edited.yaml", f" M inferencex-e2e/{RECIPES}/video.yaml", + f" M inferencex-e2e/{RECIPES}/edited.yaml", + f" M inferencex-e2e/{RECIPES}/unbound.yaml", + f" M inferencex-e2e/{RECIPES}/video.yaml", " M inferencex-e2e/configs/nvidia-master.yaml", ] changed.main(["--base", "base"]) - assert capsys.readouterr().out.splitlines()[-1] == "4 of 5 config keys changed since base." + lines = capsys.readouterr().out.splitlines() + assert lines[-2] == "5 of 6 config keys changed since base." + assert lines[-1].startswith("warning: 1 rows of unbound have no fingerprint at base: ") def test_rows_without_fingerprints_are_compared_individually(tmp_path):