diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index 329b9ccb4e..bcb971b536 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -14,8 +14,8 @@ from infx.config import MASTER_CONFIGS, RUNNER_CONFIG from .fingerprint import recipe_fingerprint, row_fingerprint -from .generate import generate_config_matrix -from .revision import snapshot +from .generate import SCENARIO_TYPES, generate_config_matrix +from .revision import Revision, snapshot from .validation import config_root, load_config_files, load_runner_file Matrix = dict[str, tuple[dict[tuple[str, int], dict[str, Any]], list[str]]] @@ -43,17 +43,33 @@ def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: return points, errors +def config_points(key: str, master: dict, runners: dict, root: Path) -> tuple[dict, list[str]]: + """Every point of config ``key``, across the scenario types a changelog entry can select.""" + rows = generate_config_matrix( + [key], master, runners, scenario_types=SCENARIO_TYPES, eval_mode="none", root=root + ) + return _points(rows, root) + + def head_matrix(config_files: list[str], runner_config: str) -> Matrix: """Generate the working tree's matrix for each config key, as points and fingerprint errors.""" master = load_config_files(config_files) runners = load_runner_file(runner_config) root = config_root(config_files) - return { - key: _points( - generate_config_matrix([key], master, runners, eval_mode="none", root=root), root - ) - for key in master - } + return {key: config_points(key, master, runners, root) for key in master} + + +def _base_video_points(revision: Revision, key: str, single: str) -> tuple[dict, list[str]]: + """The video points of config ``key`` at ``revision``, or its generator's failure as an error. + + That generator exits when a video recipe cannot be bound, where head records the error per key. + """ + flags = ["--no-evals", "--scenario-type", "video-generation"] + try: + rows = revision.generate([key], flags, [single]) + except subprocess.CalledProcessError as error: + return {}, [error.stderr.strip().rsplit("\n", 1)[-1]] + return _points(rows, revision.root) def base_matrix(ref: str, config_files: list[str]) -> Matrix: @@ -72,10 +88,15 @@ def base_matrix(ref: str, config_files: list[str]) -> Matrix: ([key], ["--no-evals"], [single]) for key, single in zip(master, singles, strict=True) ] rows = revision.generate_each(requests, os.cpu_count() or 1) - return { - key: _points(generated, revision.root) - for key, generated in zip(master, rows, strict=True) - } + matrix: Matrix = {} + for (key, entry), single, generated in zip(master.items(), singles, rows, strict=True): + points, errors = _points(generated, revision.root) + if "video-generation" in entry.get("scenarios", {}): + video_points, video_errors = _base_video_points(revision, key, single) + points.update(video_points) + errors.extend(video_errors) + matrix[key] = (points, errors) + return matrix def _sorted(points: Iterable[dict]) -> list[dict]: diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 52f4aaad53..8883945c37 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -30,6 +30,9 @@ seq_len_stoi = {"1k1k": (1024, 1024), "8k1k": (8192, 1024)} +# Every scenario type a changelog entry can select; unscoped generation leaves video out. +SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding", "video-generation") + MIN_EVAL_CONC = 16 DEFAULT_EVAL_FRAMEWORK = "lm-eval" AUTOMATIC_AGENTIC_VENDOR_EVALS = { @@ -1797,7 +1800,7 @@ def main() -> list[dict]: parent_parser.add_argument( "--scenario-type", nargs="+", - choices=["fixed-seq-len", "agentic-coding", "video-generation"], + choices=list(SCENARIO_TYPES), required=False, help="Scenario type(s) to include. If not specified, fixed-seq-len and agentic-coding are generated; video-generation runs only when named.", ) diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index f03b731a45..23ace5ed8a 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -11,6 +11,7 @@ from .fingerprint import recipe_fingerprint from .generate import ( + SCENARIO_TYPES, EvalMode, drop_app_colliding_evals, freeze_config_value, @@ -27,7 +28,6 @@ load_runner_file, ) -SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding", "video-generation") # An entry without scenario-type reruns these; video runs only for entries that name it. DEFAULT_SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding") diff --git a/inferencex-e2e/infx/srt_slurm/generate.py b/inferencex-e2e/infx/srt_slurm/generate.py index b82de73a20..05d9c61df8 100644 --- a/inferencex-e2e/infx/srt_slurm/generate.py +++ b/inferencex-e2e/infx/srt_slurm/generate.py @@ -19,7 +19,7 @@ from infx.launch.drivers.srt import config, lanes, power from infx.launch.policy import launch_path from infx.launch.request import MultiNodeRequest -from infx.matrix.generate import expand_config_keys, generate_config_matrix +from infx.matrix.generate import SCENARIO_TYPES, expand_config_keys, generate_config_matrix from infx.matrix.validation import config_root, load_config_files, load_runner_file from infx.srt_slurm.single_node import select_recipe from infx.srt_slurm.synthetic_acceptance import build_overrides, selected_recipes @@ -292,7 +292,9 @@ def generate_recipes( recipes: dict[str, dict[str, Any]] = {} records = [] for key in expand_config_keys(config_keys, master): - points = generate_config_matrix([key], master, runners, eval_mode="none", root=root) + points = generate_config_matrix( + [key], master, runners, scenario_types=SCENARIO_TYPES, eval_mode="none", root=root + ) points = [point for point in points if point.get("srt-recipe")] if not points: raise ValueError(f"{key} has no srt-slurm points") diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index 14a09273ed..62501403da 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -9,26 +9,23 @@ import yaml from infx.matrix import changed -from infx.tests.srt_recipes import single_node_fragment, write_shared_blocks +from infx.tests.srt_recipes import ( + fixed_seq_entry, + single_node_fragment, + video_entry, + video_fragment, + write_shared_blocks, + write_video_specs, +) ROOT = Path(__file__).resolve().parents[3] RECIPES = "benchmarks/single_node/srt-slurm-recipes/fixture" -def entry(recipe: str, concurrencies: list[int]) -> dict: - return { - "image": "example/image:1", "model": "org/model", "model-prefix": "dsr1", - "precision": "fp8", "framework": "sglang", "runner": "cluster:fixture", - "multinode": False, "srt-recipe-dir": "fixture", - "scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [ - {"tp": 4, "conc-list": concurrencies, "srt-recipe": recipe}, - ]}]}, - } # fmt: skip - - -def write(root: Path, master: dict, edited: dict) -> None: - (root / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) - (root / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited)) +def write(project: Path, master: dict, edited: dict, video: dict) -> None: + (project / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master)) + (project / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited)) + (project / RECIPES / "video.yaml").write_text(yaml.safe_dump(video)) def git(root: Path, *args: str) -> str: @@ -38,47 +35,72 @@ def git(root: Path, *args: str) -> str: def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, capsys): - shutil.copytree(ROOT / "infx", tmp_path / "infx", ignore=shutil.ignore_patterns("__pycache__")) - (tmp_path / RECIPES).mkdir(parents=True) - write_shared_blocks(tmp_path) - (tmp_path / "configs/amd-master.yaml").write_text("{}\n") - (tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump({ + project = tmp_path / "inferencex-e2e" + shutil.copytree(ROOT / "infx", project / "infx", ignore=shutil.ignore_patterns("__pycache__")) + (project / RECIPES).mkdir(parents=True) + write_shared_blocks(project) + write_video_specs(tmp_path) + (project / "configs/amd-master.yaml").write_text("{}\n") + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ "labels": {"cluster:fixture": ["node-a"]}, "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", "slurm": {"partition": "batch", "exclusive": True}}}, })) # fmt: skip - (tmp_path / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) - write(tmp_path, { - "edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2]), - "same": entry("stable.yaml", [8]), "gone": entry("stable.yaml", [2]), - }, single_node_fragment(4)) # fmt: skip + (project / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) + unbound = video_fragment() + del unbound["services"][0]["env"]["VIDEO_RUNTIME_REVISION"] + (project / RECIPES / "unbound.yaml").write_text(yaml.safe_dump(unbound)) + write(project, { + "edited": fixed_seq_entry("edited.yaml", [2, 4]), + "grown": fixed_seq_entry("stable.yaml", [2]), + "same": fixed_seq_entry("stable.yaml", [8]), + "video": video_entry(), + "unbound": video_entry("unbound.yaml"), + "gone": fixed_seq_entry("stable.yaml", [2]), + }, single_node_fragment(4), video_fragment()) # fmt: skip + (tmp_path / ".gitignore").write_text("__pycache__/\n") git(tmp_path, "init", "-q") git(tmp_path, "config", "user.name", "Test") git(tmp_path, "config", "user.email", "test@example.com") git(tmp_path, "add", ".") git(tmp_path, "commit", "-qm", "base") git(tmp_path, "tag", "base") - write(tmp_path, { - "edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2, 4]), - "same": entry("stable.yaml", [8]), - }, single_node_fragment(4, **{"max-running-requests": 16})) # fmt: skip - monkeypatch.chdir(tmp_path) + write(project, { + "edited": fixed_seq_entry("edited.yaml", [2, 4]), + "grown": fixed_seq_entry("stable.yaml", [2, 4]), + "same": fixed_seq_entry("stable.yaml", [8]), + "video": video_entry(), + "unbound": video_entry("unbound.yaml"), + }, single_node_fragment(4, **{"max-running-requests": 16}), video_fragment(**{"max-running-requests": "7"})) # fmt: skip + (project / RECIPES / "unbound.yaml").write_text(yaml.safe_dump(video_fragment())) + monkeypatch.chdir(project) changed.main(["--base", "base", "--json"]) report = json.loads(capsys.readouterr().out) - assert (report["config-keys"], report["errors"]) == (4, []) + assert report["config-keys"] == 6 + [error] = report["errors"] + assert (error["config-key"], error["revision"]) == ("unbound", "base") + assert error["error"].endswith("video service requires a pinned VIDEO_RUNTIME_REVISION") assert [ (change["config-key"], change["base-points"], change["head-points"], [point["conc"] for point in change["added"]], [point["conc"] for point in change["removed"]]) for change in report["changed"] - ] == [("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), ("gone", 1, 0, [], [2])] # fmt: skip + ] == [ + ("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), ("unbound", 0, 1, [1], []), + ("video", 1, 1, [1], [1]), ("gone", 1, 0, [], [2]), + ] # fmt: skip assert git(tmp_path, "status", "--porcelain").splitlines() == [ - f" M {RECIPES}/edited.yaml", " M configs/nvidia-master.yaml", + f" M inferencex-e2e/{RECIPES}/edited.yaml", + f" M inferencex-e2e/{RECIPES}/unbound.yaml", + f" M inferencex-e2e/{RECIPES}/video.yaml", + " M inferencex-e2e/configs/nvidia-master.yaml", ] changed.main(["--base", "base"]) - assert capsys.readouterr().out.splitlines()[-1] == "3 of 4 config keys changed since base." + lines = capsys.readouterr().out.splitlines() + assert lines[-2] == "5 of 6 config keys changed since base." + assert lines[-1].startswith("warning: 1 rows of unbound have no fingerprint at base: ") def test_rows_without_fingerprints_are_compared_individually(tmp_path): diff --git a/inferencex-e2e/infx/tests/srt_recipes.py b/inferencex-e2e/infx/tests/srt_recipes.py index c01f9cc60c..fa954a8e6d 100644 --- a/inferencex-e2e/infx/tests/srt_recipes.py +++ b/inferencex-e2e/infx/tests/srt_recipes.py @@ -1,11 +1,19 @@ """Launchable srt-slurm recipes for tests that plan or fingerprint matrix rows.""" +import shutil from pathlib import Path import yaml +from infx.matrix.validation import VIDEO_SPEC_DIR from infx.srt_slurm.workload import NATIVE_TELEMETRY_BLOCK, SHARED_BLOCKS, TELEMETRY_BLOCK, VIDEO_BLOCK +ROOT = Path(__file__).resolve().parents[2] +VIDEO_MODEL = "example/video-model" +VIDEO_SPEC = "fixture.yaml" +VIDEO_SET = "fixture-set" +VIDEO_PORT = "30283" + def shared_blocks() -> dict[str, str]: """Each lane's shared block (its benchmark client) and the DCGM telemetry block, by path.""" @@ -21,9 +29,13 @@ def shared_blocks() -> dict[str, str]: blocks[NATIVE_TELEMETRY_BLOCK.as_posix()] = yaml.safe_dump( {"telemetry": {"enabled": True, "required": False}} ) - blocks[VIDEO_BLOCK.as_posix()] = yaml.safe_dump( - {"frontend": {"type": "none"}, "benchmark": {"type": "custom", "command": "python3 -m infx.bench video client"}} - ) + blocks[VIDEO_BLOCK.as_posix()] = yaml.safe_dump({ + "frontend": {"type": "none"}, + "benchmark": { + "type": "custom", "command": "python3 -m infx.bench video client --request-seconds 60", + "env": {"VIDEO_SERVER_SERVICE": "video-server", "VIDEO_SERVER_PORT": VIDEO_PORT}, + }, + }) # fmt: skip return blocks @@ -33,8 +45,67 @@ def write_shared_blocks(project: Path) -> None: (project / path).write_text(text) +def write_video_specs(repository: Path) -> None: + """Put the workload helper and a one-case spec serving ``VIDEO_MODEL`` beside the project.""" + specs = repository / VIDEO_SPEC_DIR + specs.mkdir(parents=True, exist_ok=True) + shutil.copy(ROOT.parent / VIDEO_SPEC_DIR / "video_spec.py", specs / "video_spec.py") + (specs / VIDEO_SPEC).write_text(yaml.safe_dump({ + "schema_version": 1, + "model": {"id": VIDEO_MODEL, "revision": "a" * 40}, + "generation": {"num_inference_steps": 1}, + "prompts": {"fixture": {"text": "a fixture prompt", "requires_motion": False, "requires_sound": False}}, + "sets": {VIDEO_SET: { + "plan_id": "fixture-plan", "repetitions": 1, "warmup_runs": 0, + "cases": [{"case_id": "fixture", "prompt": "fixture", "seed": 1}], + }}, + "reference": {}, + })) # fmt: skip + + +def fixed_seq_entry(recipe: str, concurrencies: list[int]) -> dict: + """A master entry with one TP4 fixed-sequence point per concurrency, served by ``recipe``.""" + return { + "image": "example/image:1", "model": "org/model", "model-prefix": "dsr1", + "precision": "fp8", "framework": "sglang", "runner": "cluster:fixture", + "multinode": False, "srt-recipe-dir": "fixture", + "scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [ + {"tp": 4, "conc-list": concurrencies, "srt-recipe": recipe}, + ]}]}, + } # fmt: skip + + +def video_entry(recipe: str = "video.yaml") -> dict: + """A master entry with one 2-GPU video point of ``VIDEO_SPEC``'s set served by ``recipe``.""" + return { + "image": "example/video:1", "model": VIDEO_MODEL, "model-prefix": "h3", + "precision": "bf16", "framework": "sglang", "runner": "cluster:fixture", + "multinode": False, "srt-recipe-dir": "fixture", + "scenarios": {"video-generation": [{"spec": VIDEO_SPEC, "set": VIDEO_SET, "search-space": [ + {"gpus": 2, "conc-list": [1], "srt-recipe": recipe}, + ]}]}, + } # fmt: skip + + def single_node_fragment(tp: int, **args) -> dict: """A single-node sglang fragment serving TP ``tp`` points without speculation.""" return {"schema": 2, "engine": "sglang", "roles": {"agg": { "nodes": 1, "workers": 1, "gpus": tp, "args": {"tensor-parallel-size": tp, **args}, }}} # fmt: skip + + +def video_fragment(**args: str) -> dict: + """A native sglang video fragment the binder accepts, with extra engine ``args`` appended.""" + return {"schema": 2, "name": "video-fixture", "resources": {"gpu_type": "fixture"}, "services": [{ + "name": "video-server", "type": "generic", "nodes": 1, "critical": True, + "command": ["python3", "-m", "infx.bench", "video", "serve"], + "args": [ + "--model-view", "snapshot", "--", "python3", "-m", "evaluator.srt_selected", "serve", + "--model-path", "{model_path}", "--model-id", "{model_id}", + "--revision", "{model_revision}", "--num-gpus", "{gpus_per_node}", + "--disable-conditioning-cache", "--host", "0.0.0.0", "--port", VIDEO_PORT, + *(f"--{flag}={value}" for flag, value in args.items()), + ], + "env": {"VIDEO_SERVER_PORT": VIDEO_PORT, "VIDEO_RUNTIME_REVISION": "b" * 40}, + "readiness": {"http": {"port": int(VIDEO_PORT), "path": "/health"}}, + }]} # fmt: skip diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index d5751eecba..d301b08b40 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -9,6 +9,14 @@ from infx.cli import main from infx.launch.drivers.srt import lanes from infx.launch.policy import LaunchPath +from infx.tests.srt_recipes import ( + VIDEO_MODEL, + VIDEO_SPEC, + video_entry, + video_fragment, + write_shared_blocks, + write_video_specs, +) ROOT = Path(__file__).resolve().parents[3] FRAGMENT = { @@ -51,6 +59,7 @@ "power": True, }]}]}, }, + "video": video_entry(), } # fmt: skip SHARED = { "telemetry-native.yaml": {"telemetry": {"enabled": True, "required": False}}, @@ -64,11 +73,14 @@ @pytest.fixture def project(tmp_path): + """The end-to-end project beside the video harness, as the repository lays them out.""" + project = tmp_path / "inferencex-e2e" + write_shared_blocks(project) + write_video_specs(tmp_path) for name, block in SHARED.items(): - (tmp_path / "configs/srt-recipes" / name).parent.mkdir(parents=True, exist_ok=True) - (tmp_path / "configs/srt-recipes" / name).write_text(yaml.safe_dump(block)) - (tmp_path / "configs/master.yaml").write_text(yaml.safe_dump(MASTER)) - (tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump({ + (project / "configs/srt-recipes" / name).write_text(yaml.safe_dump(block)) + (project / "configs/master.yaml").write_text(yaml.safe_dump(MASTER)) + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ "labels": {"fixture": ["fixture_0"], "cluster:fixture": ["fixture_0"]}, "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", "slurm": { "partition": "p", "exclusive": True, @@ -88,6 +100,7 @@ def project(tmp_path): **RECIPE, "frontend": {"type": "sglang", "enable_multiple_frontends": False}, "roles": {"agg": {"nodes": 1, "workers": 1, "gpus": 4, "args": {"tensor-parallel-size": 4}}}, }, + "single_node/srt-slurm-recipes/fixture/video.yaml": video_fragment(), "multi_node/srt-slurm-recipes/fixture/agentic.yaml": { **RECIPE, "frontend": {"type": "sglang-router"}, "roles": { role: {"nodes": 1, "workers": 1, "gpus": 8, "args": {"tensor-parallel-size": 8}} @@ -96,11 +109,11 @@ def project(tmp_path): }, } # fmt: skip for path, data in recipes.items(): - (tmp_path / "benchmarks" / path).parent.mkdir(parents=True, exist_ok=True) - (tmp_path / "benchmarks" / path).write_text(yaml.safe_dump(data)) - (tmp_path / "utils/srt-slurm").mkdir(parents=True) - (tmp_path / "utils/srt-slurm/src").symlink_to(ROOT / "utils/srt-slurm/src") - return tmp_path + (project / "benchmarks" / path).parent.mkdir(parents=True, exist_ok=True) + (project / "benchmarks" / path).write_text(yaml.safe_dump(data)) + (project / "utils/srt-slurm").mkdir(parents=True) + (project / "utils/srt-slurm/src").symlink_to(ROOT / "utils/srt-slurm/src") + return project @pytest.fixture @@ -314,6 +327,21 @@ def test_output_must_be_new_or_empty(project, tmp_path, capsys): assert "Output directory must be empty" in capsys.readouterr().err +def test_a_video_point_renders_its_bound_server_recipe(project, tmp_path): + output = tmp_path / "out" + assert generate(project, output, "video") == 0 + + [(record, recipe)] = written(output) + assert (record["variant"], record["cluster"]) == (None, "fixture") + assert (record["matrix"]["scenario-type"], record["matrix"]["conc-list"]) == ("video-generation", [1]) + assert recipe["model"] == {"path": f"hf:{VIDEO_MODEL}", "container": "example/video:1", "precision": "bf16"} + assert recipe["resources"]["gpus_per_node"] == 2 + [server] = recipe["services"] + assert server["args"][server["args"].index("--num-gpus") + 1] == "2" + assert server["env"]["VIDEO_SPEC_PATH"] == f"/video-package/specs/{VIDEO_SPEC}" + assert recipe["benchmark"]["env"]["CONC_LIST"] == "1" + + def test_a_recipe_srtctl_rejects_fails_without_writing(project, tmp_path, capsys): recipe = project / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" recipe.write_text(yaml.safe_dump({**FRAGMENT, "base": {**FRAGMENT["base"], "no_such_field": 1}})) diff --git a/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py b/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py index 21c154d75b..33826373c3 100644 --- a/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py +++ b/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py @@ -1,10 +1,21 @@ """Exercise the verifier's deterministic facts against controlled inputs.""" import json +import subprocess import pytest +import yaml from infx.klaud.models import Feed +from infx.matrix.validation import VIDEO_SPEC_DIR +from infx.tests.srt_recipes import ( + VIDEO_SPEC, + fixed_seq_entry, + single_node_fragment, + video_entry, + video_fragment, + write_shared_blocks, +) from infx.workflows import signoff_facts SERIES = ["--model", "m", "--precision", "fp4", "--framework", "sglang", "--hardware", "b200"] @@ -268,3 +279,41 @@ def test_eval_gap_estimates_zero_standard_errors_from_sample_size(): def test_digest_pins_are_inspected_by_digest(monkeypatch, image, inspected): monkeypatch.setattr(signoff_facts, "_inspect", lambda reference: {"inspected": reference}) assert signoff_facts.inspect_image(image)["inspected"] == inspected + + +def test_matrix_at_generates_llm_points_only_and_runs_no_revision_code(tmp_path, monkeypatch): + project = tmp_path / "inferencex-e2e" + recipes = project / "benchmarks/single_node/srt-slurm-recipes/fixture" + write_shared_blocks(project) + (project / "configs/runners.yaml").write_text(yaml.safe_dump({ + "labels": {"cluster:fixture": ["node-a"]}, + "clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", + "slurm": {"partition": "batch", "exclusive": True}}}, + })) # fmt: skip + (project / "configs/nvidia-master.yaml").write_text( + yaml.safe_dump({"llm": fixed_seq_entry("stable.yaml", [2]), "video": video_entry()}) + ) + (project / "configs/amd-master.yaml").write_text("{}\n") + recipes.mkdir(parents=True) + (recipes / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4))) + (recipes / "video.yaml").write_text(yaml.safe_dump(video_fragment())) + # git archive rejects a DATA_PATHS entry that the tree lacks. + (project / "benchmarks/multi_node/srt-slurm-recipes").mkdir(parents=True) + (project / "benchmarks/multi_node/srt-slurm-recipes/.keep").touch() + specs = tmp_path / VIDEO_SPEC_DIR + specs.mkdir(parents=True) + (specs / "video_spec.py").write_text('raise RuntimeError("video_spec.py from the revision tree ran")\n') + (specs / VIDEO_SPEC).write_text("{}\n") + for args in (["init", "-q"], ["add", "."], ["commit", "-qm", "base"]): + subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], + cwd=tmp_path, check=True, capture_output=True, + ) # fmt: skip + monkeypatch.chdir(tmp_path) + + matrix, errors = signoff_facts.matrix_at("HEAD", tmp_path / "work") + + assert errors == {} + assert { + key: ([conc for _, conc in points], messages) for key, (points, messages) in matrix.items() + } == {"llm": ([2], []), "video": ([], [])}