Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 33 additions & 12 deletions inferencex-e2e/infx/matrix/changed.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
from infx.config import MASTER_CONFIGS, RUNNER_CONFIG

from .fingerprint import recipe_fingerprint, row_fingerprint
from .generate import generate_config_matrix
from .revision import snapshot
from .generate import SCENARIO_TYPES, generate_config_matrix
from .revision import Revision, snapshot
from .validation import config_root, load_config_files, load_runner_file

Matrix = dict[str, tuple[dict[tuple[str, int], dict[str, Any]], list[str]]]
Expand Down Expand Up @@ -43,17 +43,33 @@ def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]:
return points, errors


def config_points(key: str, master: dict, runners: dict, root: Path) -> tuple[dict, list[str]]:
"""Every point of config ``key``, across the scenario types a changelog entry can select."""
rows = generate_config_matrix(
[key], master, runners, scenario_types=SCENARIO_TYPES, eval_mode="none", root=root
)
return _points(rows, root)


def head_matrix(config_files: list[str], runner_config: str) -> Matrix:
"""Generate the working tree's matrix for each config key, as points and fingerprint errors."""
master = load_config_files(config_files)
runners = load_runner_file(runner_config)
root = config_root(config_files)
return {
key: _points(
generate_config_matrix([key], master, runners, eval_mode="none", root=root), root
)
for key in master
}
return {key: config_points(key, master, runners, root) for key in master}


def _base_video_points(revision: Revision, key: str, single: str) -> tuple[dict, list[str]]:
"""The video points of config ``key`` at ``revision``, or its generator's failure as an error.

That generator exits when a video recipe cannot be bound, where head records the error per key.
"""
flags = ["--no-evals", "--scenario-type", "video-generation"]
try:
rows = revision.generate([key], flags, [single])
except subprocess.CalledProcessError as error:
return {}, [error.stderr.strip().rsplit("\n", 1)[-1]]
return _points(rows, revision.root)


def base_matrix(ref: str, config_files: list[str]) -> Matrix:
Expand All @@ -72,10 +88,15 @@ def base_matrix(ref: str, config_files: list[str]) -> Matrix:
([key], ["--no-evals"], [single]) for key, single in zip(master, singles, strict=True)
]
rows = revision.generate_each(requests, os.cpu_count() or 1)
return {
key: _points(generated, revision.root)
for key, generated in zip(master, rows, strict=True)
}
matrix: Matrix = {}
for (key, entry), single, generated in zip(master.items(), singles, rows, strict=True):
points, errors = _points(generated, revision.root)
if "video-generation" in entry.get("scenarios", {}):
video_points, video_errors = _base_video_points(revision, key, single)
points.update(video_points)
errors.extend(video_errors)
matrix[key] = (points, errors)
return matrix


def _sorted(points: Iterable[dict]) -> list[dict]:
Expand Down
5 changes: 4 additions & 1 deletion inferencex-e2e/infx/matrix/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@

seq_len_stoi = {"1k1k": (1024, 1024), "8k1k": (8192, 1024)}

# Every scenario type a changelog entry can select; unscoped generation leaves video out.
SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding", "video-generation")

MIN_EVAL_CONC = 16
DEFAULT_EVAL_FRAMEWORK = "lm-eval"
AUTOMATIC_AGENTIC_VENDOR_EVALS = {
Expand Down Expand Up @@ -1797,7 +1800,7 @@ def main() -> list[dict]:
parent_parser.add_argument(
"--scenario-type",
nargs="+",
choices=["fixed-seq-len", "agentic-coding", "video-generation"],
choices=list(SCENARIO_TYPES),
required=False,
help="Scenario type(s) to include. If not specified, fixed-seq-len and agentic-coding are generated; video-generation runs only when named.",
)
Expand Down
2 changes: 1 addition & 1 deletion inferencex-e2e/infx/matrix/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from .fingerprint import recipe_fingerprint
from .generate import (
SCENARIO_TYPES,
EvalMode,
drop_app_colliding_evals,
freeze_config_value,
Expand All @@ -27,7 +28,6 @@
load_runner_file,
)

SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding", "video-generation")
# An entry without scenario-type reruns these; video runs only for entries that name it.
DEFAULT_SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding")

Expand Down
6 changes: 4 additions & 2 deletions inferencex-e2e/infx/srt_slurm/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from infx.launch.drivers.srt import config, lanes, power
from infx.launch.policy import launch_path
from infx.launch.request import MultiNodeRequest
from infx.matrix.generate import expand_config_keys, generate_config_matrix
from infx.matrix.generate import SCENARIO_TYPES, expand_config_keys, generate_config_matrix
from infx.matrix.validation import config_root, load_config_files, load_runner_file
from infx.srt_slurm.single_node import select_recipe
from infx.srt_slurm.synthetic_acceptance import build_overrides, selected_recipes
Expand Down Expand Up @@ -292,7 +292,9 @@ def generate_recipes(
recipes: dict[str, dict[str, Any]] = {}
records = []
for key in expand_config_keys(config_keys, master):
points = generate_config_matrix([key], master, runners, eval_mode="none", root=root)
points = generate_config_matrix(
[key], master, runners, scenario_types=SCENARIO_TYPES, eval_mode="none", root=root
)
points = [point for point in points if point.get("srt-recipe")]
if not points:
raise ValueError(f"{key} has no srt-slurm points")
Expand Down
90 changes: 56 additions & 34 deletions inferencex-e2e/infx/tests/matrix/test_changed.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,23 @@
import yaml

from infx.matrix import changed
from infx.tests.srt_recipes import single_node_fragment, write_shared_blocks
from infx.tests.srt_recipes import (
fixed_seq_entry,
single_node_fragment,
video_entry,
video_fragment,
write_shared_blocks,
write_video_specs,
)

ROOT = Path(__file__).resolve().parents[3]
RECIPES = "benchmarks/single_node/srt-slurm-recipes/fixture"


def entry(recipe: str, concurrencies: list[int]) -> dict:
return {
"image": "example/image:1", "model": "org/model", "model-prefix": "dsr1",
"precision": "fp8", "framework": "sglang", "runner": "cluster:fixture",
"multinode": False, "srt-recipe-dir": "fixture",
"scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [
{"tp": 4, "conc-list": concurrencies, "srt-recipe": recipe},
]}]},
} # fmt: skip


def write(root: Path, master: dict, edited: dict) -> None:
(root / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master))
(root / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited))
def write(project: Path, master: dict, edited: dict, video: dict) -> None:
(project / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master))
(project / RECIPES / "edited.yaml").write_text(yaml.safe_dump(edited))
(project / RECIPES / "video.yaml").write_text(yaml.safe_dump(video))


def git(root: Path, *args: str) -> str:
Expand All @@ -38,47 +35,72 @@ def git(root: Path, *args: str) -> str:


def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, capsys):
shutil.copytree(ROOT / "infx", tmp_path / "infx", ignore=shutil.ignore_patterns("__pycache__"))
(tmp_path / RECIPES).mkdir(parents=True)
write_shared_blocks(tmp_path)
(tmp_path / "configs/amd-master.yaml").write_text("{}\n")
(tmp_path / "configs/runners.yaml").write_text(yaml.safe_dump({
project = tmp_path / "inferencex-e2e"
shutil.copytree(ROOT / "infx", project / "infx", ignore=shutil.ignore_patterns("__pycache__"))
(project / RECIPES).mkdir(parents=True)
write_shared_blocks(project)
write_video_specs(tmp_path)
(project / "configs/amd-master.yaml").write_text("{}\n")
(project / "configs/runners.yaml").write_text(yaml.safe_dump({
"labels": {"cluster:fixture": ["node-a"]},
"clusters": {"fixture": {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm",
"slurm": {"partition": "batch", "exclusive": True}}},
})) # fmt: skip
(tmp_path / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4)))
write(tmp_path, {
"edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2]),
"same": entry("stable.yaml", [8]), "gone": entry("stable.yaml", [2]),
}, single_node_fragment(4)) # fmt: skip
(project / RECIPES / "stable.yaml").write_text(yaml.safe_dump(single_node_fragment(4)))
unbound = video_fragment()
del unbound["services"][0]["env"]["VIDEO_RUNTIME_REVISION"]
(project / RECIPES / "unbound.yaml").write_text(yaml.safe_dump(unbound))
write(project, {
"edited": fixed_seq_entry("edited.yaml", [2, 4]),
"grown": fixed_seq_entry("stable.yaml", [2]),
"same": fixed_seq_entry("stable.yaml", [8]),
"video": video_entry(),
"unbound": video_entry("unbound.yaml"),
"gone": fixed_seq_entry("stable.yaml", [2]),
}, single_node_fragment(4), video_fragment()) # fmt: skip
(tmp_path / ".gitignore").write_text("__pycache__/\n")
git(tmp_path, "init", "-q")
git(tmp_path, "config", "user.name", "Test")
git(tmp_path, "config", "user.email", "test@example.com")
git(tmp_path, "add", ".")
git(tmp_path, "commit", "-qm", "base")
git(tmp_path, "tag", "base")
write(tmp_path, {
"edited": entry("edited.yaml", [2, 4]), "grown": entry("stable.yaml", [2, 4]),
"same": entry("stable.yaml", [8]),
}, single_node_fragment(4, **{"max-running-requests": 16})) # fmt: skip
monkeypatch.chdir(tmp_path)
write(project, {
"edited": fixed_seq_entry("edited.yaml", [2, 4]),
"grown": fixed_seq_entry("stable.yaml", [2, 4]),
"same": fixed_seq_entry("stable.yaml", [8]),
"video": video_entry(),
"unbound": video_entry("unbound.yaml"),
}, single_node_fragment(4, **{"max-running-requests": 16}), video_fragment(**{"max-running-requests": "7"})) # fmt: skip
(project / RECIPES / "unbound.yaml").write_text(yaml.safe_dump(video_fragment()))
monkeypatch.chdir(project)

changed.main(["--base", "base", "--json"])
report = json.loads(capsys.readouterr().out)

assert (report["config-keys"], report["errors"]) == (4, [])
assert report["config-keys"] == 6
[error] = report["errors"]
assert (error["config-key"], error["revision"]) == ("unbound", "base")
assert error["error"].endswith("video service requires a pinned VIDEO_RUNTIME_REVISION")
assert [
(change["config-key"], change["base-points"], change["head-points"],
[point["conc"] for point in change["added"]], [point["conc"] for point in change["removed"]])
for change in report["changed"]
] == [("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), ("gone", 1, 0, [], [2])] # fmt: skip
] == [
("edited", 2, 2, [2, 4], [2, 4]), ("grown", 1, 2, [4], []), ("unbound", 0, 1, [1], []),
("video", 1, 1, [1], [1]), ("gone", 1, 0, [], [2]),
] # fmt: skip
assert git(tmp_path, "status", "--porcelain").splitlines() == [
f" M {RECIPES}/edited.yaml", " M configs/nvidia-master.yaml",
f" M inferencex-e2e/{RECIPES}/edited.yaml",
f" M inferencex-e2e/{RECIPES}/unbound.yaml",
f" M inferencex-e2e/{RECIPES}/video.yaml",
" M inferencex-e2e/configs/nvidia-master.yaml",
]

changed.main(["--base", "base"])
assert capsys.readouterr().out.splitlines()[-1] == "3 of 4 config keys changed since base."
lines = capsys.readouterr().out.splitlines()
assert lines[-2] == "5 of 6 config keys changed since base."
assert lines[-1].startswith("warning: 1 rows of unbound have no fingerprint at base: ")


def test_rows_without_fingerprints_are_compared_individually(tmp_path):
Expand Down
77 changes: 74 additions & 3 deletions inferencex-e2e/infx/tests/srt_recipes.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,19 @@
"""Launchable srt-slurm recipes for tests that plan or fingerprint matrix rows."""

import shutil
from pathlib import Path

import yaml

from infx.matrix.validation import VIDEO_SPEC_DIR
from infx.srt_slurm.workload import NATIVE_TELEMETRY_BLOCK, SHARED_BLOCKS, TELEMETRY_BLOCK, VIDEO_BLOCK

ROOT = Path(__file__).resolve().parents[2]
VIDEO_MODEL = "example/video-model"
VIDEO_SPEC = "fixture.yaml"
VIDEO_SET = "fixture-set"
VIDEO_PORT = "30283"


def shared_blocks() -> dict[str, str]:
"""Each lane's shared block (its benchmark client) and the DCGM telemetry block, by path."""
Expand All @@ -21,9 +29,13 @@ def shared_blocks() -> dict[str, str]:
blocks[NATIVE_TELEMETRY_BLOCK.as_posix()] = yaml.safe_dump(
{"telemetry": {"enabled": True, "required": False}}
)
blocks[VIDEO_BLOCK.as_posix()] = yaml.safe_dump(
{"frontend": {"type": "none"}, "benchmark": {"type": "custom", "command": "python3 -m infx.bench video client"}}
)
blocks[VIDEO_BLOCK.as_posix()] = yaml.safe_dump({
"frontend": {"type": "none"},
"benchmark": {
"type": "custom", "command": "python3 -m infx.bench video client --request-seconds 60",
"env": {"VIDEO_SERVER_SERVICE": "video-server", "VIDEO_SERVER_PORT": VIDEO_PORT},
},
}) # fmt: skip
return blocks


Expand All @@ -33,8 +45,67 @@ def write_shared_blocks(project: Path) -> None:
(project / path).write_text(text)


def write_video_specs(repository: Path) -> None:
"""Put the workload helper and a one-case spec serving ``VIDEO_MODEL`` beside the project."""
specs = repository / VIDEO_SPEC_DIR
specs.mkdir(parents=True, exist_ok=True)
shutil.copy(ROOT.parent / VIDEO_SPEC_DIR / "video_spec.py", specs / "video_spec.py")
(specs / VIDEO_SPEC).write_text(yaml.safe_dump({
"schema_version": 1,
"model": {"id": VIDEO_MODEL, "revision": "a" * 40},
"generation": {"num_inference_steps": 1},
"prompts": {"fixture": {"text": "a fixture prompt", "requires_motion": False, "requires_sound": False}},
"sets": {VIDEO_SET: {
"plan_id": "fixture-plan", "repetitions": 1, "warmup_runs": 0,
"cases": [{"case_id": "fixture", "prompt": "fixture", "seed": 1}],
}},
"reference": {},
})) # fmt: skip


def fixed_seq_entry(recipe: str, concurrencies: list[int]) -> dict:
"""A master entry with one TP4 fixed-sequence point per concurrency, served by ``recipe``."""
return {
"image": "example/image:1", "model": "org/model", "model-prefix": "dsr1",
"precision": "fp8", "framework": "sglang", "runner": "cluster:fixture",
"multinode": False, "srt-recipe-dir": "fixture",
"scenarios": {"fixed-seq-len": [{"isl": 1024, "osl": 128, "search-space": [
{"tp": 4, "conc-list": concurrencies, "srt-recipe": recipe},
]}]},
} # fmt: skip


def video_entry(recipe: str = "video.yaml") -> dict:
"""A master entry with one 2-GPU video point of ``VIDEO_SPEC``'s set served by ``recipe``."""
return {
"image": "example/video:1", "model": VIDEO_MODEL, "model-prefix": "h3",
"precision": "bf16", "framework": "sglang", "runner": "cluster:fixture",
"multinode": False, "srt-recipe-dir": "fixture",
"scenarios": {"video-generation": [{"spec": VIDEO_SPEC, "set": VIDEO_SET, "search-space": [
{"gpus": 2, "conc-list": [1], "srt-recipe": recipe},
]}]},
} # fmt: skip


def single_node_fragment(tp: int, **args) -> dict:
"""A single-node sglang fragment serving TP ``tp`` points without speculation."""
return {"schema": 2, "engine": "sglang", "roles": {"agg": {
Comment thread
edwingao28 marked this conversation as resolved.
Resolved
"nodes": 1, "workers": 1, "gpus": tp, "args": {"tensor-parallel-size": tp, **args},
}}} # fmt: skip


def video_fragment(**args: str) -> dict:
"""A native sglang video fragment the binder accepts, with extra engine ``args`` appended."""
return {"schema": 2, "name": "video-fixture", "resources": {"gpu_type": "fixture"}, "services": [{
"name": "video-server", "type": "generic", "nodes": 1, "critical": True,
"command": ["python3", "-m", "infx.bench", "video", "serve"],
"args": [
"--model-view", "snapshot", "--", "python3", "-m", "evaluator.srt_selected", "serve",
"--model-path", "{model_path}", "--model-id", "{model_id}",
"--revision", "{model_revision}", "--num-gpus", "{gpus_per_node}",
"--disable-conditioning-cache", "--host", "0.0.0.0", "--port", VIDEO_PORT,
*(f"--{flag}={value}" for flag, value in args.items()),
],
"env": {"VIDEO_SERVER_PORT": VIDEO_PORT, "VIDEO_RUNTIME_REVISION": "b" * 40},
"readiness": {"http": {"port": int(VIDEO_PORT), "path": "/health"}},
}]} # fmt: skip
Loading
Loading