From bb2ee1b28057392193f4e2bb92c244f21ebe0049 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:58:46 -0500 Subject: [PATCH 001/203] fix(agentic): publish the Dynamo frontend cache hit rate as the overall rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dynamo-vLLM adapter fills the overall cache hit rate when vLLM prefix metrics are absent, but it wrote only the discarded flat field, so the published nested value stayed null. It also chose with `or`, which replaced a measured 0% frontend hit rate with the router gauge. Dynamo-vLLM runs without vLLM prefix-cache metrics now publish the frontend rate, including 0.0, and fall back to the router shared rate only when the frontend counters are missing. 中文:当缺少 vLLM 前缀缓存指标时,Dynamo-vLLM 适配器会补填整体缓存命中率,但它只写入了被丢弃的扁平字段,因此发布的嵌套值始终为 null。它还用 `or` 选择取值,导致实测为 0% 的前端命中率被路由器指标替换。现在,缺少 vLLM 前缀缓存指标的 Dynamo-vLLM 运行会发布前端命中率(包括 0.0),只有在前端计数器缺失时才回退到路由器共享命中率。 --- .../results/agentic/backends/dynamo_vllm.py | 4 ++- .../agentic/test_process_agentic_result.py | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py b/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py index 273be93f58..6edae36cbb 100644 --- a/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py @@ -92,7 +92,9 @@ def _populate_dynamo( ) ) if flat["server_overall_cache_hit_rate"] is None: - flat["server_overall_cache_hit_rate"] = frontend_hit_rate or router_shared_hit_rate + overall = frontend_hit_rate if frontend_hit_rate is not None else router_shared_hit_rate + flat["server_overall_cache_hit_rate"] = overall + nested["cache"]["overall_cache_hit_rate"] = overall nested["cache"].update( { diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index e162f49685..363ce7ba1a 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -1607,6 +1607,37 @@ def test_processor_normalizes_dynamo_server_metrics(tmp_path: Path): assert any(source["role"] == "prefill" for source in agg["server_metrics"]["sources"]) +@pytest.mark.parametrize(("cached", "router_shared", "expected"), [ + (0.0, 0.55, 0.0), + (25.0, None, 0.25), + (None, 0.55, 0.55), +]) +def test_dynamo_overall_cache_hit_rate_falls_back_to_frontend_then_router( + cached, router_shared, expected, +): + metrics = { + "dynamo_frontend_input_sequence_tokens": { + "type": "counter", "series": [{"stats": {"total": 100.0}}], + }, + } + if cached is not None: + metrics["dynamo_frontend_cached_tokens"] = { + "type": "counter", "series": [{"stats": {"total": cached}}], + } + if router_shared is not None: + metrics["dynamo_component_router_shared_cache_hit_rate"] = { + "type": "gauge", "series": [{"stats": {"avg": router_shared}}], + } + + _, nested, _ = compute_server_metrics( + {"metrics": metrics}, framework="dynamo-vllm", records=[], + ) + + assert nested["adapter"] == "dynamo-vllm" + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(expected) + + + def test_processor_uses_aiperf_theoretical_cache_metric(tmp_path: Path): """Aiperf's exported profile aggregate is the theoretical cache source.""" result_dir = _write_fixture(tmp_path) From ca30549a2a881c978f128dc502849644010841a5 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:59:33 -0500 Subject: [PATCH 002/203] fix(agentic): exclude warmup and failed requests from the workload summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workload distribution summary read every row of profile_export.jsonl, while the AgentX aggregate drops warmup rows and errored requests. A run with warmup or failed requests therefore reported a larger request count and different ISL and OSL statistics than its aggregate. The summary now uses the aggregate's loader, so only runs that contain warmup or errored rows change, and their percentile definition is unchanged. 中文:工作负载分布摘要会读取 profile_export.jsonl 中的每一行,而 AgentX 聚合结果会丢弃预热行和出错的请求。因此,包含预热或失败请求的运行所报告的请求数以及 ISL 和 OSL 统计值会与其聚合结果不一致。现在摘要改用聚合结果的加载函数,只有包含预热行或出错行的运行会发生变化,百分位数的定义保持不变。 --- .../analyze_benchmark_distributions.py | 14 ++----- .../test_analyze_benchmark_distributions.py | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py diff --git a/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py b/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py index 213da4ebc9..fdb80c4cb5 100644 --- a/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py +++ b/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py @@ -10,20 +10,14 @@ from __future__ import annotations import argparse -import json from pathlib import Path +from .artifacts import load_records as load_profiled_records + def load_records(artifacts_dir: Path) -> list[dict]: - """Load per-request records from profile_export.jsonl.""" - jsonl_path = artifacts_dir / "profile_export.jsonl" - records = [] - with open(jsonl_path) as f: - for line in f: - line = line.strip() - if line: - records.append(json.loads(line)) - return records + """Load successful profiling records from profile_export.jsonl.""" + return load_profiled_records(artifacts_dir / "profile_export.jsonl") def _stats(values: list[int]) -> dict[str, float]: diff --git a/inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py b/inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py new file mode 100644 index 0000000000..b55d6e4f8a --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +E2E_ROOT = Path(__file__).resolve().parents[4] + + +def _record(isl: int, osl: int, *, phase: str = "profiling", error: dict | None = None) -> dict: + return { + "metadata": {"benchmark_phase": phase}, + "metrics": { + "input_sequence_length": {"value": isl, "unit": "tokens"}, + "output_sequence_length": {"value": osl, "unit": "tokens"}, + }, + "error": error, + } + + +def test_summary_excludes_warmup_and_failed_requests(tmp_path: Path) -> None: + artifacts = tmp_path / "aiperf_artifacts" + artifacts.mkdir() + rows = [ + _record(9000, 900, phase="warmup"), + _record(100, 10), + _record(5000, 500, error={"type": "HTTPStatusError", "message": "500"}), + _record(300, 30), + ] + (artifacts / "profile_export.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows)) + + subprocess.run( + [sys.executable, "-m", "infx.results.agentic.analyze_benchmark_distributions", + str(artifacts), "-o", str(tmp_path / "out")], + cwd=E2E_ROOT, check=True, capture_output=True, text=True, timeout=30, + ) + + summary = (tmp_path / "out" / "workload_distribution_summary.txt").read_text() + assert "Total requests: 2\n" in summary + assert "ALL REQUESTS ISL:\n n=2 mean=200 " in summary + assert "ALL REQUESTS OSL:\n n=2 mean=20 " in summary From 89d46f8aa7bf37a8aac2aa92f58ebd9b23016266 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:00:15 -0500 Subject: [PATCH 003/203] fix(agentic): report non-object profile JSON as invalid power artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AgentX power window builder maps unreadable profile artifacts to `profile_artifacts_invalid`, but a top-level JSON aggregate that is not an object, or a JSONL row that is not an object, raised AttributeError instead. The `--write-multinode-window completed` command then crashed with exit 1 even when power was optional. Only those malformed artifacts change behavior, and they now return the invalid-artifacts reason like other unreadable profiles. 中文:AgentX 功耗窗口构建器会把无法读取的性能剖析产物映射为 `profile_artifacts_invalid`,但当顶层 JSON 聚合结果不是对象,或 JSONL 中某一行不是对象时,它会抛出 AttributeError。这样,即使功耗是可选项,`--write-multinode-window completed` 命令也会崩溃并以退出码 1 结束。只有这类格式错误的产物行为会改变,现在它们会像其他无法读取的剖析产物一样返回无效产物原因。 --- .../infx/results/agentic/power_adapter.py | 4 +++- .../tests/results/agentic/test_power_adapter.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/results/agentic/power_adapter.py b/inferencex-e2e/infx/results/agentic/power_adapter.py index 9973c0ed7a..13eaee27ca 100644 --- a/inferencex-e2e/infx/results/agentic/power_adapter.py +++ b/inferencex-e2e/infx/results/agentic/power_adapter.py @@ -85,7 +85,9 @@ def build_power_window( try: aggregate = load_aggregate(aggregate_path) records = load_records(records_path) - except (OSError, json.JSONDecodeError, TypeError, ValueError): + except (AttributeError, OSError, json.JSONDecodeError, TypeError, ValueError): + return None, ["profile_artifacts_invalid"] + if not isinstance(aggregate, dict): return None, ["profile_artifacts_invalid"] raw_start = aggregate.get("start_time") diff --git a/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py b/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py index a6da1f2848..9715c0729d 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py @@ -287,6 +287,20 @@ def test_build_power_window_rejects_ambiguous_inputs( assert expected_reason in reasons +@pytest.mark.parametrize( + ("aggregate", "records"), + [(["2023-11-14T22:13:21+00:00"], None), (None, [["not", "an", "object"]])], +) +def test_build_power_window_rejects_non_object_profile_json( + tmp_path: Path, aggregate: list | None, records: list | None +): + from infx.results.agentic.power_adapter import build_power_window + + result_dir = _write_artifacts(tmp_path, aggregate=aggregate, records=records) + + assert build_power_window(result_dir) == (None, ["profile_artifacts_invalid"]) + + _RETIRED_FORK_WINDOW_ENV = ( "SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE", "SRT_MEASUREMENT_WINDOW_CONCURRENCIES", From 25b0eb8fae624ab96820fa0d04bbac37b6f99f16 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:01:36 -0500 Subject: [PATCH 004/203] fix(power): record a non-object aggregate as unwritable instead of crashing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multinode power validation catches ValueError from patching the aggregate and records `aggregate_result_unwritable` in the sidecar, but a JSON list or scalar aggregate raised TypeError or AttributeError. The traceback escaped before the validation sidecar was written, and the run failed even when power was optional. Only non-object aggregates change behavior, and they now raise ValueError, leave the file untouched and produce an invalid sidecar. The existing non-object test pinned the old TypeError and AttributeError types, so it now expects ValueError while still checking that the file is not overwritten. 中文:多节点功耗校验会捕获修补聚合结果时抛出的 ValueError,并在旁车文件中记录 `aggregate_result_unwritable`,但当聚合结果是 JSON 列表或标量时会抛出 TypeError 或 AttributeError。该回溯在写入校验旁车文件之前就已抛出,即使功耗是可选项,运行也会失败。只有非对象的聚合结果行为会改变,现在它们会抛出 ValueError,保持文件不变,并生成无效的旁车文件。原有的非对象测试固定了旧的 TypeError 和 AttributeError 类型,因此现在改为期望 ValueError,同时仍检查文件未被覆盖。 --- inferencex-e2e/infx/results/power/common.py | 2 ++ .../tests/results/power/test_aggregate_power.py | 12 +++--------- .../power/test_aggregate_power_multinode.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/infx/results/power/common.py b/inferencex-e2e/infx/results/power/common.py index a9b33c2eeb..528a94db65 100644 --- a/inferencex-e2e/infx/results/power/common.py +++ b/inferencex-e2e/infx/results/power/common.py @@ -264,6 +264,8 @@ def patch_power_metrics( ) -> None: """Validate replacement metrics before atomically updating an aggregate.""" data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"aggregate is not a JSON object: {path}") data = with_power_metrics( data, metric_keys=metric_keys, diff --git a/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py b/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py index 7cdf87142b..a1fd53410a 100644 --- a/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py +++ b/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py @@ -57,17 +57,11 @@ def test_power_replacement_rejects_nonfinite_without_overwriting( assert path.read_bytes() == original -@pytest.mark.parametrize(("raw", "error_type"), [ - ("[]", TypeError), ('[["model","fixture"]]', TypeError), - ("null", AttributeError), ('"fixture"', AttributeError), - ("4", AttributeError), ("false", AttributeError), -]) -def test_power_replacement_rejects_nonobject_json( - tmp_path, patch_validated_power, raw, error_type -): +@pytest.mark.parametrize("raw", ["[]", '[["model","fixture"]]', "null", '"fixture"', "4", "false"]) +def test_power_replacement_rejects_nonobject_json(tmp_path, patch_validated_power, raw): path = tmp_path / "agg.json" path.write_text(raw) - with pytest.raises(error_type): + with pytest.raises(ValueError, match="aggregate is not a JSON object"): patch_validated_power(path, power_valid=False, metrics={}) assert path.read_text() == raw diff --git a/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py b/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py index f3d959881c..ec2198f047 100644 --- a/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py +++ b/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py @@ -754,3 +754,17 @@ def test_v3_samples_reject_invalid_temperature(tmp_path, temperature): rows, reasons = apm.read_samples(path) assert rows == () assert reasons == ("samples_csv_malformed",) + + +@pytest.mark.parametrize("aggregate", ["[]", "7"]) +def test_non_object_aggregate_is_recorded_as_unwritable(tmp_path, aggregate): + pkg = build_package(tmp_path) + pkg.agg_result.write_text(aggregate) + + assert pkg.run(require_power=False) == 0 + sidecar = pkg.sidecar() + assert sidecar["power_valid"] is False + assert "aggregate_result_unwritable" in sidecar["reasons"] + assert sidecar["metrics"] == {} + assert pkg.agg_result.read_text() == aggregate + assert pkg.run(require_power=True) == 1 From 91ec41cfc49fafa03f25c5c5e1823b04c4a0c05b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:57:01 -0500 Subject: [PATCH 005/203] fix(results): ignore raw agentic dirs of superseded run attempts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an agentic point was rerun at a later run attempt, the earlier attempt's point file was dropped as superseded but its raw agentic_ artifact directory still downloaded. Reusable sweep validation then reported that directory as unexpected and failed. Raw directories whose point results were all superseded by a later attempt are now ignored, and every other raw directory check is unchanged. 中文:当某个 agentic 测试点在后续的 run attempt 中重跑时,较早 attempt 的结果文件会被视为已被取代而丢弃,但它的原始 agentic_ 产物目录仍会被下载。可复用 sweep 校验因此把该目录报告为意外目录并失败。现在,若某原始目录对应的测试点结果全部被后续 attempt 取代,则忽略该目录,其他原始目录的检查保持不变。 --- inferencex-e2e/infx/results/artifacts.py | 13 +++++++++---- .../test_validate_reusable_sweep_artifacts.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/results/artifacts.py b/inferencex-e2e/infx/results/artifacts.py index 0abfd5e55d..d1386a4e2e 100644 --- a/inferencex-e2e/infx/results/artifacts.py +++ b/inferencex-e2e/infx/results/artifacts.py @@ -285,7 +285,8 @@ def validate_agentic_artifacts( artifacts_dir: Path, ) -> list[str]: """Validate agentic point, raw, and aggregate artifacts agree.""" - point_rows = agentic_keys_from_paths(agentic_point_files(artifacts_dir)) + point_files = agentic_point_files(artifacts_dir) + point_rows = agentic_keys_from_paths(point_files) errors = duplicate_identity_errors("agentic point", point_rows) results_bmk = artifacts_dir / "results_bmk" @@ -301,14 +302,18 @@ def validate_agentic_artifacts( ) point_names = { - path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") - for path in agentic_point_files(artifacts_dir) + path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") for path in point_files } + # A rerun attempt's results replace an earlier attempt whose raw dir still downloads. + superseded_names = { + path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") + for path in artifacts_dir.glob("bmk_agentic_*/**/*.json") + } - point_names raw_names = { path.name for path in artifacts_dir.iterdir() if path.is_dir() and path.name.startswith("agentic_") - } + } - superseded_names if point_names != raw_names: missing_raw = point_names - raw_names extra_raw = raw_names - point_names diff --git a/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py b/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py index 532b8ab7a6..cf63970a28 100644 --- a/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py +++ b/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py @@ -947,6 +947,19 @@ def test_agentic_validation_requires_point_and_raw_artifacts( assert "agentic aggregate artifacts contain 1 unexpected row(s)" in errors +def test_agentic_validation_ignores_raw_dir_of_superseded_attempt(tmp_path: Path) -> None: + for attempt in (1, 2): + name = f"agentic_dsv4_conc16_a{attempt}_runner" + point_dir = tmp_path / f"bmk_{name}" + point_dir.mkdir() + (point_dir / "point.json").write_text( + json.dumps({**agentic_result(), "run_attempt": attempt}) + ) + (tmp_path / name).mkdir() + + assert validate_agentic_artifacts(tmp_path) == [] + + def test_agentic_validation_rejects_duplicate_point_identity( tmp_path: Path, ) -> None: From d0ca47e0b7bb1341a8e55aad9b8f63d5a5966e2f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:58:23 -0500 Subject: [PATCH 006/203] perf(results): parse job event times once when reconciling records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconciling job event records with the run's job listing parsed the event start and the job start and end timestamps again for every event and job pair. A sweep with 1500 jobs and 1500 records that lack a job id made about 3.4 million timestamp parses and now makes 6000. The matching rule is unchanged and the written records are byte-identical. 中文:在将作业事件记录与运行的作业列表进行对账时,原代码会对每一对事件与作业重复解析事件开始时间以及作业的开始和结束时间。对于包含 1500 个作业和 1500 条缺少 job id 的记录的 sweep,时间戳解析次数从约 340 万次降至 6000 次。匹配规则保持不变,写出的记录逐字节一致。 --- inferencex-e2e/infx/results/collect_events.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/infx/results/collect_events.py b/inferencex-e2e/infx/results/collect_events.py index e15ca916c2..f17338c56f 100644 --- a/inferencex-e2e/infx/results/collect_events.py +++ b/inferencex-e2e/infx/results/collect_events.py @@ -53,12 +53,21 @@ def reconcile( ) -> list[dict[str, Any]]: """``events`` with their jobs' outcomes, then a record for each benchmark job without one.""" events = [dict(event) for event in events] + starts = [_time(event.get("started_at")) for event in events] synthesized = [] for job in jobs: name = job.get("name") or "" if job.get("conclusion") in {None, "skipped"} or not _BENCHMARK_JOB.search(name): continue - event = next((event for event in events if _ran(event, job)), None) + began, ended = _time(job.get("started_at")), _time(job.get("completed_at")) + event = next( + ( + event + for event, started in zip(events, starts, strict=True) + if _ran(event, started, job, began, ended) + ), + None, + ) if event is None: synthesized.append(_synthesized(job, run_url)) elif isinstance(event.get("github"), dict): @@ -66,16 +75,18 @@ def reconcile( return [*events, *synthesized] -def _ran(event: dict[str, Any], job: dict[str, Any]) -> bool: +def _ran( + event: dict[str, Any], + started: datetime | None, + job: dict[str, Any], + began: datetime | None, + ended: datetime | None, +) -> bool: """Whether ``event`` is ``job``'s: by check-run id, else by runner and start time.""" recorded = event.get("github") job_id = recorded.get("job_id") if isinstance(recorded, dict) else None if job_id is not None: return job_id == job.get("id") - started, began, ended = ( - _time(value) - for value in (event.get("started_at"), job.get("started_at"), job.get("completed_at")) - ) return ( event.get("runner") is not None and event.get("runner") == job.get("runner_name") From 56473ce319bf5fbb9c3f0be48c865c8a16f55ef6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:00:11 -0500 Subject: [PATCH 007/203] fix(results): compare eval framework and precision case-insensitively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eval collector lowercases framework and precision in aggregate rows, but the eval identity built from a raw meta_env.json kept their original case. A raw eval whose metadata said FP4 or VLLM was then reported both as missing from and unexpected in the aggregate. The identity now lowercases string framework and precision values, so only mixed-case inputs change. 中文:评测收集器会把聚合行中的 framework 和 precision 转为小写,但根据原始 meta_env.json 构建的评测标识保留了原始大小写。因此,元数据写为 FP4 或 VLLM 的原始评测会同时被报告为聚合结果中缺失和多余。现在该标识会把字符串类型的 framework 和 precision 转为小写,只有含大写字母的输入行为发生变化。 --- inferencex-e2e/infx/results/eval_artifacts.py | 13 +++++++++---- .../test_validate_reusable_sweep_artifacts.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/results/eval_artifacts.py b/inferencex-e2e/infx/results/eval_artifacts.py index 790b17d893..1308d697bf 100644 --- a/inferencex-e2e/infx/results/eval_artifacts.py +++ b/inferencex-e2e/infx/results/eval_artifacts.py @@ -31,6 +31,11 @@ def normalized_runner(value: Any) -> str: return str(value or "").lower() +def _lowered(value: Any) -> Any: + """Match aggregate rows, which lowercase framework and precision.""" + return value.lower() if isinstance(value, str) else value + + LEGACY_EVAL_SUITE = "" @@ -47,8 +52,8 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: "multi", normalized_runner(row.get("hw")), row.get("model_prefix", row.get("infmax_model_prefix")), - row.get("framework"), - row.get("precision"), + _lowered(row.get("framework")), + _lowered(row.get("precision")), row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), @@ -78,8 +83,8 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: "single", normalized_runner(row.get("hw")), row.get("model_prefix", row.get("infmax_model_prefix")), - row.get("framework"), - row.get("precision"), + _lowered(row.get("framework")), + _lowered(row.get("precision")), row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), diff --git a/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py b/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py index cf63970a28..9a07b76023 100644 --- a/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py +++ b/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py @@ -409,6 +409,19 @@ def write_agentic_artifacts( (root / f"agentic_{result_name}").mkdir() +def test_eval_validation_matches_mixed_case_raw_meta_to_lowercased_aggregate( + tmp_path: Path, +) -> None: + write_raw_eval_artifact(tmp_path, 32) + meta_path = tmp_path / "eval_result_conc32_h100-dgxc-slurm_00" / "meta_env.json" + meta_path.write_text( + json.dumps({**single_eval_meta(32), "framework": "VLLM", "precision": "FP4"}) + ) + write_eval_aggregate(tmp_path, [single_eval_result(32)]) + + assert validate_eval_artifacts(tmp_path) == [] + + def test_eval_validation_accepts_legacy_results_alongside_debug_artifacts( tmp_path: Path, ) -> None: From 48bbbe9dd13a9aea9f12cdb2deb96939f697a880 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:05 -0500 Subject: [PATCH 008/203] fix(results): include spec decoding in single-node agentic identities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single-node agentic identity left out spec_decoding and disagg, while the multinode identity and the fixed-sequence identity include them. Two single-node points that differ only in speculative decoding therefore collided, so a later run attempt of one dropped the other as superseded and validation could report false duplicates. Single-node agentic identities now carry both fields, which changes only points that differ in them. 中文:单节点 agentic 标识遗漏了 spec_decoding 和 disagg,而多节点标识和固定序列标识都包含这两个字段。因此,仅在推测解码上不同的两个单节点测试点会发生标识冲突,其中一个的后续 run attempt 会把另一个当作已被取代而丢弃,校验也可能报告虚假的重复项。现在单节点 agentic 标识包含这两个字段,只有在这两个字段上不同的测试点行为发生变化。 --- inferencex-e2e/infx/results/artifacts.py | 2 ++ .../infx/tests/results/test_agentic_point_files.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/inferencex-e2e/infx/results/artifacts.py b/inferencex-e2e/infx/results/artifacts.py index d1386a4e2e..2f45b1d232 100644 --- a/inferencex-e2e/infx/results/artifacts.py +++ b/inferencex-e2e/infx/results/artifacts.py @@ -172,6 +172,8 @@ def agentic_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("infmax_model_prefix"), row.get("framework"), row.get("precision"), + row.get("spec_decoding", "none"), + as_bool(row.get("disagg", False)), as_int(row.get("tp")), as_int(row.get("pp", 1), 1), as_int(row.get("dcp_size", 1), 1), diff --git a/inferencex-e2e/infx/tests/results/test_agentic_point_files.py b/inferencex-e2e/infx/tests/results/test_agentic_point_files.py index d546ddaedb..d61c218828 100644 --- a/inferencex-e2e/infx/tests/results/test_agentic_point_files.py +++ b/inferencex-e2e/infx/tests/results/test_agentic_point_files.py @@ -24,3 +24,9 @@ def test_keeps_files_without_run_attempt(tmp_path: Path): first = write(tmp_path, "old_runner", ROW) second = write(tmp_path, "new_runner", {**ROW, "run_attempt": 2}) assert agentic_point_files(tmp_path) == sorted([first, second]) + + +def test_rerun_does_not_supersede_a_point_with_other_spec_decoding(tmp_path: Path): + mtp = write(tmp_path, "mtp_a1_runner", {**ROW, "spec_decoding": "mtp", "run_attempt": 1}) + plain = write(tmp_path, "none_a2_runner", {**ROW, "spec_decoding": "none", "run_attempt": 2}) + assert agentic_point_files(tmp_path) == sorted([mtp, plain]) From e6051b3cf320ad7231ca6b9e5ed1cb1c0fd1c7e3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:56:24 -0500 Subject: [PATCH 009/203] fix(evals): fail non-finite and boolean eval scores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_scores compared each score with `val < min_score`, which is false for NaN, so a result file with a NaN score printed PASS and exited 0. Infinite and boolean values also passed. Scores that are NaN, infinite or a JSON boolean now print a FAIL line and fail the run, while finite numeric scores are thresholded as before. 中文:validate_scores 用 `val < min_score` 比较分数,而 NaN 的比较结果为假,因此分数为 NaN 的结果文件会打印 PASS 并以 0 退出。无穷大和布尔值同样会通过。现在 NaN、无穷大或 JSON 布尔值分数会打印 FAIL 行并使运行失败,有限数值分数仍按原有阈值检查。 --- inferencex-e2e/infx/evals/validate_scores.py | 8 +++++- .../infx/tests/evals/test_batched_eval.py | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index a50073c030..d23d4aa27d 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -352,7 +352,13 @@ def main() -> int: if not isinstance(val, (int, float)): continue checked += 1 - if val < min_score: + if isinstance(val, bool) or not math.isfinite(val): + print( + f"FAIL: {conc_label}{task} {name} = {val} is not a finite number", + file=sys.stderr, + ) + failed = True + elif val < min_score: print( f"FAIL: {conc_label}{task} {name} = {val:.4f} (< {min_score} from {source})", file=sys.stderr, diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index 245e22dc1b..902e82062c 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +import pytest + from infx.evals.validate_scores import main as validate_scores_main from infx.evals.validate_scores import validate_batch_manifest @@ -212,3 +214,29 @@ def test_validate_scores_accepts_legacy_result_without_effective_count( assert validate_scores_main() == 0 captured = capsys.readouterr() assert "PASS: gsm8k exact_match,strict-match" in captured.out + + +@pytest.mark.parametrize( + ("score", "shown"), [(float("nan"), "nan"), (float("inf"), "inf"), (True, "True")] +) +def test_validate_scores_fails_a_non_finite_or_boolean_score( + tmp_path: Path, + monkeypatch, + capsys, + score: object, + shown: str, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({"results": {"gsm8k": {"exact_match,strict-match": score}}})) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: gsm8k exact_match,strict-match = {shown} is not a finite number" in captured.err + assert "PASS" not in captured.out From 9427c856d98f49e22eccd47b3b44f9d18153e768 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:56:51 -0500 Subject: [PATCH 010/203] fix(evals): fail validation when the thresholds file cannot be loaded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the thresholds file was missing, unreadable or invalid, validate_scores printed a WARN and kept going with no configured thresholds. Every task then used the 0.85 --min-score fallback instead of its stricter configured minimum, so a gsm8k score of 0.87 passed. A thresholds load failure now prints a FAIL line and exits 1. Runs with a loadable thresholds file, or with no thresholds file at all, behave as before. 中文:当阈值文件缺失、不可读或无效时,validate_scores 只打印 WARN 并在没有任何阈值配置的情况下继续执行。于是每个任务都使用 0.85 的 --min-score 兜底值,而不是更严格的配置最小值,导致 0.87 的 gsm8k 分数也能通过。现在阈值加载失败会打印 FAIL 行并以 1 退出。阈值文件可正常加载或根本不存在阈值文件的运行,行为保持不变。 --- inferencex-e2e/infx/evals/validate_scores.py | 3 ++- .../infx/tests/evals/test_batched_eval.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index d23d4aa27d..e98745ef39 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -287,9 +287,10 @@ def main() -> int: print(f"Loaded thresholds from {thresholds_path}") except (json.JSONDecodeError, OSError, ValueError) as e: print( - f"WARN: could not load thresholds from {thresholds_path}: {e}", + f"FAIL: could not load thresholds from {thresholds_path}: {e}", file=sys.stderr, ) + return 1 prefix = detect_model_prefix(args.meta_env, args.model_prefix) if prefix and prefix in config.get("models", {}): diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index 902e82062c..3c62577d13 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -240,3 +240,28 @@ def test_validate_scores_fails_a_non_finite_or_boolean_score( captured = capsys.readouterr() assert f"FAIL: gsm8k exact_match,strict-match = {shown} is not a finite number" in captured.err assert "PASS" not in captured.out + + +def test_validate_scores_fails_when_thresholds_cannot_be_loaded( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + thresholds = tmp_path / "thresholds.yaml" + thresholds.write_text("default: [gsm8k\n") + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.87}}})) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--thresholds", + str(thresholds), + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: could not load thresholds from {thresholds}: invalid YAML" in captured.err + assert "PASS" not in captured.out From dd9179a1c3c13fd9f44a6b9388480c472735177f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:57:24 -0500 Subject: [PATCH 011/203] fix(evals): treat empty threshold sections as empty and reject non-mappings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thresholds YAML with an empty `models:` or `default:` key loads that key as None, and validate_scores then crashed with a TypeError or AttributeError traceback instead of validating. An empty section is now treated as an empty mapping, so the other section still applies. A section that is neither empty nor a mapping, such as a list, is reported as a FAIL thresholds load error. 中文:阈值 YAML 中若 `models:` 或 `default:` 键为空,加载结果为 None,validate_scores 随后会抛出 TypeError 或 AttributeError 回溯而不是进行校验。现在空的配置段被视为空映射,另一个配置段仍然生效。既非空也不是映射的配置段(例如列表)会被报告为 FAIL 的阈值加载错误。 --- inferencex-e2e/infx/evals/validate_scores.py | 6 ++- .../infx/tests/evals/test_batched_eval.py | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index e98745ef39..7b217e7504 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -38,7 +38,11 @@ def load_config(path: str) -> dict: raise ValueError("thresholds config must be a mapping") if "default" not in cfg and "models" not in cfg: return {"default": cfg, "models": {}} - return {"default": cfg.get("default", {}), "models": cfg.get("models", {})} + sections = {"default": cfg.get("default") or {}, "models": cfg.get("models") or {}} + for name, section in sections.items(): + if not isinstance(section, dict): + raise ValueError(f"thresholds config {name!r} must be a mapping") + return sections def detect_model_prefix(meta_env_path: str, override: str | None) -> str | None: diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index 3c62577d13..bf1159216b 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -265,3 +265,43 @@ def test_validate_scores_fails_when_thresholds_cannot_be_loaded( captured = capsys.readouterr() assert f"FAIL: could not load thresholds from {thresholds}: invalid YAML" in captured.err assert "PASS" not in captured.out + + +@pytest.mark.parametrize( + ("thresholds_text", "expected"), + [ + ( + "default:\n gsm8k: 0.9\nmodels:\n", + "FAIL: gsm8k exact_match,strict-match = 0.8700 (< 0.9 from default)", + ), + ( + "default:\n gsm8k: 0.9\nmodels: [dsr1]\n", + "thresholds config 'models' must be a mapping", + ), + ], +) +def test_validate_scores_handles_empty_or_malformed_threshold_sections( + tmp_path: Path, + monkeypatch, + capsys, + thresholds_text: str, + expected: str, +) -> None: + thresholds = tmp_path / "thresholds.yaml" + thresholds.write_text(thresholds_text) + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.87}}})) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--thresholds", + str(thresholds), + "--model-prefix", + "dsr1", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + assert expected in capsys.readouterr().err From 4a0ac68a76e8434ee9909375d6559482201f0eca Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:58:00 -0500 Subject: [PATCH 012/203] fix(evals): report malformed result files as failures and keep checking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A truncated results*.json, a top-level value that is not an object, or a null "results" field made validate_scores crash with a traceback. The crash printed no FAIL line and skipped every remaining result file. Each such file now gets a FAIL line naming it, the run exits 1, and the remaining files are still thresholded. Well-formed result files are unaffected. 中文:截断的 results*.json、顶层不是对象的值,或值为 null 的 "results" 字段,都会让 validate_scores 抛出回溯而崩溃。崩溃时不会打印 FAIL 行,并跳过其余所有结果文件。现在每个此类文件都会得到一条注明文件名的 FAIL 行,运行以 1 退出,其余文件仍会进行阈值检查。格式正确的结果文件不受影响。 --- inferencex-e2e/infx/evals/validate_scores.py | 26 +++++++++++--- .../infx/tests/evals/test_batched_eval.py | 34 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index 7b217e7504..17d3bd1cf7 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -330,9 +330,18 @@ def main() -> int: for f in result_files: match = CONC_SUFFIX_RE.search(Path(f).name) conc_label = f"[conc={match.group(1)}] " if match else "" - with open(f) as fh: - data = json.load(fh) - if "integration_error" in data: + try: + with open(f) as fh: + data = json.load(fh) + except json.JSONDecodeError as exc: + print(f"FAIL: unreadable result file {f}: invalid JSON: {exc}", file=sys.stderr) + failed = True + continue + except OSError as exc: + print(f"FAIL: unreadable result file {f}: {exc}", file=sys.stderr) + failed = True + continue + if isinstance(data, dict) and "integration_error" in data: print( f"FAIL: {conc_label}integration failure: " f"{integration_error_message(data['integration_error'])}", @@ -340,7 +349,16 @@ def main() -> int: ) failed = True continue - for task, metrics in data.get("results", {}).items(): + results = data.get("results", {}) if isinstance(data, dict) else None + if not isinstance(results, dict): + print( + f"FAIL: unreadable result file {f}: " + "expected a JSON object with a 'results' mapping", + file=sys.stderr, + ) + failed = True + continue + for task, metrics in results.items(): invalid_effective, effective = invalid_effective_count(data, task) if invalid_effective: print( diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index bf1159216b..8c957b7cf8 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -305,3 +305,37 @@ def test_validate_scores_handles_empty_or_malformed_threshold_sections( assert validate_scores_main() == 1 assert expected in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("content", "reason"), + [ + ('{"results": {"gsm8k": ', "invalid JSON"), + ("[1]", "expected a JSON object with a 'results' mapping"), + ('{"results": null}', "expected a JSON object with a 'results' mapping"), + ], +) +def test_validate_scores_fails_a_malformed_result_file_and_checks_the_rest( + tmp_path: Path, + monkeypatch, + capsys, + content: str, + reason: str, +) -> None: + bad = tmp_path / "results_a.json" + bad.write_text(content) + (tmp_path / "results_b.json").write_text( + json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.95}}}) + ) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(tmp_path / "results*.json"), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: unreadable result file {bad}: {reason}" in captured.err + assert "PASS: gsm8k exact_match,strict-match = 0.9500" in captured.out From ae58bfbb408dfaff93f84441a8689eccb8caf3c8 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:59:05 -0500 Subject: [PATCH 013/203] fix(evals): fail each result file that contributes no checked metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_scores counted thresholded metrics with one global counter. In a batched eval, a concurrency whose result file had no matching score metric was silently ignored as long as another file contributed one. Every result file that yields no checked metric now gets a FAIL line naming it, and the run exits 1. Runs where every file contributes at least one checked metric are unchanged. 中文:validate_scores 用一个全局计数器统计已做阈值检查的指标。在批量评测中,只要另一个文件提供了匹配指标,某个并发度的结果文件即使没有任何匹配的分数指标也会被静默忽略。现在凡是没有产生任何已检查指标的结果文件,都会得到一条注明文件名的 FAIL 行,运行以 1 退出。每个文件都至少贡献一个已检查指标的运行保持不变。 --- inferencex-e2e/infx/evals/validate_scores.py | 6 +++- .../infx/tests/evals/test_batched_eval.py | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index 17d3bd1cf7..9e454a73b0 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -306,6 +306,7 @@ def main() -> int: failed = False checked = 0 + selector = args.metric_prefix or "declared task metrics" result_files = sorted(glob.glob(args.results_glob)) manifest_errors = validate_batch_manifest( @@ -358,6 +359,7 @@ def main() -> int: ) failed = True continue + file_checked = checked for task, metrics in results.items(): invalid_effective, effective = invalid_effective_count(data, task) if invalid_effective: @@ -391,9 +393,11 @@ def main() -> int: print( f"PASS: {conc_label}{task} {name} = {val:.4f} (>= {min_score} from {source})" ) + if checked == file_checked: + print(f"FAIL: {conc_label}no metrics matched {selector!r} in {f}", file=sys.stderr) + failed = True if checked == 0: - selector = args.metric_prefix or "declared task metrics" print(f"WARN: no metrics matched {selector!r}", file=sys.stderr) return 1 if (failed or checked == 0) else 0 diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index 8c957b7cf8..a19d3b6de9 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -339,3 +339,36 @@ def test_validate_scores_fails_a_malformed_result_file_and_checks_the_rest( captured = capsys.readouterr() assert f"FAIL: unreadable result file {bad}: {reason}" in captured.err assert "PASS: gsm8k exact_match,strict-match = 0.9500" in captured.out + + +def test_validate_scores_fails_a_result_file_with_no_checked_metric( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "eval_concs": [1, 4], + "completed_eval_concs": [1, 4], + "failed_eval_concs": [], + })) + (tmp_path / "results_test_conc1.json").write_text( + json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.95}}}) + ) + (tmp_path / "results_test_conc4.json").write_text( + json.dumps({"results": {"gsm8k": {"acc,none": 0.95}}}) + ) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(tmp_path / "results*.json"), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "PASS: [conc=1] gsm8k exact_match,strict-match = 0.9500" in captured.out + assert ( + "FAIL: [conc=4] no metrics matched 'declared task metrics' in " + f"{tmp_path / 'results_test_conc4.json'}" + ) in captured.err From 2d81f5711dea38069517223d907102a2b4f6cc5d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:01:05 -0500 Subject: [PATCH 014/203] fix(evals): split MiniMax native results on newlines only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pinned MiniMax verifier writes results.jsonl rows with ensure_ascii=False, so model text containing U+2028, U+2029 or U+0085 lands in the file unescaped. str.splitlines() treats those characters as line breaks, so a valid 102-row full run, or the one-row smoke run, was rejected as having the wrong row count. Rows are now split on "\n" only. Files without those characters produce the same rows as before. 中文:固定版本的 MiniMax 验证器以 ensure_ascii=False 写入 results.jsonl 的各行,因此包含 U+2028、U+2029 或 U+0085 的模型文本会以未转义形式写入文件。str.splitlines() 会把这些字符当作换行,导致有效的 102 行完整运行或单行冒烟运行因行数不符而被拒绝。现在只按 "\n" 拆分行。不含这些字符的文件得到的行与之前相同。 --- .../infx/evals/minimax_m3_full_eval.py | 2 +- .../infx/evals/minimax_provider_eval.py | 3 ++- .../tests/evals/test_minimax_m3_full_eval.py | 17 +++++++++++++++++ .../tests/evals/test_minimax_provider_eval.py | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py index 303bc5c401..ade23de791 100755 --- a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py @@ -294,7 +294,7 @@ def _read_native_report(path: Path) -> dict[str, Any]: def _read_native_results(path: Path) -> list[dict[str, Any]]: try: - lines = path.read_text(encoding="utf-8").splitlines() + lines = path.read_text(encoding="utf-8").removesuffix("\n").split("\n") except OSError as exc: raise FullSuiteError(f"native results are unavailable: {exc}") from exc if len(lines) != EXPECTED_RESULT_COUNT or any(not line.strip() for line in lines): diff --git a/inferencex-e2e/infx/evals/minimax_provider_eval.py b/inferencex-e2e/infx/evals/minimax_provider_eval.py index 5fd71f7a26..8f6c07d9d2 100755 --- a/inferencex-e2e/infx/evals/minimax_provider_eval.py +++ b/inferencex-e2e/infx/evals/minimax_provider_eval.py @@ -227,7 +227,8 @@ def project_native_artifacts(*, output_dir: Path, model: str) -> Path: json.loads((output_dir / NATIVE_REPORT_FILENAME).read_text(encoding="utf-8")), "native summary", ) - result_lines = (output_dir / NATIVE_RESULTS_FILENAME).read_text(encoding="utf-8").splitlines() + result_text = (output_dir / NATIVE_RESULTS_FILENAME).read_text(encoding="utf-8") + result_lines = result_text.removesuffix("\n").split("\n") if len(result_lines) != 1 or not result_lines[0].strip(): raise SmokeSuiteError("native results must contain exactly one row") result = _mapping(json.loads(result_lines[0]), "native result") diff --git a/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py b/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py index 35f32bd1a3..163a210ead 100644 --- a/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py @@ -271,6 +271,23 @@ def test_projection_rejects_incomplete_or_transport_failed_native_results( full.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3") +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85"]) +def test_projection_counts_rows_whose_text_holds_unicode_line_separators( + tmp_path: Path, separator: str +) -> None: + _write_native_run(tmp_path) + rows = [{"data_index": index, "status": "success"} for index in range(1, 103)] + rows[4]["response"] = f"first{separator}second" + (tmp_path / full.NATIVE_RESULTS_FILENAME).write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8" + ) + + compatibility_path = full.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3") + + compatibility = json.loads(compatibility_path.read_text(encoding="utf-8")) + assert compatibility["n-samples"]["minimax_m3_full"] == {"original": 102, "effective": 102} + + def test_upstream_process_failure_is_nonzero_and_publishes_all_failure_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py b/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py index 87c3e7f5ff..3333e7b8b0 100644 --- a/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py @@ -48,6 +48,21 @@ def _compatibility(output_dir: Path) -> dict[str, Any]: return json.loads(paths[0].read_text(encoding="utf-8")) +def test_projection_reads_a_row_whose_text_holds_a_unicode_line_separator( + tmp_path: Path, +) -> None: + _native_outputs(tmp_path, match_rate=0.5) + (tmp_path / mpe.NATIVE_RESULTS_FILENAME).write_text( + json.dumps({"data_index": 1, "status": "success", "response": "a\u2028b"}, ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + + mpe.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3") + + assert _compatibility(tmp_path)["results"][mpe.TASK_NAME]["exact_match,strict-match"] == 0.5 + + def test_prepare_smoke_input_preserves_request_and_unicode( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 0df936ec99941a2e0c57e71e0506fb654aa1c164 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:16 -0500 Subject: [PATCH 015/203] fix(evals): retry a MiniMax source download whose SHA256 does not match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _fetch_source retried network errors, but it checked the pinned SHA256 only after leaving the retry loop. http.client returns a short body without raising when a sized read loses its connection, so a truncated download failed the run on the first attempt. The digest check now runs inside the retry loop. A mismatched download is retried like any other transient failure, and a persistent mismatch fails with the existing "after N attempts" error. 中文:_fetch_source 会对网络错误进行重试,但它只在离开重试循环之后才校验固定的 SHA256。带长度的读取在连接中断时,http.client 会返回不完整的内容而不抛异常,因此一次被截断的下载会在第一次尝试时就让运行失败。现在摘要校验在重试循环内进行。哈希不匹配的下载会像其他瞬时故障一样重试,持续不匹配则以现有的 "after N attempts" 错误失败。 --- .../infx/evals/minimax_m3_full_eval.py | 2 +- .../tests/evals/test_minimax_m3_full_eval.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py index ade23de791..b0b3eb1077 100755 --- a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py @@ -111,6 +111,7 @@ def _fetch_source(relative_path: str) -> bytes: if declared_size is not None and int(declared_size) > MAX_SOURCE_BYTES: raise FullSuiteError(f"pinned source {relative_path} exceeds the size limit") content = response.read(MAX_SOURCE_BYTES + 1) + verify_source_content(relative_path, content) except ( FullSuiteError, OSError, @@ -122,7 +123,6 @@ def _fetch_source(relative_path: str) -> bytes: time.sleep(DOWNLOAD_RETRY_DELAY_SECONDS) continue break - verify_source_content(relative_path, content) return content raise FullSuiteError( f"failed to download pinned source {relative_path} after " diff --git a/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py b/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py index 163a210ead..1a8c1ced97 100644 --- a/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py @@ -138,6 +138,40 @@ def open(self, request, *, timeout): assert sleeps == [0.5] +def test_fetch_source_retries_a_truncated_download(monkeypatch: pytest.MonkeyPatch) -> None: + content = b"pinned source" + monkeypatch.setattr( + full, "REQUIRED_SOURCE_SHA256", {"verify.py": hashlib.sha256(content).hexdigest()} + ) + monkeypatch.setattr(full.time, "sleep", lambda _: None) + bodies = [content[:6], content] + + class Response: + status = 200 + headers: dict[str, str] = {} + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def geturl(self) -> str: + return full.source_url("verify.py") + + def read(self, _: int) -> bytes: + return bodies.pop(0) + + class Opener: + def open(self, request, *, timeout): + return Response() + + monkeypatch.setattr(full, "_NO_REDIRECT_OPENER", Opener()) + + assert full._fetch_source("verify.py") == content + assert bodies == [] + + def test_verifier_command_is_one_102_row_run_with_fixed_m3_settings( tmp_path: Path, ) -> None: From 9d9f13f72f2bc8c6972ed1e91a3ea8c9e788da0e Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:03:08 -0500 Subject: [PATCH 016/203] fix(evals): retry a Kimi verifier archive download that was cut short MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archive download loop retried HTTP and connection errors, but it compared the SHA256 only after the loop. A body that ended early without an error, which http.client allows for sized reads, failed the run on the first attempt. A chunked body that ended early raised IncompleteRead, which also skipped the retry. A digest mismatch and IncompleteRead are now retried within the existing three attempts. A mismatch on the last attempt still fails with the existing SHA256 mismatch error. 中文:归档下载循环会对 HTTP 和连接错误进行重试,但只在循环结束后才比较 SHA256。若响应体在没有报错的情况下提前结束(http.client 对带长度的读取允许这种情况),运行会在第一次尝试时就失败。提前结束的分块响应会抛出 IncompleteRead,同样跳过了重试。现在摘要不匹配和 IncompleteRead 都会在现有的三次尝试内重试。最后一次尝试仍不匹配时,依旧以现有的 SHA256 不匹配错误失败。 --- .../infx/evals/_kimi_verifier_archive.py | 13 ++++++-- .../tests/evals/test_kimi_verifier_archive.py | 32 +++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py index 29973d952e..93b19bb4a6 100644 --- a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py @@ -6,6 +6,7 @@ import tempfile import time from hashlib import sha256 +from http.client import IncompleteRead from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import quote, urlsplit, urlunsplit @@ -89,7 +90,6 @@ def main() -> None: ) archive_file.write(chunk) digest.update(chunk) - break except HTTPError as error: if error.code not in (408, 429) and not 500 <= error.code < 600: raise @@ -101,7 +101,7 @@ def main() -> None: file=sys.stderr, ) time.sleep(attempt) - except (TimeoutError, URLError, ConnectionError) as error: + except (TimeoutError, URLError, ConnectionError, IncompleteRead) as error: if attempt == 3: raise print( @@ -110,6 +110,15 @@ def main() -> None: file=sys.stderr, ) time.sleep(attempt) + else: + if digest.hexdigest() == expected_archive_sha256 or attempt == 3: + break + print( + f"WARN: Kimi-Vendor-Verifier archive download attempt " + f"{attempt}/3 failed: archive SHA256 mismatch; retrying", + file=sys.stderr, + ) + time.sleep(attempt) if downloaded == 0: raise ValueError("downloaded archive is empty") actual_archive_sha256 = digest.hexdigest() diff --git a/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py b/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py index f2429cc8fa..125b47e529 100644 --- a/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py @@ -66,7 +66,9 @@ def _archive(*, missing: str | None = None, extra: tarfile.TarInfo | None = None @contextmanager -def _serve(payload: bytes, *, transient_failures: int = 0) -> Iterator[tuple[str, list[str]]]: +def _serve( + payload: bytes, *, transient_failures: int = 0, truncate: str | None = None +) -> Iterator[tuple[str, list[str]]]: paths: list[str] = [] class Handler(BaseHTTPRequestHandler): @@ -77,8 +79,16 @@ def do_GET(self) -> None: self.end_headers() return self.send_response(200) + if truncate == "chunked" and len(paths) == transient_failures + 1: + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(f"{len(payload):x}\r\n".encode() + payload[:10]) + return self.send_header("Content-Length", str(len(payload))) self.end_headers() + if truncate == "short" and len(paths) == transient_failures + 1: + self.wfile.write(payload[: len(payload) // 2]) + return self.wfile.write(payload) def log_message(self, *args: object) -> None: @@ -102,11 +112,15 @@ def _fetch( *, sha256: str | None = None, transient_failures: int = 0, + truncate: str | None = None, ) -> list[str]: """Fetch ``payload`` from a local archive server into ``checkout``; return the paths asked.""" checkout.mkdir() monkeypatch.setattr(archive.time, "sleep", lambda _: None) - with _serve(payload, transient_failures=transient_failures) as (repo_url, paths): + with _serve(payload, transient_failures=transient_failures, truncate=truncate) as ( + repo_url, + paths, + ): digest = sha256 or hashlib.sha256(payload).hexdigest() monkeypatch.setattr(sys, "argv", ["archive", repo_url, REF, digest, str(checkout)]) archive.main() @@ -127,6 +141,20 @@ def test_extracts_only_the_required_subset_after_a_transient_server_error( assert (checkout / "pyproject.toml").read_text() == "pyproject.toml" +@pytest.mark.parametrize("truncate", ["short", "chunked"]) +def test_retries_a_download_cut_short_by_the_connection( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], truncate: str +) -> None: + checkout = tmp_path / "checkout" + + paths = _fetch(monkeypatch, checkout, _archive(), truncate=truncate) + + assert len(paths) == 2 + assert "archive download attempt 1/3 failed" in capsys.readouterr().err + extracted = {p.relative_to(checkout).as_posix() for p in checkout.rglob("*") if p.is_file()} + assert extracted == REQUIRED_FILES + + def _escaping_member() -> tarfile.TarInfo: member = tarfile.TarInfo("verifier-pinned/../../escaped") member.size = 6 From f9c90e5394bbf138979dcac1d910db52b58b28ca Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:03:37 -0500 Subject: [PATCH 017/203] fix(evals): compare the Kimi archive SHA256 case-insensitively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archive fetcher accepts an expected SHA256 in upper or lower case, but it compared that value with the lowercase hexdigest. An uppercase pin therefore always failed as a mismatch, even for the correct archive. The expected digest is now lowercased after validation. Lowercase pins, including the current vendor pin, behave as before. 中文:归档获取脚本接受大写或小写的预期 SHA256,却把它与小写的 hexdigest 比较。因此大写的固定值即使对应正确的归档也总会被判为不匹配。现在预期摘要在校验后统一转为小写。小写固定值(包括当前的 vendor 固定值)行为不变。 --- inferencex-e2e/infx/evals/_kimi_verifier_archive.py | 1 + .../infx/tests/evals/test_kimi_verifier_archive.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py index 93b19bb4a6..3cb7743e83 100644 --- a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py @@ -35,6 +35,7 @@ def main() -> None: raise ValueError( f"expected a 64-character archive SHA256, got {expected_archive_sha256!r}" ) + expected_archive_sha256 = expected_archive_sha256.lower() parsed_repo_url = urlsplit(repo_url) if parsed_repo_url.scheme not in ("http", "https") or not parsed_repo_url.netloc: diff --git a/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py b/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py index 125b47e529..d32c8ff2dc 100644 --- a/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py @@ -155,6 +155,18 @@ def test_retries_a_download_cut_short_by_the_connection( assert extracted == REQUIRED_FILES +def test_accepts_an_uppercase_pinned_archive_sha256( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + checkout = tmp_path / "checkout" + payload = _archive() + + paths = _fetch(monkeypatch, checkout, payload, sha256=hashlib.sha256(payload).hexdigest().upper()) + + assert len(paths) == 1 + assert (checkout / "pyproject.toml").read_text() == "pyproject.toml" + + def _escaping_member() -> tarfile.TarInfo: member = tarfile.TarInfo("verifier-pinned/../../escaped") member.size = 6 From b1ac38c08d68ec990b14a74cdfd6f4ccc733f3fd Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:04:55 -0500 Subject: [PATCH 018/203] fix(evals): reject non-numeric score thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_config accepted any threshold value. A NaN minimum made `val < min_score` false for every score, so every task passed. A string minimum or an empty per-model section crashed validate_scores with a TypeError traceback. Each threshold must now be a finite number that is not a boolean, otherwise the run prints a FAIL thresholds load error and exits 1. An empty per-model section is treated as empty, so the default thresholds apply to that model. 中文:load_config 此前接受任意阈值。NaN 最小值会使 `val < min_score` 对所有分数都为假,因而所有任务都会通过。字符串最小值或空的单模型配置段会让 validate_scores 抛出 TypeError 回溯而崩溃。现在每个阈值都必须是有限且非布尔值的数字,否则运行会打印 FAIL 阈值加载错误并以 1 退出。空的单模型配置段被视为空,该模型因此使用默认阈值。 --- inferencex-e2e/infx/evals/validate_scores.py | 17 ++++++++++++++++- .../infx/tests/evals/test_batched_eval.py | 12 ++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index 9e454a73b0..858e63ba1f 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -37,11 +37,26 @@ def load_config(path: str) -> dict: if not isinstance(cfg, dict): raise ValueError("thresholds config must be a mapping") if "default" not in cfg and "models" not in cfg: - return {"default": cfg, "models": {}} + cfg = {"default": cfg} sections = {"default": cfg.get("default") or {}, "models": cfg.get("models") or {}} for name, section in sections.items(): if not isinstance(section, dict): raise ValueError(f"thresholds config {name!r} must be a mapping") + sections["models"] = {prefix: tasks or {} for prefix, tasks in sections["models"].items()} + tables = {"default": sections["default"]} + tables.update((f"models.{prefix}", tasks) for prefix, tasks in sections["models"].items()) + for name, tasks in tables.items(): + if not isinstance(tasks, dict): + raise ValueError(f"thresholds config {name!r} must be a mapping") + for task, value in tasks.items(): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError( + f"thresholds config {name}.{task} must be a finite number, got {value!r}" + ) return sections diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index a19d3b6de9..1fb38e0eb2 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -274,10 +274,22 @@ def test_validate_scores_fails_when_thresholds_cannot_be_loaded( "default:\n gsm8k: 0.9\nmodels:\n", "FAIL: gsm8k exact_match,strict-match = 0.8700 (< 0.9 from default)", ), + ( + "default:\n gsm8k: 0.9\nmodels:\n dsr1:\n", + "FAIL: gsm8k exact_match,strict-match = 0.8700 (< 0.9 from default)", + ), ( "default:\n gsm8k: 0.9\nmodels: [dsr1]\n", "thresholds config 'models' must be a mapping", ), + ( + "default:\n gsm8k: .nan\n", + "thresholds config default.gsm8k must be a finite number, got nan", + ), + ( + "default:\n gsm8k: 0.8\nmodels:\n dsr1:\n gsm8k: '0.9'\n", + "thresholds config models.dsr1.gsm8k must be a finite number, got '0.9'", + ), ], ) def test_validate_scores_handles_empty_or_malformed_threshold_sections( From 41db6cd215d5d30f5946c6ee304f0955200326c4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:58:48 -0500 Subject: [PATCH 019/203] fix(workflows): keep same-named checks from different workflows apart in merge gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wait_for_checks deduplicated check runs by name only, so a failed job named like a passing job in another workflow was dropped and the merge went ahead. Check runs from GitHub Actions are now keyed on name, workflow and triggering event, matching gh pr checks, so re-runs of one workflow still supersede each other. Only commits whose check runs share a name across different workflows or events change behavior. 中文:wait_for_checks 过去只按名称对 check run 去重,因此当另一个工作流中存在同名且已通过的作业时,失败的作业会被丢弃,合并照常进行。现在来自 GitHub Actions 的 check run 按名称、工作流和触发事件区分,与 gh pr checks 一致,同一工作流的重跑仍会覆盖旧结果。只有不同工作流或事件之间存在同名 check run 的提交行为会改变。 --- .../tests/workflows/test_merge_with_reuse.py | 39 +++++++++++++++++++ .../infx/workflows/merge_with_reuse.py | 38 ++++++++++++------ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py index d92e82db8f..cd24d789c9 100644 --- a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py +++ b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py @@ -100,6 +100,7 @@ def make_check_run( started_at: str = "2024-01-01T00:00:00Z", details_url: str = "", cr_id: int = 1, + check_suite_id: int | None = None, ) -> MagicMock: """Build a mock CheckRun.""" cr = MagicMock() @@ -109,6 +110,7 @@ def make_check_run( cr.started_at = started_at cr.details_url = details_url cr.id = cr_id + cr.check_suite_id = check_suite_id return cr @@ -541,6 +543,43 @@ def test_transient_error_retried_then_succeeds(self): result = wait_for_checks(pull, sha, gh_repo, timeout=60) assert result == 0 + @pytest.mark.parametrize( + ("second_workflow_id", "expected"), + [(2, 1), (1, 0)], + ids=["other-workflow-failure-blocks", "same-workflow-rerun-supersedes"], + ) + def test_same_job_name_in_two_workflows(self, second_workflow_id, expected): + gh_repo = MagicMock() + commit = MagicMock() + gh_repo.get_commit.return_value = commit + commit.get_check_runs.return_value = [ + make_check_run( + name="setup", + conclusion="failure", + started_at="2024-01-01T00:00:00Z", + cr_id=10, + check_suite_id=100, + ), + make_check_run( + name="setup", + conclusion="success", + started_at="2024-01-01T01:00:00Z", + cr_id=20, + check_suite_id=200, + ), + ] + gh_repo.get_workflow_runs.return_value = [ + SimpleNamespace(check_suite_id=100, workflow_id=1, event="pull_request"), + SimpleNamespace( + check_suite_id=200, workflow_id=second_workflow_id, event="pull_request" + ), + ] + commit.get_combined_status.return_value = SimpleNamespace(statuses=[]) + + result = wait_for_checks(MagicMock(), "a" * 40, gh_repo, timeout=10) + + assert result == expected + diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index d3514976f6..221959d4a9 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -279,18 +279,21 @@ def find_eligible_run( return None -def _latest_check_runs(check_runs: list) -> list: - """Dedupe check runs by name, keeping the one with the latest started_at/id. +def _latest_check_runs(check_runs: list, suite_workflows: dict | None = None) -> list: + """Dedupe check runs, keeping the one with the latest started_at/id. GitHub can return superseded re-runs for the same check name. This mirrors - ``gh pr checks --watch --fail-fast`` which only considers the latest run - per check name. + ``gh pr checks --watch --fail-fast``, which keys Actions check runs on + name, workflow and event, and other check runs on name alone. + *suite_workflows* maps a check suite id to its ``(workflow_id, event)``. """ - by_name: dict[str, object] = {} + suite_workflows = suite_workflows or {} + by_key: dict[tuple, object] = {} for cr in check_runs: - existing = by_name.get(cr.name) + key = (cr.name, suite_workflows.get(getattr(cr, "check_suite_id", None))) + existing = by_key.get(key) if existing is None: - by_name[cr.name] = cr + by_key[key] = cr continue cr_key = (str(cr.started_at or ""), getattr(cr, "id", 0)) ex_key = ( @@ -298,8 +301,16 @@ def _latest_check_runs(check_runs: list) -> list: getattr(existing, "id", 0), ) if cr_key > ex_key: - by_name[cr.name] = cr - return list(by_name.values()) + by_key[key] = cr + return list(by_key.values()) + + +def _suite_workflows(gh_repo: Repository, sha: str) -> dict: + """Map each Actions check suite on *sha* to its ``(workflow_id, event)``.""" + return { + run.check_suite_id: (run.workflow_id, run.event) + for run in gh_repo.get_workflow_runs(head_sha=sha) + } def _latest_statuses(statuses: list) -> list: @@ -344,8 +355,9 @@ def wait_for_checks( is treated as pending (the check needs re-evaluation). ``skipped`` and ``neutral`` count as passing. - Check runs are deduped by name (latest ``started_at``/``id`` wins) and - commit statuses by context to ignore superseded re-runs. + Check runs are deduped by name, workflow and event (latest + ``started_at``/``id`` wins) and commit statuses by context to ignore + superseded re-runs. Transient API errors (5xx, rate limits, connection failures) are retried with backoff until the timeout expires. @@ -359,7 +371,9 @@ def wait_for_checks( while time.monotonic() < deadline: try: - check_runs = _latest_check_runs(list(commit.get_check_runs())) + check_runs = _latest_check_runs( + list(commit.get_check_runs()), _suite_workflows(gh_repo, sha) + ) all_completed = True for cr in check_runs: From c4f4db17a9980b67fcefbb9bf82f72c3ae7bad05 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:59:35 -0500 Subject: [PATCH 020/203] fix(workflows): drop only main's new entries when resolving changelog conflicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_conflict_bytes skipped any PR entry whose content matched an entry anywhere in main, so a PR that repeats a historical entry's keys and description lost its entry or failed with "no changelog contribution remaining". It now compares PR entries only with the entries main appended after the merge base. Only PR entries that duplicate a pre-merge-base entry change behavior, and they are now kept. 中文:resolve_conflict_bytes 过去会跳过与 main 中任意条目内容相同的 PR 条目,因此当 PR 重复某个历史条目的 config-keys 和描述时,该条目会丢失,或报错 "no changelog contribution remaining"。现在只与 main 在合并基点之后追加的条目比较。只有与合并基点之前条目重复的 PR 条目行为会改变,它们现在会被保留。 --- .../test_prepare_perf_changelog_merge.py | 15 +++++++++++++++ .../workflows/prepare_perf_changelog_merge.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py b/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py index cc1fef602b..37b69704a8 100644 --- a/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py +++ b/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py @@ -131,6 +131,21 @@ def test_conflict_resolution_rejects_duplicate_remaining_contribution() -> None: ) +def test_conflict_resolution_keeps_pr_entry_repeating_a_historical_entry() -> None: + base = block("same-config", "https://github.com/SemiAnalysisAI/InferenceX/pull/1") + pr = base + b"\n" + block("same-config", "XXX") + main = base + b"\n" + block( + "main-config", + "https://github.com/SemiAnalysisAI/InferenceX/pull/41", + ) + + result = resolve_conflict_bytes(base, pr, main, 42, "SemiAnalysisAI/InferenceX") + + assert result == main + b"\n" + block( + "same-config", "https://github.com/SemiAnalysisAI/InferenceX/pull/42" + ) + + def test_conflict_resolution_separates_multiple_contributions_by_one_blank_line() -> None: base = block("base-config", "https://github.com/SemiAnalysisAI/InferenceX/pull/1") pr = base + b"\n" + block("new-a", "XXX") + b"\n" + block("new-b", "XXX") diff --git a/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py b/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py index 79ccdc71c0..a42ba71fa6 100644 --- a/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py +++ b/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py @@ -152,7 +152,7 @@ def resolve_conflict_bytes( expected_link = f"https://github.com/{repo}/pull/{pr_number}" if additions: - main_signatures = {entry_signature(entry) for entry in main_entries} + main_signatures = {entry_signature(entry) for entry in main_entries[len(base_entries) :]} pr_spans = entry_spans(pr_raw, pr_entries) contribution_blocks: list[bytes] = [] for index, entry in enumerate( From 8765196224b269f9b955527cc4162c584205f1dc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:02 -0500 Subject: [PATCH 021/203] fix(github): read the HTTP status when gh prints a hint after it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit APIError only recognised "(HTTP nnn)" at the very end of gh's stderr, but gh prints scope and SSO hints on the following lines, so status became None and status-based fallbacks such as the 404 handling in signoff_publish re-raised instead. The status is now taken from the end of any stderr line. Only gh failures with trailing hint lines change behavior, and their status is now set. 中文:APIError 过去只识别 gh stderr 最末尾的 "(HTTP nnn)",但 gh 会在其后的行打印权限范围或 SSO 提示,导致 status 为 None,signoff_publish 中针对 404 的回退等基于状态码的处理会直接重新抛出异常。现在从 stderr 任意一行的行尾读取状态码。只有带尾随提示行的 gh 失败行为会改变,它们的 status 现在会被正确设置。 --- inferencex-e2e/infx/github.py | 3 ++- inferencex-e2e/infx/tests/test_github.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/github.py b/inferencex-e2e/infx/github.py index 91a9a57a28..7aade6e5f7 100644 --- a/inferencex-e2e/infx/github.py +++ b/inferencex-e2e/infx/github.py @@ -19,7 +19,8 @@ class APIError(RuntimeError): """Authenticated gh failure, with an HTTP status when gh reports one.""" def __init__(self, path: str, stderr: str) -> None: - match = re.search(r"\(HTTP ([1-5][0-9]{2})\)\s*$", stderr) + # gh can print scope or SSO hints after its "(HTTP nnn)" line. + match = re.search(r"\(HTTP ([1-5][0-9]{2})\)\s*$", stderr, re.MULTILINE) self.status = int(match[1]) if match else None super().__init__(f"GitHub API {path} failed: {stderr.strip()}") diff --git a/inferencex-e2e/infx/tests/test_github.py b/inferencex-e2e/infx/tests/test_github.py index 9dd05935ab..b2c4245974 100644 --- a/inferencex-e2e/infx/tests/test_github.py +++ b/inferencex-e2e/infx/tests/test_github.py @@ -14,6 +14,8 @@ ("gh: Not Found (HTTP 404)\n", 404), ("gh: Forbidden (HTTP 403)", 403), ("gh: Bad Gateway (HTTP 502)", 502), + ('gh: Not Found (HTTP 404)\ngh: This API operation needs the "admin:org" scope. To request ' + "it, run: gh auth refresh -h github.com -s admin:org\n", 404), ("network timeout mentioning 404 without HTTP status", None), ]) def test_authenticated_api_errors_expose_status_without_guessing(monkeypatch, stderr, status): From 1500bfbed119efcb2fc7e8d6da6325d5db28c4db Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:52 -0500 Subject: [PATCH 022/203] fix(workflows): stop retrying permission-denied 403s while waiting for checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check polling treated every HTTP 403 as a transient rate limit, so a plain permission denial was retried until the check timeout, 900 seconds by default, instead of failing at once. A 403 is now retried only when PyGithub classifies it as a rate-limit response. Only non-rate-limit 403 errors during check polling change behavior, and they now raise immediately. 中文:检查轮询过去把所有 HTTP 403 都当作暂时性限流,因此单纯的权限拒绝会一直重试到检查超时(默认 900 秒),而不是立即失败。现在只有 PyGithub 判定为限流响应的 403 才会重试。只有检查轮询期间的非限流 403 错误行为会改变,现在会立即抛出。 --- .../infx/tests/workflows/test_merge_with_reuse.py | 13 +++++++++++++ inferencex-e2e/infx/workflows/merge_with_reuse.py | 13 +++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py index cd24d789c9..cd361a5b27 100644 --- a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py +++ b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py @@ -619,6 +619,19 @@ def test_github_429_is_transient(self): ) assert _is_transient_error(exc) is True + @pytest.mark.parametrize( + ("exc_type", "message", "expected"), + [ + ("GithubException", "Resource not accessible by integration", False), + ("RateLimitExceededException", "API rate limit exceeded for user", True), + ], + ) + def test_github_403_is_transient_only_when_rate_limited(self, exc_type, message, expected): + import github + + exc = getattr(github, exc_type)(status=403, data={"message": message}, headers={}) + assert _is_transient_error(exc) is expected + def test_github_404_is_not_transient(self): from github import GithubException diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index 221959d4a9..4a12f02322 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -37,7 +37,7 @@ try: import git as gitpython - from github import Github, GithubException + from github import Github, GithubException, RateLimitExceededException except ImportError: gitpython = None # type: ignore[assignment] Github = None # type: ignore[assignment, misc] @@ -49,6 +49,9 @@ class GithubException(Exception): # type: ignore[no-redef] # noqa: N818 data: object = None headers: dict = {} # type: ignore[type-arg] # noqa: RUF012 + class RateLimitExceededException(GithubException): # type: ignore[no-redef] + """Stub when PyGithub is not installed.""" + _WORKFLOWS_AVAILABLE = gitpython is not None @@ -143,15 +146,17 @@ def _make_github(token: str) -> Github: return Github(login_or_token=token, base_url=base_url) -_TRANSIENT_HTTP_STATUSES = frozenset({403, 429, 500, 502, 503, 504}) +_TRANSIENT_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504}) def _is_transient_error(exc: Exception) -> bool: """Return True for API errors safe to retry during check polling. - Covers server errors (5xx), rate limits (403/429), and connection-level - failures from requests/urllib3. + Covers server errors (5xx), rate limits (429, and 403 rate-limit + responses), and connection-level failures from requests/urllib3. """ + if isinstance(exc, RateLimitExceededException): + return True if isinstance(exc, GithubException): return exc.status in _TRANSIENT_HTTP_STATUSES return isinstance(exc, (ConnectionError, TimeoutError)) From 65159a1c706928c9a14522bf7305f15a296f88da Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:04:05 -0500 Subject: [PATCH 023/203] fix(workflows): treat a queued check re-run as newer than the run it replaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check polling ordered re-runs by started_at with a missing value sorting first, so a queued re-run that had not started yet lost to the completed run it replaced and an old failure ended the merge. Both wait_for_check and wait_for_checks now rank a run without started_at as the newest and break ties by check run id. Only checks that have a queued, unstarted re-run change behavior, and they are now awaited. 中文:检查轮询过去按 started_at 排序重跑,缺失值排在最前,因此尚未开始的排队重跑会输给它所替代的已完成运行,旧的失败结果会终止合并。现在 wait_for_check 和 wait_for_checks 都把没有 started_at 的运行视为最新,并以 check run id 作为并列时的判定依据。只有存在排队且未开始重跑的检查行为会改变,现在会等待其完成。 --- .../tests/workflows/test_merge_with_reuse.py | 41 +++++++++++++++++++ .../infx/workflows/merge_with_reuse.py | 17 ++++---- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py index cd361a5b27..7b85b1d638 100644 --- a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py +++ b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py @@ -352,6 +352,27 @@ def test_transient_error_retried_then_succeeds(self): result = wait_for_check("a" * 40, "check-changelog", gh_repo, timeout=60) assert result == 0 + def test_unstarted_rerun_supersedes_completed_failure(self): + old = make_check_run( + name="check-changelog", conclusion="failure", started_at="2024-01-01T00:00:00Z", cr_id=10 + ) + queued = make_check_run( + name="check-changelog", status="queued", conclusion=None, started_at=None, cr_id=20 + ) + passed = make_check_run( + name="check-changelog", started_at="2024-01-01T01:00:00Z", cr_id=20 + ) + first, second = MagicMock(), MagicMock() + first.get_check_runs.return_value = [old, queued] + second.get_check_runs.return_value = [old, passed] + gh_repo = MagicMock() + gh_repo.get_commit.side_effect = [first, second] + + with patch("infx.workflows.merge_with_reuse.time.sleep"): + result = wait_for_check("a" * 40, "check-changelog", gh_repo, timeout=60) + + assert result == 0 + @@ -580,6 +601,26 @@ def test_same_job_name_in_two_workflows(self, second_workflow_id, expected): assert result == expected + def test_unstarted_rerun_supersedes_completed_failure(self): + old = make_check_run( + name="build", conclusion="failure", started_at="2024-01-01T00:00:00Z", cr_id=10 + ) + queued = make_check_run( + name="build", status="queued", conclusion=None, started_at=None, cr_id=20 + ) + passed = make_check_run(name="build", started_at="2024-01-01T01:00:00Z", cr_id=20) + first, second = MagicMock(), MagicMock() + first.get_check_runs.return_value = [old, queued] + second.get_check_runs.return_value = [old, passed] + for commit in (first, second): + commit.get_combined_status.return_value = SimpleNamespace(statuses=[]) + gh_repo = MagicMock() + gh_repo.get_commit.side_effect = [first, second] + + with patch("infx.workflows.merge_with_reuse.time.sleep"): + result = wait_for_checks(MagicMock(), "a" * 40, gh_repo, timeout=60) + + assert result == 0 diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index 4a12f02322..eedf991d73 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -284,8 +284,14 @@ def find_eligible_run( return None +def _check_run_recency(cr: object) -> tuple: + """Order check runs oldest first. A queued run without started_at counts as newest.""" + started_at = getattr(cr, "started_at", None) + return (started_at is None, str(started_at or ""), getattr(cr, "id", 0)) + + def _latest_check_runs(check_runs: list, suite_workflows: dict | None = None) -> list: - """Dedupe check runs, keeping the one with the latest started_at/id. + """Dedupe check runs, keeping the most recent per key. GitHub can return superseded re-runs for the same check name. This mirrors ``gh pr checks --watch --fail-fast``, which keys Actions check runs on @@ -300,12 +306,7 @@ def _latest_check_runs(check_runs: list, suite_workflows: dict | None = None) -> if existing is None: by_key[key] = cr continue - cr_key = (str(cr.started_at or ""), getattr(cr, "id", 0)) - ex_key = ( - str(getattr(existing, "started_at", "") or ""), - getattr(existing, "id", 0), - ) - if cr_key > ex_key: + if _check_run_recency(cr) > _check_run_recency(existing): by_key[key] = cr return list(by_key.values()) @@ -442,7 +443,7 @@ def wait_for_check( check_runs = list(commit.get_check_runs()) matching = [cr for cr in check_runs if cr.name == check_name] if matching: - matching.sort(key=lambda cr: str(cr.started_at or "")) + matching.sort(key=_check_run_recency) latest = matching[-1] if latest.status == "completed": From 7b7828b748a0af6d3dcd29e005c571745114cd10 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:18:29 -0500 Subject: [PATCH 024/203] perf(yaml): parse repository YAML with libyaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every plan, generate and validation run parsed the master configs and hundreds of srt recipes with PyYAML's pure-Python loader, which was 90% of infx.matrix.plan's time. infx.yamlio parses with PyYAML's bundled libyaml loader and returns the same objects. Documents where libyaml and PyYAML disagree go to PyYAML: directives, tabs, byte order marks, line breaks other than "\n", comments glued to block scalar headers, tags, '?' inside flow plain scalars, nesting deeper than 50 collection tokens and characters PyYAML's reader refuses, as do all documents libyaml rejects, so errors and their marks are unchanged. lab/yaml_parity.py found no difference over every YAML blob in git history plus 420,525 seeded mutations, for SafeLoader and both duplicate-key loaders. INFX_PURE_YAML=1 restores the pure loader. Plan for f817514d3: 472.7M to about 46M bytecode instructions. 中文:plan、generate 和校验每次都用 PyYAML 的纯 Python 加载器解析主配置和数百个 srt 配方,占 infx.matrix.plan 90% 的时间。infx.yamlio 改用 PyYAML 自带的 libyaml 加载器 解析,返回相同的对象。libyaml 与 PyYAML 结果可能不同的文档(指令、制表符、字节序标记、 "\n" 以外的换行符、紧贴块标量头的注释、标签、流式普通标量中的 '?'、超过 50 层集合标记的 嵌套、PyYAML 读取器拒绝的字符)以及 libyaml 拒绝的文档都交给 PyYAML 解析,因此错误及 其位置保持不变。lab/yaml_parity.py 在 git 历史中的全部 YAML 内容和 420,525 个随机变体上, 对 SafeLoader 和两个拒绝重复键的加载器均未发现差异。设置 INFX_PURE_YAML=1 可恢复纯 Python 加载器。f817514d3 的 plan 字节码指令数从 4.727 亿降至约 4600 万。 --- inferencex-e2e/infx/clusters/__init__.py | 4 +- .../infx/golden_al_distribution/curves.py | 4 +- .../infx/launch/drivers/srt/recipe.py | 3 +- inferencex-e2e/infx/matrix/generate.py | 7 +- inferencex-e2e/infx/matrix/plan.py | 5 +- inferencex-e2e/infx/matrix/validation.py | 7 +- .../infx/srt_slurm/synthetic_acceptance.py | 3 +- inferencex-e2e/infx/srt_slurm/workload.py | 5 +- inferencex-e2e/infx/tests/test_yamlio.py | 59 ++++++++ .../infx/workflows/recover_failed_ingest.py | 4 +- .../infx/workflows/validate_perf_changelog.py | 3 +- inferencex-e2e/infx/yamlio.py | 131 ++++++++++++++++++ 12 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 inferencex-e2e/infx/tests/test_yamlio.py create mode 100644 inferencex-e2e/infx/yamlio.py diff --git a/inferencex-e2e/infx/clusters/__init__.py b/inferencex-e2e/infx/clusters/__init__.py index e9138b12a2..0a06183c5e 100644 --- a/inferencex-e2e/infx/clusters/__init__.py +++ b/inferencex-e2e/infx/clusters/__init__.py @@ -11,7 +11,6 @@ from pathlib import Path, PurePosixPath from typing import Annotated, Any, Literal, Self -import yaml from pydantic import ( Field, PrivateAttr, @@ -21,6 +20,7 @@ model_validator, ) +from infx import yamlio from infx.clusters.base import Record, SchedulerSettings from infx.clusters.slurm import Fabric, SlurmSettings from infx.config import RUNNER_CONFIG, repository_root @@ -220,7 +220,7 @@ def load_inventory(runner_config: str | Path | Mapping[str, Any] | None = None) data = runner_config else: with open(runner_config) as file: - data = yaml.safe_load(file) + data = yamlio.safe_load(file) if not isinstance(data, Mapping): raise ValueError(f"{runner_config}: runner config must be a mapping") return RunnerInventory.model_validate(data) diff --git a/inferencex-e2e/infx/golden_al_distribution/curves.py b/inferencex-e2e/infx/golden_al_distribution/curves.py index 64d6e6d713..6c75f80816 100644 --- a/inferencex-e2e/infx/golden_al_distribution/curves.py +++ b/inferencex-e2e/infx/golden_al_distribution/curves.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -import yaml +from infx import yamlio GOLDEN_DIR = Path(__file__).resolve().parent THINKING_MODES = ("thinking_on", "thinking_off") @@ -66,7 +66,7 @@ def load_curve(name: str, golden_dir: Path = GOLDEN_DIR) -> Curve: path = golden_dir / f"{name}.yaml" if not path.is_file(): raise ValueError(f"No committed golden curve {path.name} in {golden_dir}") - data = yaml.safe_load(path.read_text()) + data = yamlio.safe_load(path.read_text()) if not isinstance(data, dict) or len(data) != 1: raise ValueError(f"Expected one model in golden curve {path.name}") [(model, modes)] = data.items() diff --git a/inferencex-e2e/infx/launch/drivers/srt/recipe.py b/inferencex-e2e/infx/launch/drivers/srt/recipe.py index 074e4b0fed..57e63c9fe6 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/recipe.py +++ b/inferencex-e2e/infx/launch/drivers/srt/recipe.py @@ -12,6 +12,7 @@ import yaml +from infx import yamlio from infx.launch.context import LaunchError if TYPE_CHECKING: @@ -40,7 +41,7 @@ def prepare_recipe(recipe: Path, job_name: str, dist_timeout_s: int | None) -> N """Edit the bound recipe, the variant srtctl submits, for this job: ``job_name`` becomes srtctl's run name, the health check waits at least HEALTH_ATTEMPTS attempts, and each role with a ``watchdog-timeout`` gets the cluster's ``dist-timeout``.""" - bound = yaml.safe_load(recipe.read_text()) + bound = yamlio.safe_load(recipe.read_text()) if "name" in bound: bound["name"] = job_name if "max_attempts" in (health := bound.get("health_check") or {}): diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index c7bed03f91..070ca32036 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -10,8 +10,7 @@ from pathlib import Path from typing import Any, Literal -import yaml - +from infx import yamlio from infx.clusters import CLUSTER_LABEL_PREFIX from infx.clusters.slurm import Fabric from infx.srt_slurm.variants import deep_merge, expand_variants @@ -236,7 +235,7 @@ def recipe_auxiliary_node_count(recipe: dict) -> int: def recipe_node_count(root: Path, srt_recipe: str) -> int | None: path, _, selector = srt_recipe.partition(":") recipe_path = root / path - recipe = yaml.safe_load(recipe_path.read_text()) + recipe = yamlio.safe_load(recipe_path.read_text()) if "base" in recipe: if not (selector == "base" or selector.startswith("override_")): return None @@ -274,7 +273,7 @@ def _label_fabrics(label: str, runner_data: dict) -> dict[str, Fabric]: def check_fabric_references(root: Path, srt_recipe: str, label: str, runner_data: dict) -> None: path, _, selector = srt_recipe.partition(":") - variants = expand_variants(yaml.safe_load((root / path).read_text()), selector or None) + variants = expand_variants(yamlio.safe_load((root / path).read_text()), selector or None) fabrics = { f"cluster {cluster_id!r}": fabric.rendered() for cluster_id, fabric in _label_fabrics(label, runner_data).items() diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index dda30fd68e..b4061a5b8e 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -6,8 +6,7 @@ from collections import defaultdict from contextlib import ExitStack -import yaml - +from infx import yamlio from infx.config import MASTER_CONFIGS, RUNNER_CONFIG, git_path_at_ref from .fingerprint import recipe_fingerprint @@ -468,7 +467,7 @@ def main() -> None: if not added_yaml.strip(): raise ValueError("No additions found in the changelog file.") - changelog_data = yaml.safe_load(added_yaml) + changelog_data = yamlio.safe_load(added_yaml) try: result = build_plan( diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index ea040e79c1..aa65e1776c 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -15,6 +15,7 @@ model_validator, ) +from infx import yamlio from infx.clusters import CLUSTER_LABEL_PREFIX, RunnerInventory DEFAULT_AGENTIC_DURATION_SECONDS = 3600 @@ -976,7 +977,7 @@ def _check_srt_recipe(root: Path, reference: str, recipes: dict[str, Any]) -> No if not (root / path).is_file(): raise ValueError(f"{reference} does not exist") try: - recipes[path] = yaml.safe_load((root / path).read_text()) + recipes[path] = yamlio.safe_load((root / path).read_text()) except yaml.YAMLError as error: raise ValueError(f"{path} is not valid YAML") from error recipe = recipes[path] @@ -1129,7 +1130,7 @@ def load_config_files(config_files: list[str], validate: bool = True) -> dict: for config_file in config_files: try: with open(config_file) as f: - config_data = yaml.safe_load(f) + config_data = yamlio.safe_load(f) if not isinstance(config_data, dict): raise ValueError(f"Config file '{config_file}' must contain a dictionary") @@ -1162,7 +1163,7 @@ def load_config_files(config_files: list[str], validate: bool = True) -> dict: def load_runner_file(runner_file: str, validate: bool = True) -> dict: try: with open(runner_file) as f: - runner_config = yaml.safe_load(f) + runner_config = yamlio.safe_load(f) except FileNotFoundError as e: raise ValueError(f"Runner config file '{runner_file}' does not exist.") from e diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 6ab1a17a40..8a93c6451a 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -15,6 +15,7 @@ import yaml +from infx import yamlio from infx.golden_al_distribution import GOLDEN_DIR, golden_length from infx.srt_slurm.variants import ZIP_VARIANT, without_nulls @@ -294,7 +295,7 @@ def plan_commands( from srtctl.core.overrides import apply_overrides_to_recipe, parse_overrides path, _, selector = config.partition(":") - raw = yaml.safe_load(Path(path).read_text()) + raw = yamlio.safe_load(Path(path).read_text()) if not isinstance(raw, dict): raise ValueError("Recipe must be a mapping") parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 6a1f2ea22e..9eb8ce900a 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -19,6 +19,7 @@ import yaml +from infx import yamlio from infx.config import repository_root from infx.results.metadata import parse_component_metadata from infx.srt_slurm.synthetic_acceptance import selected_recipes, spec_parameters @@ -158,7 +159,7 @@ def check_fragment(raw: Mapping[str, Any], source: Path, *, agentic: bool, multi def _block(path: Path) -> dict[str, Any]: - block = yaml.safe_load(path.read_text()) + block = yamlio.safe_load(path.read_text()) if not isinstance(block, dict): raise ValueError(f"{path}: shared block must be a mapping") return block @@ -193,7 +194,7 @@ def compose_recipe( ``power_port`` adds the DCGM telemetry block with its exporter on that port. """ - raw = yaml.safe_load(path.read_text()) + raw = yamlio.safe_load(path.read_text()) if not isinstance(raw, dict): raise ValueError(f"{path}: recipe must be a mapping") check_fragment(raw, path, agentic=agentic, multinode=multinode) diff --git a/inferencex-e2e/infx/tests/test_yamlio.py b/inferencex-e2e/infx/tests/test_yamlio.py new file mode 100644 index 0000000000..178c1e9798 --- /dev/null +++ b/inferencex-e2e/infx/tests/test_yamlio.py @@ -0,0 +1,59 @@ +import pytest +import yaml + +from infx import yamlio +from infx.workflows.validate_perf_changelog import UniqueKeyLoader + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("a:\n b\tc: 1\n", "that cannot start any token"), + ("a: {b: c?d}\n", "expected ',' or '}', but got '?'"), + ("a: 1\n\ufeff\nb: 2\n", "could not find expected ':'"), + ("a: |#\n x\n", "expected chomping or indentation indicators"), + ("[!!str, a]\n", "could not determine a constructor for the tag"), + ], + ids=[ + "tab-in-plain-scalar", + "question-mark-in-flow-scalar", + "inner-byte-order-mark", + "comment-on-block-scalar-header", + "tag-before-flow-comma", + ], +) +def test_documents_only_libyaml_accepts_raise_pyyamls_error(text, message): + with pytest.raises(yaml.YAMLError, match=message): + yamlio.safe_load(text) + + +def test_a_tagged_empty_flow_key_is_null_as_in_pyyaml(): + assert yamlio.safe_load("{a: 1, ! }\n") == {"a": 1, None: None} + + +@pytest.mark.parametrize( + "text", + ["[" * 2000 + "]" * 2000, "#\r" + "- " * 600 + "x\r"], + ids=["flow", "block-after-carriage-returns"], +) +def test_nesting_past_pyyamls_recursion_limit_raises_its_recursion_error(text): + with pytest.raises(RecursionError): + yamlio.safe_load(text) + + +def test_stream_errors_name_the_file_with_pyyamls_wording(tmp_path): + path = tmp_path / "bad.yaml" + path.write_text("a: [1\n") + + with path.open() as stream, pytest.raises(yaml.YAMLError) as error: + yamlio.safe_load(stream) + + assert str(error.value) == ( + f'while parsing a flow sequence\n in "{path}", line 1, column 4\n' + f"expected ',' or ']', but got ''\n in \"{path}\", line 2, column 1" + ) + + +def test_custom_loader_constructors_apply(): + with pytest.raises(yaml.YAMLError, match="found duplicate key 'a'"): + yamlio.load("x: 1\ny: {a: 1, a: 2}\n", UniqueKeyLoader) diff --git a/inferencex-e2e/infx/workflows/recover_failed_ingest.py b/inferencex-e2e/infx/workflows/recover_failed_ingest.py index d7f53c6ecb..8efae03701 100644 --- a/inferencex-e2e/infx/workflows/recover_failed_ingest.py +++ b/inferencex-e2e/infx/workflows/recover_failed_ingest.py @@ -14,7 +14,7 @@ import yaml -from infx import github +from infx import github, yamlio from infx.config import git_path_at_ref, git_repository_root, project_root from infx.matrix.revision import PLANNER, Revision @@ -473,7 +473,7 @@ def build_config( def validate_recovery_workflow(path: Path, pr_number: int) -> None: """Reject recovery workflows that could schedule benchmark compute.""" text = path.read_text() - data = yaml.load(text, Loader=UniqueKeyLoader) # noqa: S506 + data = yamlio.load(text, UniqueKeyLoader) if not isinstance(data, dict): raise RecoveryError("recovery workflow root must be a mapping") diff --git a/inferencex-e2e/infx/workflows/validate_perf_changelog.py b/inferencex-e2e/infx/workflows/validate_perf_changelog.py index 799296efd5..68e079040e 100644 --- a/inferencex-e2e/infx/workflows/validate_perf_changelog.py +++ b/inferencex-e2e/infx/workflows/validate_perf_changelog.py @@ -15,6 +15,7 @@ from yaml.constructor import ConstructorError from yaml.resolver import BaseResolver +from infx import yamlio from infx.config import git_path_at_ref from infx.matrix.validation import ChangelogEntry @@ -102,7 +103,7 @@ def parse_changelog(raw: bytes, label: str) -> list[dict[str, Any]]: raise ChangelogValidationError(f"{label} is not UTF-8: {exc}") from exc try: - data = yaml.load(text, Loader=UniqueKeyLoader) # noqa: S506 + data = yamlio.load(text, UniqueKeyLoader) except yaml.YAMLError as exc: raise ChangelogValidationError(f"{label} is not valid YAML: {exc}") from exc diff --git a/inferencex-e2e/infx/yamlio.py b/inferencex-e2e/infx/yamlio.py new file mode 100644 index 0000000000..ed3123cf34 --- /dev/null +++ b/inferencex-e2e/infx/yamlio.py @@ -0,0 +1,131 @@ +"""PyYAML loading through libyaml, with PyYAML's own results and errors.""" + +from __future__ import annotations + +import functools +import io +import os +import re +from typing import IO, Any + +import yaml + +_C_SAFE_LOADER = ( + None if os.environ.get("INFX_PURE_YAML") == "1" else getattr(yaml, "CSafeLoader", None) +) +_TABLES = ( + "yaml_constructors", + "yaml_multi_constructors", + "yaml_implicit_resolvers", + "yaml_path_resolvers", +) +_CLASS_ATTRIBUTES = { + *_TABLES, + "__module__", + "__qualname__", + "__doc__", + "__firstlineno__", + "__static_attributes__", +} +# libyaml accepts documents PyYAML rejects when they hold directives, tabs, byte order marks, a +# comment glued to a block scalar header, or characters PyYAML's reader refuses. Line breaks +# other than "\n" go to PyYAML too, because the line-start patterns here only see "\n". +_DIVERGENT = re.compile(r"(?m)^%|[\t\ufeff\r\x85\u2028\u2029]|[|>][-+0-9]*#") +# Deeper documents are left to PyYAML: its recursive composer raises RecursionError near 500 +# levels, where libyaml's recursive C composer returns a value or overflows the C stack. +# Indentless sequences add at most one level per counted collection token. +_MAX_TOKEN_DEPTH = 50 +_OPENING_TOKENS = ( + yaml.BlockMappingStartToken, + yaml.BlockSequenceStartToken, + yaml.FlowMappingStartToken, + yaml.FlowSequenceStartToken, +) +_CLOSING_TOKENS = (yaml.BlockEndToken, yaml.FlowMappingEndToken, yaml.FlowSequenceEndToken) +_FLOW_TOKENS = { + yaml.FlowMappingStartToken: 1, + yaml.FlowSequenceStartToken: 1, + yaml.FlowMappingEndToken: -1, + yaml.FlowSequenceEndToken: -1, +} + + +class _NamedText(io.StringIO): + """Text that reports the original stream's name in PyYAML error marks.""" + + def __init__(self, text: str, name: str) -> None: + super().__init__(text) + self.name = name + + +@functools.cache +def _fast_loader(loader: type) -> type | None: + """libyaml twin of a SafeLoader subclass that only customizes its lookup tables.""" + if _C_SAFE_LOADER is None or not issubclass(loader, yaml.SafeLoader): + return None + mro = loader.__mro__ + if any(set(vars(cls)) - _CLASS_ATTRIBUTES for cls in mro[: mro.index(yaml.SafeLoader)]): + return None + return type( + f"Fast{loader.__name__}", (_C_SAFE_LOADER,), {t: getattr(loader, t) for t in _TABLES} + ) + + +def _may_nest_deeply(text: str) -> bool: + """Whether the token depth can exceed the limit: flow nesting is at most the number of + openers, and block nesting at most one more than the longest run of spaces and block + indicators that starts a line.""" + room = _MAX_TOKEN_DEPTH - 1 - text.count("[") - text.count("{") + return room < 0 or re.search(rf"(?m)^[ ?:-]{{{room + 1},}}", text) is not None + + +def _needs_pyyaml(text: str, fast: type) -> bool: + """Whether the document has a tag, which libyaml and PyYAML scan differently, nesting too + deep for libyaml's composer, or a plain scalar inside a flow collection holding ``?``, + which ends it in PyYAML.""" + if "?" not in text and "!" not in text and not _may_nest_deeply(text): + return False + depth = flow = 0 + for token in yaml.scan(text, Loader=fast): + kind = type(token) + if kind in _OPENING_TOKENS: + depth += 1 + if depth > _MAX_TOKEN_DEPTH: + return True + elif kind in _CLOSING_TOKENS: + depth -= 1 + elif kind is yaml.TagToken or ( + flow and kind is yaml.ScalarToken and token.plain and "?" in token.value + ): + return True + flow += _FLOW_TOKENS.get(kind, 0) + return False + + +def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: + """Return exactly what ``yaml.load(stream, Loader=loader)`` returns or raises. + + Documents libyaml accepts are parsed by libyaml, which builds the same nodes as PyYAML; + PyYAML reparses everything else, so errors and their marks are PyYAML's own. + ``INFX_PURE_YAML=1`` turns libyaml off. + """ + fast = _fast_loader(loader) + if fast is None or not isinstance(stream, str | io.TextIOBase): + return yaml.load(stream, Loader=loader) # noqa: S506 + if isinstance(stream, str): + text, fallback = stream, stream + else: + text = stream.read() + fallback = _NamedText(text, getattr(stream, "name", "")) + if not _DIVERGENT.search(text) and not yaml.reader.Reader.NON_PRINTABLE.search(text): + try: + if not _needs_pyyaml(text, fast): + return yaml.load(text, Loader=fast) # noqa: S506 + except yaml.YAMLError: + pass + return yaml.load(fallback, Loader=loader) # noqa: S506 + + +def safe_load(stream: str | IO[str]) -> Any: + """Drop-in for ``yaml.safe_load`` on text.""" + return load(stream, yaml.SafeLoader) From b30f588de45fb02260e7c031214b4fc02b4c8b52 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:01:33 -0500 Subject: [PATCH 025/203] fix(datasets): clear stale stage outputs before rebuilding a weka dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-running build_weka_hf_dataset in an existing --work-dir kept every session file from earlier runs in proxy/, per_trace/, per_trace_isl/ and per_trace_256k/. Those leftovers were then concatenated into traces.jsonl, counted in the stats and uploaded with the new build. Each stage now empties its own directory under the work dir before writing. Builds in a fresh work dir and runs that reuse cached stages with --skip-sample or --skip-convert are unchanged. 中文:在已有的 --work-dir 中重新运行 build_weka_hf_dataset 时,proxy/、per_trace/、per_trace_isl/ 和 per_trace_256k/ 中保留了之前运行的所有会话文件。这些残留文件随后被合并进 traces.jsonl、计入统计并随新构建一起上传。现在每个阶段在写入前会先清空它在工作目录下自己的目录。在全新工作目录中构建,以及通过 --skip-sample 或 --skip-convert 复用缓存阶段的运行,行为不变。 --- .../infx/datasets/build_weka_hf_dataset.py | 14 +++- .../datasets/test_build_weka_hf_dataset.py | 67 +++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index da20cbcb6c..555dbf553b 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -32,6 +32,7 @@ import json import os import shlex +import shutil import subprocess import sys from collections.abc import Callable @@ -145,13 +146,20 @@ def _run(cmd: list, **kw: Any) -> None: subprocess.run(cmd, check=True, **kw) +def _recreate(stage_dir: Path) -> None: + """Empty a work-dir stage directory so earlier runs' traces are not republished.""" + if stage_dir.exists(): + shutil.rmtree(stage_dir) + stage_dir.mkdir(parents=True) + + def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: proxy_dir = work_dir / "proxy" if args.skip_sample and proxy_dir.exists() and any(proxy_dir.glob("*.jsonl")): n = sum(1 for _ in proxy_dir.glob("*.jsonl")) print(f"[sample] reusing {n} cached session JSONLs in {proxy_dir}") return proxy_dir - proxy_dir.mkdir(parents=True, exist_ok=True) + _recreate(proxy_dir) cmd: list = [ sys.executable, str(SAMPLER), @@ -188,7 +196,7 @@ def stage_convert(args: argparse.Namespace, proxy_dir: Path, work_dir: Path) -> n = sum(1 for _ in per_trace.glob("*.json")) print(f"[convert] reusing {n} cached per-trace JSONs in {per_trace}") return per_trace - per_trace.mkdir(parents=True, exist_ok=True) + _recreate(per_trace) _run( [ sys.executable, @@ -577,7 +585,7 @@ def _stage_filter( tag: str, ) -> Path: """Apply ``filter_fn`` to every per-trace JSON; write survivors to out_dir.""" - out_dir.mkdir(parents=True, exist_ok=True) + _recreate(out_dir) kept = 0 dropped = 0 for p in sorted(per_trace_dir.glob("*.json")): diff --git a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py index 4fc4ac0c38..2fba2ae4de 100644 --- a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py @@ -1,7 +1,74 @@ from __future__ import annotations +import json +import sys +from pathlib import Path + +import pytest + +from infx.datasets import build_weka_hf_dataset from infx.datasets.build_weka_hf_dataset import _filter_trace_256k +STUB_SAMPLER = """ +import json, os, sys +from pathlib import Path + +argv = sys.argv[1:] +out = Path(argv[argv.index("--out") + 1]) +Path(os.environ["STUB_SAMPLER_RECORD"]).write_text( + json.dumps({"argv": argv, "db_url": os.environ.get("AGENTIC_PROXY_DB_URL")}) +) +for sid in os.environ["STUB_SESSIONS"].split(","): + row = { + "timestamp": "2026-05-01T00:00:00+00:00", + "t_sec": 0.0, + "model": "claude-test", + "input_tokens": 10, + "output_tokens": 5, + "duration_ms": 1000, + "hash_ids": ["h1"], + } + (out / f"{sid}.jsonl").write_text(json.dumps(row) + "\\n") +""" + + +@pytest.fixture +def build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Run the build CLI with the DB sampler stubbed to write STUB_SESSIONS, plots skipped.""" + sampler = tmp_path / "stub_sampler.py" + sampler.write_text(STUB_SAMPLER) + record = tmp_path / "sampler-record.json" + no_plot = tmp_path / "no_plot.py" + no_plot.write_text("") + monkeypatch.setattr(build_weka_hf_dataset, "SAMPLER", sampler) + monkeypatch.setattr(build_weka_hf_dataset, "PLOT_WEKA", no_plot) + monkeypatch.setattr(build_weka_hf_dataset, "PLOT_SUBAGENT", no_plot) + monkeypatch.setenv("STUB_SAMPLER_RECORD", str(record)) + monkeypatch.delenv("AGENTIC_PROXY_DB_URL", raising=False) + + def run(sessions: str, *flags: str) -> dict: + monkeypatch.setenv("STUB_SESSIONS", sessions) + argv = ["build", "--repo-base", "org/base", "--work-dir", str(tmp_path / "work")] + monkeypatch.setattr(sys, "argv", [*argv, "--skip-upload", *flags]) + assert build_weka_hf_dataset.main() == 0 + return json.loads(record.read_text()) + + return run + + +def _trace_ids(traces_jsonl: Path) -> list[str]: + return [json.loads(line)["id"] for line in traces_jsonl.read_text().splitlines()] + + +def test_rebuild_in_same_work_dir_publishes_only_this_runs_sessions(build, tmp_path: Path) -> None: + filters = ("--repo-256k", "org/256k", "--base-max-isl", "990016") + build("old-a,old-b", *filters) + build("new-c", *filters) + + work = tmp_path / "work" + assert _trace_ids(work / "base/traces.jsonl") == ["new-c"] + assert _trace_ids(work / "256k/traces.jsonl") == ["new-c"] + def _request(t: float, api_time: float, tokens: int = 1) -> dict: return { From e0e5ac2a0622250256c071432cbeb749983c6315 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:17 -0500 Subject: [PATCH 026/203] fix(datasets): stop echoing the proxy DB URL in the build log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_weka_hf_dataset appended --db-url to the sampler command line, and the build prints every command before running it, so the Postgres password reached stdout, CI logs and the process list. main already exports the URL as AGENTIC_PROXY_DB_URL, which the sampler reads, so the command-line copy is dropped. Only runs that pass --db-url change, and the sampler still connects to the same database. 中文:build_weka_hf_dataset 会把 --db-url 附加到采样器命令行上,而构建脚本会在运行前打印每条命令,因此 Postgres 密码会出现在标准输出、CI 日志和进程列表中。main 已经把该 URL 导出为采样器读取的 AGENTIC_PROXY_DB_URL,所以去掉了命令行中的这份副本。只有传入 --db-url 的运行受影响,采样器仍然连接同一个数据库。 --- .../infx/datasets/build_weka_hf_dataset.py | 2 -- .../tests/datasets/test_build_weka_hf_dataset.py | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index 555dbf553b..5a5958ad11 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -184,8 +184,6 @@ def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: cmd.append("--exclude-dynamic-workflow-bug") if args.dwbug_min_peak is not None: cmd += ["--dwbug-min-peak", str(args.dwbug_min_peak)] - if args.db_url: - cmd += ["--db-url", args.db_url] _run(cmd) return proxy_dir diff --git a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py index 2fba2ae4de..6e81e34bf0 100644 --- a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py @@ -44,7 +44,9 @@ def build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(build_weka_hf_dataset, "PLOT_WEKA", no_plot) monkeypatch.setattr(build_weka_hf_dataset, "PLOT_SUBAGENT", no_plot) monkeypatch.setenv("STUB_SAMPLER_RECORD", str(record)) - monkeypatch.delenv("AGENTIC_PROXY_DB_URL", raising=False) + # setenv first so teardown also removes the value main() exports. + monkeypatch.setenv("AGENTIC_PROXY_DB_URL", "") + monkeypatch.delenv("AGENTIC_PROXY_DB_URL") def run(sessions: str, *flags: str) -> dict: monkeypatch.setenv("STUB_SESSIONS", sessions) @@ -70,6 +72,18 @@ def test_rebuild_in_same_work_dir_publishes_only_this_runs_sessions(build, tmp_p assert _trace_ids(work / "256k/traces.jsonl") == ["new-c"] +def test_db_url_reaches_sampler_without_being_echoed( + build, capfd: pytest.CaptureFixture[str] +) -> None: + url = "postgresql://reader:hunter2-secret@db.example/proxy" + record = build("s1", "--db-url", url) + + assert record["db_url"] == url + assert all("hunter2-secret" not in arg for arg in record["argv"]) + out, err = capfd.readouterr() + assert "hunter2-secret" not in out + err + + def _request(t: float, api_time: float, tokens: int = 1) -> dict: return { "t": t, From 7502bdaf39c90a4e6719891770a339e1851ea0c0 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:03:01 -0500 Subject: [PATCH 027/203] fix(datasets): let weka dataset builds use random sampling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_weka_hf_dataset offered --sampling random but had no way to pass the seed that the sampler requires in that mode, so every random build exited with "--sampling random requires --seed". The build now accepts --seed and forwards it to the sampler and to the README's source command. Builds without --seed send the same sampler arguments as before. 中文:build_weka_hf_dataset 提供了 --sampling random 选项,却无法传入采样器在该模式下必需的种子,因此所有随机采样构建都会以 "--sampling random requires --seed" 退出。现在构建脚本接受 --seed,并将其转发给采样器以及 README 中的源命令。不带 --seed 的构建传给采样器的参数与之前相同。 --- inferencex-e2e/infx/datasets/build_weka_hf_dataset.py | 8 ++++++++ .../infx/tests/datasets/test_build_weka_hf_dataset.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index 5a5958ad11..04e7573af0 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -105,6 +105,12 @@ def parse_args() -> argparse.Namespace: help="Cap session count (for smoke tests). Implies --sampling top.", ) p.add_argument("--sampling", choices=("top", "recent", "random"), default="top") + p.add_argument( + "--seed", + type=int, + default=None, + help="Salt for --sampling random (required by the sampler in that mode).", + ) p.add_argument( "--exclude-dynamic-workflow-bug", action="store_true", @@ -177,6 +183,7 @@ def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: ("--require-cli-min", args.require_cli_min), ("--max-parallel-subagents", args.max_parallel_subagents), ("--limit", args.limit), + ("--seed", args.seed), ]: if val is not None: cmd += [flag, str(val)] @@ -662,6 +669,7 @@ def _reconstruct_sampler_cmd(args: argparse.Namespace) -> list: ("--require-cli-min", args.require_cli_min), ("--max-parallel-subagents", args.max_parallel_subagents), ("--limit", args.limit), + ("--seed", args.seed), ]: if val is not None: cmd += [flag, str(val)] diff --git a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py index 6e81e34bf0..305d4ddc85 100644 --- a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py @@ -84,6 +84,16 @@ def test_db_url_reaches_sampler_without_being_echoed( assert "hunter2-secret" not in out + err +def test_random_sampling_forwards_seed_to_sampler_and_readme(build, tmp_path: Path) -> None: + record = build("s1", "--sampling", "random", "--seed", "7", "--limit", "3") + + argv = record["argv"] + assert argv[argv.index("--sampling") + 1] == "random" + assert argv[argv.index("--seed") + 1] == "7" + readme = (tmp_path / "work/base/README.md").read_text() + assert "--seed 7" in readme + + def _request(t: float, api_time: float, tokens: int = 1) -> dict: return { "t": t, From b6677de13817ac3ddbb11216d6ab64d60acbbf77 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:03:38 -0500 Subject: [PATCH 028/203] fix(datasets): make --limit pick the same sessions on every run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy sampler sorted candidates only by request count (top) or last activity (recent), and the candidate query has no ORDER BY. When sessions tied at the cut-off, which of them fell inside --limit depended on the order Postgres happened to return. Ties are now broken by session id, so the same filters always select the same sessions. Only runs whose limit boundary falls inside a tie, and the dump order of tied sessions, change. 中文:代理采样器只按请求数(top)或最后活动时间(recent)对候选会话排序,而候选查询没有 ORDER BY。当多个会话在截断位置并列时,哪些会话落入 --limit 取决于 Postgres 恰好返回的顺序。现在并列时按会话 ID 排序,因此相同的筛选条件总会选出相同的会话。只有截断边界落在并列区间内的运行,以及并列会话的导出顺序会改变。 --- .../infx/datasets/sample_proxy_traces.py | 2 ++ .../datasets/test_sample_proxy_traces.py | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py diff --git a/inferencex-e2e/infx/datasets/sample_proxy_traces.py b/inferencex-e2e/infx/datasets/sample_proxy_traces.py index 18a622af93..b8effd2a80 100644 --- a/inferencex-e2e/infx/datasets/sample_proxy_traces.py +++ b/inferencex-e2e/infx/datasets/sample_proxy_traces.py @@ -466,6 +466,8 @@ def _sort_and_limit(rows: list[dict], args: argparse.Namespace) -> list[dict]: 'random' mode, so deterministic-random output is identical to the one-pass version. """ + # Phase 1 has no ORDER BY, so pre-sort by session_id to make ties reproducible. + rows = sorted(rows, key=lambda r: r["session_id"]) if args.sampling == "top": rows = sorted(rows, key=lambda r: r["req_count"], reverse=True) elif args.sampling == "recent": diff --git a/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py b/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py new file mode 100644 index 0000000000..3d9911abc6 --- /dev/null +++ b/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import argparse +import sys +import types +from datetime import UTC, datetime + +import pytest + +if "psycopg" not in sys.modules: + # The sampler's DB driver is not a test dependency; nothing here opens a connection. + _psycopg = types.ModuleType("psycopg") + _psycopg.rows = types.ModuleType("psycopg.rows") + _psycopg.rows.dict_row = object() + sys.modules["psycopg"] = _psycopg + sys.modules["psycopg.rows"] = _psycopg.rows + +from infx.datasets.sample_proxy_traces import _sort_and_limit # noqa: E402 + +TIED_TS = datetime(2026, 5, 1, tzinfo=UTC) +OLDER_TS = datetime(2026, 4, 1, tzinfo=UTC) + + +@pytest.mark.parametrize("sampling", ["top", "recent"]) +def test_limit_picks_the_same_sessions_whatever_order_the_db_returns(sampling: str) -> None: + rows = [ + {"session_id": "sess-b", "req_count": 10, "last_ts": TIED_TS}, + {"session_id": "sess-a", "req_count": 10, "last_ts": TIED_TS}, + {"session_id": "sess-c", "req_count": 5, "last_ts": OLDER_TS}, + ] + args = argparse.Namespace(sampling=sampling, limit=1, seed=None) + + picked = [_sort_and_limit(order, args)[0]["session_id"] for order in (rows, rows[::-1])] + + assert picked == ["sess-a", "sess-a"] From 9e272abe8c6acd7d91fc505fb0a3de2d91e2cce3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:04:16 -0500 Subject: [PATCH 029/203] fix(datasets): reject a non-positive sampler --limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy sampler sliced its sorted candidates with rows[:limit], so --limit -1 silently dropped the last session instead of capping the count, and --limit 0 dumped nothing. parse_args now rejects any --limit below 1 with a usage error. Positive limits and runs without --limit are unchanged. 中文:代理采样器用 rows[:limit] 截取排序后的候选会话,因此 --limit -1 会悄悄丢掉最后一个会话而不是限制数量,--limit 0 则什么都不导出。现在 parse_args 会以用法错误拒绝任何小于 1 的 --limit。正数限制以及不带 --limit 的运行不受影响。 --- .../infx/datasets/sample_proxy_traces.py | 2 ++ .../tests/datasets/test_sample_proxy_traces.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/datasets/sample_proxy_traces.py b/inferencex-e2e/infx/datasets/sample_proxy_traces.py index b8effd2a80..e75546f5de 100644 --- a/inferencex-e2e/infx/datasets/sample_proxy_traces.py +++ b/inferencex-e2e/infx/datasets/sample_proxy_traces.py @@ -308,6 +308,8 @@ def parse_args() -> argparse.Namespace: p.error("--sampling random requires --seed") if args.limit is not None and args.sampling is None: p.error("--limit requires --sampling so the LIMIT is deterministic") + if args.limit is not None and args.limit < 1: + p.error("--limit must be >= 1") if args.max_parallel_subagents is not None: if args.max_parallel_subagents < 0: p.error("--max-parallel-subagents must be >= 0") diff --git a/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py b/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py index 3d9911abc6..a6e0fe06f3 100644 --- a/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py +++ b/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py @@ -15,7 +15,7 @@ sys.modules["psycopg"] = _psycopg sys.modules["psycopg.rows"] = _psycopg.rows -from infx.datasets.sample_proxy_traces import _sort_and_limit # noqa: E402 +from infx.datasets.sample_proxy_traces import _sort_and_limit, parse_args # noqa: E402 TIED_TS = datetime(2026, 5, 1, tzinfo=UTC) OLDER_TS = datetime(2026, 4, 1, tzinfo=UTC) @@ -33,3 +33,16 @@ def test_limit_picks_the_same_sessions_whatever_order_the_db_returns(sampling: s picked = [_sort_and_limit(order, args)[0]["session_id"] for order in (rows, rows[::-1])] assert picked == ["sess-a", "sess-a"] + + +def test_zero_limit_is_rejected( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + argv = ["sample", "--out", "out", "--sampling", "top", "--limit", "0"] + monkeypatch.setattr(sys, "argv", argv) + + with pytest.raises(SystemExit) as exit_info: + parse_args() + + assert exit_info.value.code == 2 + assert "--limit must be >= 1" in capsys.readouterr().err From 1645c9ff14c03f28ad1f26dd8cd2bfb5e3cee6b3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:05:25 -0500 Subject: [PATCH 030/203] fix(datasets): keep log histograms valid for tiny or single-valued metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dataset plot scripts floored the lowest log bin edge at 1e-3, so a metric whose values were all below a millisecond produced decreasing bin edges. A metric with a single distinct value made np.geomspace round to non-monotonic edges. Both cases crashed the plot with "bins must increase monotonically", and the build then published a dataset card without plots. The floor now applies only when the largest value is above it, and a single value gets a range centred on it. Metrics with spread above 1e-3 keep the same bins. 中文:数据集绘图脚本把对数直方图最低的分箱边界下限设为 1e-3,因此当某个指标的所有值都小于一毫秒时,分箱边界会递减。只有一个不同取值的指标会让 np.geomspace 舍入出非单调的边界。这两种情况都会使绘图以 "bins must increase monotonically" 崩溃,构建随后发布的数据集卡片就没有图。现在只有在最大值高于该下限时才应用下限,单一取值则使用以它为中心的范围。取值分布在 1e-3 以上的指标分箱不变。 --- .../datasets/plot_subagent_distributions.py | 7 +++-- .../infx/datasets/plot_weka_distributions.py | 7 +++-- .../tests/datasets/test_plot_distributions.py | 29 +++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 inferencex-e2e/infx/tests/datasets/test_plot_distributions.py diff --git a/inferencex-e2e/infx/datasets/plot_subagent_distributions.py b/inferencex-e2e/infx/datasets/plot_subagent_distributions.py index d9239d89d8..33394f7e14 100644 --- a/inferencex-e2e/infx/datasets/plot_subagent_distributions.py +++ b/inferencex-e2e/infx/datasets/plot_subagent_distributions.py @@ -191,9 +191,12 @@ def _draw_histogram( ax.set_title(f"{title}\n(no positive values)") ax.axis("off") return - lo = max(positive.min(), 1e-3) hi = float(positive.max()) - edges = np.geomspace(lo, hi, bins + 1) + lo = max(float(positive.min()), 1e-3) if hi > 1e-3 else float(positive.min()) + # geomspace(v, v) can round to non-monotonic edges; centre a single value instead. + edges = ( + np.geomspace(lo, hi, bins + 1) if hi > lo else np.geomspace(lo / 2, hi * 2, bins + 1) + ) ax.hist(positive, bins=edges, color="#888", edgecolor="#222", linewidth=0.4) ax.set_xscale("log") else: diff --git a/inferencex-e2e/infx/datasets/plot_weka_distributions.py b/inferencex-e2e/infx/datasets/plot_weka_distributions.py index b8795b94cd..29f58f17fa 100644 --- a/inferencex-e2e/infx/datasets/plot_weka_distributions.py +++ b/inferencex-e2e/infx/datasets/plot_weka_distributions.py @@ -214,9 +214,12 @@ def _draw_histogram( ax.set_title(f"{title}\n(no positive values)") ax.axis("off") return - lo = max(positive.min(), 1e-3) hi = float(positive.max()) - edges = np.geomspace(lo, hi, bins + 1) + lo = max(float(positive.min()), 1e-3) if hi > 1e-3 else float(positive.min()) + # geomspace(v, v) can round to non-monotonic edges; centre a single value instead. + edges = ( + np.geomspace(lo, hi, bins + 1) if hi > lo else np.geomspace(lo / 2, hi * 2, bins + 1) + ) ax.hist(positive, bins=edges, color="#888", edgecolor="#222", linewidth=0.4) ax.set_xscale("log") else: diff --git a/inferencex-e2e/infx/tests/datasets/test_plot_distributions.py b/inferencex-e2e/infx/tests/datasets/test_plot_distributions.py new file mode 100644 index 0000000000..95e3deeafa --- /dev/null +++ b/inferencex-e2e/infx/tests/datasets/test_plot_distributions.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from types import ModuleType + +import pytest +from matplotlib.figure import Figure + +from infx.datasets import plot_subagent_distributions, plot_weka_distributions + + +@pytest.mark.parametrize("module", [plot_weka_distributions, plot_subagent_distributions]) +@pytest.mark.parametrize("values", [[0.0002, 0.0005], [64.0, 64.0, 64.0]]) +def test_log_histogram_counts_every_positive_value( + module: ModuleType, values: list[float] +) -> None: + ax = Figure().subplots() + + module._draw_histogram( + ax=ax, + values=values, + title="metric", + xlabel="x", + bins=40, + log_x=True, + value_fmt="{:,.4f}", + linear_clip_pct=99.0, + ) + + assert sum(bar.get_height() for bar in ax.patches) == len(values) From 1e7c0dd00d19dfa952ee1a98d9dcbc9422af0bea Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:06:21 -0500 Subject: [PATCH 031/203] fix(srt-slurm): record no golden AL when vLLM verification is adaptive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_overrides leaves vLLM roles with enable_adaptive_verification on real verification, but acceptance_length still looked up and returned the golden curve value. The launcher then recorded that unused value as golden_acceptance_length, and a draft length without a curve cell failed the submission even though nothing would be simulated. acceptance_length now returns None when every speculative vLLM role uses adaptive verification. Recipes where any speculative role still gets synthetic acceptance are unchanged. 中文:build_overrides 会让启用 enable_adaptive_verification 的 vLLM 角色保持真实验证,但 acceptance_length 仍然查找并返回黄金曲线的值。启动器随后把这个未被使用的值记录为 golden_acceptance_length;如果草稿长度在曲线中没有对应条目,即使不会进行任何模拟,提交也会失败。现在当所有投机 vLLM 角色都使用自适应验证时,acceptance_length 返回 None。只要还有投机角色使用合成接受,相应方案的行为就不变。 --- .../infx/srt_slurm/synthetic_acceptance.py | 7 +++++++ .../tests/srt_slurm/test_synthetic_acceptance.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 8a93c6451a..aded6b4410 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -129,6 +129,13 @@ def acceptance_length( spec = spec_parameters(generation, engine) if not (synthetic and spec): return None + if engine == "vllm" and all( + spec_parameters(role, engine).get("enable_adaptive_verification") is True + for name, role in roles.items() + if name in ("agg", "prefill", "decode") and spec_parameters(role, engine) + ): + # build_overrides leaves adaptive verification real, so no golden AL is simulated. + return None al = golden_length(environment["MODEL_PREFIX"], spec, environment["THINKING_MODE"], golden_dir) if ( engine == "sglang" diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index 0e5c429cee..533aa9e58f 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -12,6 +12,7 @@ import yaml from infx.srt_slurm.synthetic_acceptance import ( + acceptance_length, build_overrides, plan_commands, selected_recipes, @@ -202,6 +203,21 @@ def test_adaptive_dspark_skips_golden_acceptance_on_throughput(golden_dir: Path) assert spec["rejection_sample_method"] == "synthetic" +def test_adaptive_verification_records_no_golden_acceptance(golden_dir: Path) -> None: + adaptive = { + "args": { + "speculative-config": json.dumps( + {"method": "dspark", "num_speculative_tokens": 5, "enable_adaptive_verification": True} + ) + } + } + recipe = {"roles": {"prefill": adaptive, "decode": adaptive}} + env = {**ENV, "SPEC_DECODING": "dspark"} + + assert acceptance_length(recipe, "vllm-disagg", env, golden_dir=golden_dir) is None + assert build_overrides(recipe, "vllm-disagg", env, golden_dir=golden_dir) == [] + + @pytest.mark.parametrize("sampling", [None, "unknown"]) def test_kimi_curve_requires_an_explicit_supported_sampler( golden_dir: Path, sampling: str | None From a85bb6a3b369b0f3aeba83a3f5452fe714fff840 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:07:26 -0500 Subject: [PATCH 032/203] fix(srt-slurm): report a variant with a missing field instead of aborting selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit select_recipe collected only ValueError from binding and validating each variant. A variant missing a field read by key, such as an SGLang role without tensor-parallel-size or a vLLM speculative-config without method, raised a bare KeyError that aborted the whole bundle and printed only the field name. Such variants are now collected like other mismatches with the variant name in the message. Bundles where every variant is well formed select the same recipe as before. 中文:select_recipe 在绑定和校验每个变体时只收集 ValueError。如果某个变体缺少按键读取的字段,例如 SGLang 角色没有 tensor-parallel-size,或 vLLM 的 speculative-config 没有 method,就会抛出裸 KeyError,导致整个 bundle 的选择中止,并且只打印字段名。现在这类变体会像其他不匹配一样被收集,错误信息中带有变体名称。所有变体都格式正确的 bundle 选出的方案与之前相同。 --- inferencex-e2e/infx/srt_slurm/single_node.py | 2 ++ .../infx/tests/srt_slurm/test_srt_single_node.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index dc5b9d9b1b..4f71c0e4ee 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -100,6 +100,8 @@ def select_recipe( validate_recipe(recipe, environment) except ValueError as exc: errors.append(f"{name}: {exc}") + except (KeyError, TypeError) as exc: + errors.append(f"{name}: missing or malformed field {exc}") else: matches.append((f"{path}:{name}" if name else path, recipe)) if len(matches) != 1: diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index da3d073d34..cbdd4f2d16 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -114,6 +114,15 @@ def test_ambiguous_native_variants_are_rejected(point): runtime_arguments(str(path), env) +def test_a_variant_missing_a_required_field_is_reported_not_raised(point): + path, recipe, env = point + broken = {"roles": {"agg": {"args": {"tensor-parallel-size": None}}}} + path.write_text(yaml.safe_dump({"base": recipe, "override_good": {}, "override_broken": broken})) + assert select_recipe(str(path), env)[0] == f"{path}:override_good" + with pytest.raises(ValueError, match="override_broken: .*'tensor-parallel-size'"): + select_recipe(f"{path}:override_broken", env) + + @pytest.mark.parametrize("algorithm,spec_decoding", [("EAGLE", "mtp"), ("DFLASH", "draft_model")]) def test_speculative_binding_uses_real_verification_and_the_chat_template( point, tmp_path, algorithm, spec_decoding From 68377ac7c78fb9ff81487cb5653ebc519cec284d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:08:07 -0500 Subject: [PATCH 033/203] fix(srt-slurm): refuse a prepare output that would overwrite the bound recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit single_node prepare writes the bound recipe as recipe.yaml next to its output file and then writes the NUL-separated srtctl arguments to the output. When the output itself was named recipe.yaml, the arguments replaced the bound recipe and srtctl was pointed at a file that was no longer YAML. prepare now exits with a usage error for that name before writing anything. Every other output name behaves as before. 中文:single_node prepare 会把绑定后的方案写成输出文件旁边的 recipe.yaml,然后把以 NUL 分隔的 srtctl 参数写入输出文件。当输出文件本身也叫 recipe.yaml 时,参数会覆盖绑定后的方案,srtctl 读到的文件也就不再是 YAML。现在对这个文件名,prepare 会在写入任何内容之前以用法错误退出。其他输出文件名的行为不变。 --- inferencex-e2e/infx/srt_slurm/single_node.py | 2 ++ .../tests/srt_slurm/test_srt_single_node.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index 4f71c0e4ee..a93a867358 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -223,6 +223,8 @@ def main() -> None: submitted = commands.add_parser("submission") submitted.add_argument("manifest", type=Path) parsed = parser.parse_args() + if parsed.command == "prepare" and parsed.output.name == BOUND_RECIPE: + parser.error(f"{parsed.output} would overwrite the bound recipe written beside it") try: if parsed.command == "prepare": selected, recipe = select_recipe(parsed.recipe, os.environ) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index cbdd4f2d16..21310b3f52 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -8,7 +8,7 @@ import pytest import yaml -from infx.srt_slurm.single_node import runtime_arguments, select_recipe, submission_fields +from infx.srt_slurm.single_node import main, runtime_arguments, select_recipe, submission_fields from infx.srt_slurm.synthetic_acceptance import Planned, plan_commands ROOT = Path(__file__).resolve().parents[3] @@ -123,6 +123,23 @@ def test_a_variant_missing_a_required_field_is_reported_not_raised(point): select_recipe(f"{path}:override_broken", env) +def test_prepare_refuses_an_arguments_file_named_like_the_bound_recipe( + point, tmp_path, monkeypatch, capsys +): + path, _, env = point + for name, value in env.items(): + monkeypatch.setenv(name, value) + output = tmp_path / "job" / "recipe.yaml" + output.parent.mkdir() + argv = ["single_node", "prepare", f"{path}:base", str(output), "--fabric", "{}"] + monkeypatch.setattr(sys, "argv", argv) + with pytest.raises(SystemExit) as exit_info: + main() + assert exit_info.value.code == 2 + assert "would overwrite the bound recipe" in capsys.readouterr().err + assert not output.exists() + + @pytest.mark.parametrize("algorithm,spec_decoding", [("EAGLE", "mtp"), ("DFLASH", "draft_model")]) def test_speculative_binding_uses_real_verification_and_the_chat_template( point, tmp_path, algorithm, spec_decoding From ace948ffcb4f7a879270c172dd8b879bf32480e8 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:08:47 -0500 Subject: [PATCH 034/203] fix(golden-al): reject curve names that leave the golden directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_curve joined the requested name onto the golden directory without checking it, so "python -m infx.golden_al_distribution show ../x" read and printed any YAML file outside that directory. curve_name already restricted curve identities to lowercase letters, digits, dot, dash and underscore. load_curve now applies the same check. Every committed curve name already passes it. 中文:load_curve 在把请求的名称拼接到黄金曲线目录上时没有做检查,因此 "python -m infx.golden_al_distribution show ../x" 会读取并打印该目录之外的任意 YAML 文件。curve_name 本来就把曲线标识限制为小写字母、数字、点、连字符和下划线。现在 load_curve 也使用同样的检查。所有已提交的曲线名称都能通过该检查。 --- inferencex-e2e/infx/golden_al_distribution/curves.py | 5 ++++- .../test_golden_al_distribution.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/golden_al_distribution/curves.py b/inferencex-e2e/infx/golden_al_distribution/curves.py index 6c75f80816..78b0ff3fa1 100644 --- a/inferencex-e2e/infx/golden_al_distribution/curves.py +++ b/inferencex-e2e/infx/golden_al_distribution/curves.py @@ -13,6 +13,7 @@ GOLDEN_DIR = Path(__file__).resolve().parent THINKING_MODES = ("thinking_on", "thinking_off") +CURVE_NAME = re.compile(r"[a-z0-9_.-]+") @dataclass(frozen=True) @@ -57,12 +58,14 @@ def curve_name(model: str, spec: Mapping[str, Any]) -> str: curve += "_probabilistic_sample_method_block_rejection_sample_method" elif sampling != "greedy": raise ValueError(f"No Kimi DSpark golden curve for draft sampling {sampling!r}") - if not re.fullmatch(r"[a-z0-9_.-]+", curve): + if not CURVE_NAME.fullmatch(curve): raise ValueError(f"Invalid golden curve identity: {curve!r}") return curve def load_curve(name: str, golden_dir: Path = GOLDEN_DIR) -> Curve: + if not CURVE_NAME.fullmatch(name): + raise ValueError(f"Invalid golden curve identity: {name!r}") path = golden_dir / f"{name}.yaml" if not path.is_file(): raise ValueError(f"No committed golden curve {path.name} in {golden_dir}") diff --git a/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py b/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py index 1f6046cb9d..44aab4506f 100644 --- a/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py +++ b/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py @@ -90,3 +90,12 @@ def test_cli_list_and_show(capsys) -> None: def test_cli_reports_errors(capsys) -> None: assert main(["show", "missing_curve"]) == 1 assert "No committed golden curve" in capsys.readouterr().err + + +def test_cli_show_rejects_a_curve_outside_the_golden_dir(tmp_path, capsys) -> None: + golden = tmp_path / "golden" + golden.mkdir() + (tmp_path / "outside.yaml").write_text("some-model:\n thinking_on:\n 3: 2.5\n") + + assert main(["--golden-dir", str(golden), "show", "../outside"]) == 1 + assert "Invalid golden curve identity" in capsys.readouterr().err From abddeb695bfeab599bfba83d2b9963845c1f0bf1 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:59:39 -0500 Subject: [PATCH 035/203] fix(matrix): compare rows without a recipe fingerprint one by one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When infx.matrix.changed could not fingerprint a row, it keyed every such point as (None, conc), so two unbindable rows of one config key at the same concurrency overwrote each other. A change to those rows between base and head then went unreported. Unfingerprintable rows are now keyed by their row fingerprint, so each one is counted and compared on its own. Rows with a recipe fingerprint keep their current keys. 中文:infx.matrix.changed 无法为某行计算指纹时,会把这些点统一记为 (None, conc),因此同一 config key 下并发数相同的两个无法绑定的行会相互覆盖,base 与 head 之间这些行的变化也就无法被报告。现在无法计算指纹的行改用行指纹作为键,每一行都会被单独计数和比较。已有 recipe 指纹的行键值保持不变。 --- inferencex-e2e/infx/matrix/changed.py | 8 +++++--- .../infx/tests/matrix/test_changed.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index c129b953f3..b188a6ef5c 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -13,12 +13,12 @@ from infx.config import MASTER_CONFIGS, RUNNER_CONFIG -from .fingerprint import recipe_fingerprint +from .fingerprint import recipe_fingerprint, row_fingerprint from .generate import generate_config_matrix from .revision import snapshot from .validation import config_root, load_config_files, load_runner_file -Matrix = dict[str, tuple[dict[tuple[str | None, int], dict[str, Any]], list[str]]] +Matrix = dict[str, tuple[dict[tuple[str, int], dict[str, Any]], list[str]]] def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: @@ -26,11 +26,13 @@ def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: for row in rows: try: fingerprint = recipe_fingerprint(row, root) + identity = fingerprint except ValueError as error: fingerprint = None + identity = row_fingerprint(row) errors.append(str(error)) for conc in row["conc"] if isinstance(row["conc"], list) else [row["conc"]]: - points[fingerprint, conc] = {**row, "conc": conc, "recipe-fingerprint": fingerprint} + points[identity, conc] = {**row, "conc": conc, "recipe-fingerprint": fingerprint} return points, errors diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index 77efa8bab3..dab31fb64f 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -78,3 +78,22 @@ def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, c changed.main(["--base", "base"]) assert capsys.readouterr().out.splitlines()[-1] == "3 of 4 config keys changed since base." + + +def test_rows_without_fingerprints_are_compared_individually(tmp_path): + def matrix(*tps: int) -> changed.Matrix: + rows = [ + {"exp-name": f"key_tp{tp}", "tp": tp, "ep": 1, "conc": [4], "srt-recipe": "gone.yaml"} + for tp in tps + ] + return {"key": changed._points(rows, tmp_path)} + + base, head = matrix(4, 8), matrix(4, 2) + + assert [len(errors) for _, errors in (base["key"], head["key"])] == [2, 2] + assert [ + (change["base-points"], change["head-points"], + [point["tp"] for point in change["added"]], [point["tp"] for point in change["removed"]], + [point["recipe-fingerprint"] for point in change["added"] + change["removed"]]) + for change in changed.compare(base, head) + ] == [(2, 2, [2], [8], [None, None])] # fmt: skip From 0107f37bdd7a70e991c200f3b1fab473344caf3c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:01:33 -0500 Subject: [PATCH 036/203] fix(matrix): stop --model-prefix from selecting longer model prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit full-sweep --model-prefix kept every config key that merely started with the given text, so --model-prefix dsv4 also swept the dsv41flash keys. Config keys begin with the model prefix followed by a dash, so the filter now requires that dash. Only keys whose model prefix extends the requested one drop out of the selection. 中文:full-sweep --model-prefix 会保留所有仅以给定文本开头的 config key,因此 --model-prefix dsv4 也会扫到 dsv41flash 的 key。config key 以模型前缀加一个短横线开头,所以过滤条件现在要求带上这个短横线。只有模型前缀是所请求前缀加长版本的 key 会被移出选择范围。 --- inferencex-e2e/infx/matrix/generate.py | 2 +- .../matrix/test_generate_sweep_configs.py | 32 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 070ca32036..8127f59808 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1165,7 +1165,7 @@ def expand_full_sweep( configs = ( config for key, config in master_config.items() - if (not options.model_prefix or any(key.startswith(p) for p in options.model_prefix)) + if (not options.model_prefix or any(key.startswith(f"{p}-") for p in options.model_prefix)) and (not options.precision or config[Fields.PRECISION.value] in options.precision) and (not options.framework or config[Fields.FRAMEWORK.value] in options.framework) and (not options.runner_type or config[Fields.RUNNER.value] in options.runner_type) diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index b3b07209e9..8e6dc4443f 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -1235,21 +1235,25 @@ def test_matrix_entry_structure(self, sample_single_node_config, sample_runner_c } == {(2, 2, 2)} def test_filter_by_model_prefix(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): - full_sweep_args_single_node.model_prefix = ["dsr1"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) > 0 + entry = sample_single_node_config["dsr1-fp8-mi300x-sglang"] + sample_single_node_config["dsr1flash-fp8-mi300x-sglang"] = {**entry, "model-prefix": "dsr1flash"} - full_sweep_args_single_node.model_prefix = ["nonexistent"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) == 0 + selected = {} + for prefixes in (["dsr1"], ["dsr1flash"], ["dsr1", "dsr1flash"], ["nonexistent"]): + full_sweep_args_single_node.model_prefix = prefixes + result = generate_full_sweep( + full_sweep_args_single_node, + sample_single_node_config, + sample_runner_config + ) + selected[tuple(prefixes)] = sorted({row["model-prefix"] for row in result}) + + assert selected == { + ("dsr1",): ["dsr1"], + ("dsr1flash",): ["dsr1flash"], + ("dsr1", "dsr1flash"): ["dsr1", "dsr1flash"], + ("nonexistent",): [], + } def test_filter_by_precision(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): full_sweep_args_single_node.precision = ["fp8"] From 0177242a8d7ace25c49a9e92d9914f0215826514 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:07:32 -0500 Subject: [PATCH 037/203] fix(matrix): reject repeated concurrencies in conc-list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Master config validation only checked that conc-list values were positive, so a list such as [4, 8, 4] passed. A multi-node row then carried the repeated value into CONC_LIST, which the srt-slurm launcher rejects on the cluster, and a single-node entry emitted two identical rows with the same result file. Validation now rejects any conc-list with a repeated value. No current master config repeats a concurrency. 中文:master config 校验只检查 conc-list 的值是否为正数,所以像 [4, 8, 4] 这样的列表也能通过。多节点行会把重复值带进 CONC_LIST,而 srt-slurm 启动器会在集群上拒绝它;单节点条目则会生成两行完全相同、结果文件也相同的行。现在校验会拒绝任何含重复值的 conc-list。当前所有 master config 都没有重复的并发数。 --- inferencex-e2e/infx/matrix/validation.py | 3 +++ inferencex-e2e/infx/tests/matrix/test_validation.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index aa65e1776c..f70ce9b2b8 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -464,6 +464,9 @@ def _validate_conc_fields(self: Any) -> Any: if has_list and not all(x > 0 for x in self.conc_list): raise ValueError(f"Input '{Fields.CONC_LIST.value}' entries must be greater than 0.") + if has_list and len(set(self.conc_list)) != len(self.conc_list): + raise ValueError(f"Input '{Fields.CONC_LIST.value}' entries must not repeat.") + return self diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index 54413d0a28..f182783919 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -632,6 +632,13 @@ def test_conc_list_values_must_be_positive(self): }) assert "must be greater than 0" in str(exc_info.value) + def test_conc_list_values_must_be_unique(self): + with pytest.raises(ValidationError, match="must not repeat"): + SingleNodeSearchSpaceEntry(**{ + "tp": 4, + "conc-list": [4, 8, 4], + }) + class TestMultiNodeSearchSpaceEntry: From 2c45acb0fbdf34525bce83d254c4f30ad695df25 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:08:00 -0500 Subject: [PATCH 038/203] fix(matrix): reject a lone conc-start or conc-end next to conc-list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check that forbids mixing conc-list with a range only fired when both conc-start and conc-end were set. An entry with conc-list plus only one bound passed validation, and the generator silently ignored that bound. Validation now rejects conc-list together with either bound. Entries that set both bounds, or a list alone, behave as before. 中文:禁止同时使用 conc-list 和范围的检查只在 conc-start 与 conc-end 都存在时才会触发。一个条目如果同时写了 conc-list 和其中一个边界,就能通过校验,而生成器会悄悄忽略这个边界。现在只要 conc-list 与任一边界同时出现,校验就会拒绝。只写两个边界或只写列表的条目行为不变。 --- inferencex-e2e/infx/matrix/validation.py | 2 +- inferencex-e2e/infx/tests/matrix/test_validation.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index f70ce9b2b8..2bb52876b1 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -429,7 +429,7 @@ def _validate_conc_fields(self: Any) -> Any: has_range = self.conc_start is not None and self.conc_end is not None has_list = self.conc_list is not None and len(self.conc_list) > 0 - if has_range and has_list: + if has_list and (self.conc_start is not None or self.conc_end is not None): raise ValueError( f"Cannot specify both '{Fields.CONC_LIST.value}' list and " f"'{Fields.CONC_START.value}'/'{Fields.CONC_END.value}'. " diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index f182783919..0b99f95c2d 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -639,6 +639,15 @@ def test_conc_list_values_must_be_unique(self): "conc-list": [4, 8, 4], }) + @pytest.mark.parametrize("bound", ["conc-start", "conc-end"]) + def test_conc_list_rejects_a_lone_range_bound(self, bound): + with pytest.raises(ValidationError, match="Cannot specify both"): + SingleNodeSearchSpaceEntry(**{ + "tp": 4, + "conc-list": [4], + bound: 1, + }) + class TestMultiNodeSearchSpaceEntry: From f77186fc9866c69de334d86a4b97137410248330 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:09:50 -0500 Subject: [PATCH 039/203] fix(matrix): require a positive tensor parallel size in master configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Search-space entries and worker blocks accepted tp: 0 or a negative tp. A multi-node agentic DRAM-offload entry with tp: 0 then crashed plan generation with a ZeroDivisionError when it sized workers per node, and other entries produced rows that cannot be served. Validation now requires tp to be greater than 0 in single-node, agentic, aggregate and prefill or decode worker entries. Configs with a positive tp are unaffected. 中文:search-space 条目和 worker 块之前接受 tp: 0 或负数 tp。多节点 agentic DRAM offload 条目若 tp 为 0,在按节点计算 worker 规模时会以 ZeroDivisionError 中断计划生成,其他条目则会生成无法部署的行。现在单节点、agentic、聚合以及 prefill 或 decode worker 条目的校验都要求 tp 大于 0。tp 为正数的配置不受影响。 --- inferencex-e2e/configs/CONFIGS.md | 2 +- inferencex-e2e/infx/matrix/validation.py | 8 ++++---- inferencex-e2e/infx/tests/matrix/test_validation.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index ce1fb997f8..1eb2e42c2a 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -101,7 +101,7 @@ The below list describes what each field is: - `isl`: An integer representing the input sequence length, e.g., `1024` - `osl`: An integer representing the output sequence length, e.g., `8192` - `search-space`: A list of configurations to run with respective `isl` and `osl`, each entry must be a dict with the following fields: - - `tp`: An integer representing the tensor parallelism level that the configuration will be served at. + - `tp`: A positive integer representing the tensor parallelism level that the configuration will be served at. - `conc-start`: An integer representing the starting level of concurrency e.g., `4` - `conc-end`: An integer representing the ending level of concurrency (inclusive) e.g., `128` - Note: the step factor between `conc-start` and `conc-end` is 2, so if `conc-start` is 4 and `conc-end` is 128, all concurrencies `4, 8, 16, 32, ..., 128` will be run. diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index 2bb52876b1..06186ceb5e 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -181,7 +181,7 @@ class WorkerConfig(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) num_worker: int = Field(alias=Fields.NUM_WORKER.value) - tp: int + tp: int = Field(gt=0) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -210,7 +210,7 @@ class AggregateWorkerConfig(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) num_worker: int = Field(default=1, alias=Fields.NUM_WORKER.value, gt=0, strict=True) - tp: int + tp: int = Field(gt=0) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -531,7 +531,7 @@ def _recipe_reference(value: str) -> str: class SingleNodeSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - tp: int + tp: int = Field(gt=0) srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) @@ -619,7 +619,7 @@ class MultiNodeSeqLenConfig(BaseModel): class AgenticCodingSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - tp: int | None = None + tp: int | None = Field(default=None, gt=0) srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) eval_srt_recipe: RecipeReference | None = Field( default=None, alias=Fields.EVAL_SRT_RECIPE.value diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index 0b99f95c2d..696a0652d0 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -649,6 +649,18 @@ def test_conc_list_rejects_a_lone_range_bound(self, bound): }) +@pytest.mark.parametrize(("schema", "entry"), [ + (SingleNodeSearchSpaceEntry, {"tp": 0, "conc-list": [4]}), + (AgenticCodingSearchSpaceEntry, {"tp": -2, "kv-offloading": "none", "conc-list": [4]}), + (WorkerConfig, {"num-worker": 1, "tp": 0, "ep": 1, "dp-attn": False}), + (MultiNodeSearchSpaceEntry, + {"worker": {"tp": 0, "ep": 1, "dp-attn": False}, "num-nodes": 1, "conc-list": [4]}), +]) # fmt: skip +def test_tensor_parallelism_must_be_positive(schema, entry): + with pytest.raises(ValidationError, match="greater than 0"): + schema(**entry) + + class TestMultiNodeSearchSpaceEntry: def test_valid_aggregate_worker(self): From 6a37a915aae9b86304c4816b898032a36f0d654d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:11:43 -0500 Subject: [PATCH 040/203] fix(matrix): count recipe nodes when recipe sections are null MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recipe_node_count crashed with AttributeError or TypeError when a recipe set services, infra, frontend, benchmark or a service placement to YAML null, when a role or its nodes was null, or when the recipe file was empty. Plan generation then aborted with a bare traceback that did not name the recipe. Null sections now count as absent, as the srt-slurm binder already treats them, and a null role, null nodes or an empty file raise a ValueError naming the recipe. Recipes without null values get the same node count as before. 中文:当 recipe 把 services、infra、frontend、benchmark 或某个 service 的 placement 设为 YAML null,或某个 role 及其 nodes 为 null,或 recipe 文件为空时,recipe_node_count 会抛出 AttributeError 或 TypeError,计划生成随之以不包含 recipe 名称的裸 traceback 中断。现在 null 的段落按缺省处理,与 srt-slurm binder 的现有做法一致;null 的 role、null 的 nodes 或空文件会抛出带有 recipe 名称的 ValueError。不含 null 值的 recipe 计算出的节点数与之前相同。 --- inferencex-e2e/infx/matrix/generate.py | 18 +++++++------- .../matrix/test_generate_sweep_configs.py | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 8127f59808..4c28c8351b 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -206,13 +206,13 @@ def _worker_node_override(worker: dict, setting_name: str) -> int | None: def recipe_auxiliary_node_count(recipe: dict) -> int: - services = [service for service in recipe.get("services", []) if service.get("enabled", True)] + services = [service for service in recipe.get("services") or [] if service.get("enabled", True)] pool_nodes = sum(service.get("nodes") or 0 for service in services) - infra_dedicated = recipe.get("infra", {}).get("etcd_nats_dedicated_node", False) + infra_dedicated = (recipe.get("infra") or {}).get("etcd_nats_dedicated_node", False) discovery = [service for service in services if service.get("type") in ("etcd", "nats")] if discovery: placements = { - service.get("placement", {}).get("node") == "dedicated" for service in discovery + (service.get("placement") or {}).get("node") == "dedicated" for service in discovery } if len(placements) != 1: raise ValueError("Recipe etcd and nats must agree on dedicated placement") @@ -222,12 +222,12 @@ def recipe_auxiliary_node_count(recipe: dict) -> int: ("frontend", "dedicated_node"), ("benchmark", "client_dedicated_node"), ): - settings = recipe.get(section, {}) + settings = recipe.get(section) or {} dedicated_roles += int( - settings.get("placement", {}).get("node") == "dedicated" + (settings.get("placement") or {}).get("node") == "dedicated" or settings.get(legacy_key, False) ) - if dedicated_roles and recipe.get("benchmark", {}).get("colocate_with_frontend", True): + if dedicated_roles and (recipe.get("benchmark") or {}).get("colocate_with_frontend", True): dedicated_roles = 1 return pool_nodes + dedicated_roles @@ -236,7 +236,7 @@ def recipe_node_count(root: Path, srt_recipe: str) -> int | None: path, _, selector = srt_recipe.partition(":") recipe_path = root / path recipe = yamlio.safe_load(recipe_path.read_text()) - if "base" in recipe: + if isinstance(recipe, dict) and "base" in recipe: if not (selector == "base" or selector.startswith("override_")): return None schema = recipe.get("schema") @@ -244,12 +244,12 @@ def recipe_node_count(root: Path, srt_recipe: str) -> int | None: recipe.setdefault("schema", schema) if recipe.get("schema") != 2: return None - if recipe.get("schema") != 2: + if not isinstance(recipe, dict) or recipe.get("schema") != 2: raise ValueError(f"srt-slurm recipes must declare schema: 2: {recipe_path}") roles = recipe.get("roles") if roles: for name, role in roles.items(): - if "nodes" not in role: + if not isinstance(role, dict) or role.get("nodes") is None: raise ValueError(f"Recipe role {name!r} must specify nodes: {recipe_path}") worker_nodes = sum( 0 if role["nodes"] == "colocate" else int(role["nodes"]) for role in roles.values() diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index 8e6dc4443f..912e5ffa05 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -119,6 +119,8 @@ def write(recipe: dict, name: str = "test.yaml") -> str: ({"prefill": {"nodes": 2}, "decode": {"nodes": 4}}, 6), ({"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}, 2), ({"prefill": {"nodes": 2}, "decode": {"workers": 1}}, None), + ({"prefill": {"nodes": 2}, "decode": {"nodes": None}}, None), + ({"prefill": {"nodes": 2}, "decode": None}, None), ]) def test_multinode_node_count_reads_schema_two_roles(tmp_path, multinode_recipe, roles, expected): import infx.matrix.generate as generate @@ -130,6 +132,28 @@ def test_multinode_node_count_reads_schema_two_roles(tmp_path, multinode_recipe, assert generate.recipe_node_count(tmp_path, srt_recipe) == expected +@pytest.mark.parametrize("sections", [ + {"services": None}, + {"infra": None}, + {"frontend": None}, + {"benchmark": None}, + {"services": [{"type": "etcd", "placement": None}, {"type": "nats"}]}, +]) +def test_recipe_node_count_treats_null_sections_as_absent(tmp_path, multinode_recipe, sections): + import infx.matrix.generate as generate + srt_recipe = multinode_recipe( + {"schema": 2, "roles": {"prefill": {"nodes": 1}, "decode": {"nodes": 2}}, **sections} + ) + assert generate.recipe_node_count(tmp_path, srt_recipe) == 3 + + +def test_recipe_node_count_rejects_an_empty_recipe(tmp_path, multinode_recipe): + import infx.matrix.generate as generate + srt_recipe = multinode_recipe(None) + with pytest.raises(ValueError, match="must declare schema: 2: .*test.yaml"): + generate.recipe_node_count(tmp_path, srt_recipe) + + @pytest.mark.parametrize("selector, expected", [ ("base", 3), ("override_wide", 5), From 999adf0142ed0a03e1a51555debfb82ba22692c9 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:13:16 -0500 Subject: [PATCH 041/203] fix(matrix): report a failing git diff when reading changelog additions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_added_lines ran git diff with check=False and never looked at the exit status. When the diff failed, its empty output surfaced as "No additions found in the changelog file." and git's own error was dropped. A non-zero git diff exit now raises a RuntimeError that carries git's stderr. Successful diffs are parsed as before. 中文:get_added_lines 以 check=False 运行 git diff,且从不检查退出码。diff 失败时,空输出会显示为 "No additions found in the changelog file.",git 自身的错误信息则被丢弃。现在 git diff 以非零状态退出时会抛出包含 git stderr 的 RuntimeError。成功的 diff 仍按原方式解析。 --- inferencex-e2e/infx/matrix/plan.py | 2 + .../tests/matrix/test_process_changelog.py | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index b4061a5b8e..a8cada4904 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -42,6 +42,8 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: capture_output=True, text=True, ) + if result.returncode != 0: + raise RuntimeError(f"git diff of {filepath} failed: {result.stderr.strip()}") added_lines = [] for line in result.stdout.split("\n"): diff --git a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py index 3ad666dbb3..8f969f8317 100644 --- a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py +++ b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py @@ -804,3 +804,43 @@ def git(*args): git("commit", "-qam", "append entry") monkeypatch.chdir(tmp_path / "inferencex-e2e") assert process_changelog.get_added_lines(base, "HEAD", "perf-changelog.yaml") == "new entry" + + +@pytest.fixture +def commit_changelog(tmp_path, monkeypatch): + """Commit ``perf-changelog.yaml`` with the given text in a scratch repo; return the SHA.""" + monkeypatch.chdir(tmp_path) + + def git(*args): + return subprocess.run( + ["git", *args], check=True, capture_output=True, text=True, + ).stdout.strip() + + git("init", "-q") + git("config", "user.name", "Test") + git("config", "user.email", "test@example.com") + + def commit(text): + (tmp_path / "perf-changelog.yaml").write_text(text) + git("add", ".") + git("commit", "-qm", "changelog") + return git("rev-parse", "HEAD") + + return commit + + +def test_added_lines_report_a_failing_git_diff(commit_changelog, tmp_path, monkeypatch): + base = commit_changelog("- old\n") + head = commit_changelog("- old\n- new\n") + stub = tmp_path / "bin/git" + stub.parent.mkdir() + stub.write_text( + "#!/bin/sh\n" + 'if [ "$1" = diff ]; then echo "fatal: diff exploded" >&2; exit 128; fi\n' + f'exec {shutil.which("git")} "$@"\n' + ) + stub.chmod(0o755) + monkeypatch.setenv("PATH", f"{stub.parent}{os.pathsep}{os.environ['PATH']}") + + with pytest.raises(RuntimeError, match="fatal: diff exploded"): + process_changelog.get_added_lines(base, head, "perf-changelog.yaml") From 0696d5b793bfba1528d89cbc021c73826a1fc085 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:14:33 -0500 Subject: [PATCH 042/203] fix(matrix): parse changelog diff headers only before the first hunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_added_lines skipped every diff line that started with --- or +++, treating it as a file header. Deleting a changelog line that itself starts with -- (for example a YAML --- separator) therefore bypassed the append-only check, and an added line starting with ++ was dropped. Header lines are now skipped only before the first @@ hunk marker, so such deletions are rejected and such additions are kept. Diffs without lines like these parse as before. 中文:get_added_lines 会跳过所有以 --- 或 +++ 开头的 diff 行,把它们当成文件头。因此删除一行本身以 -- 开头的 changelog 内容(例如 YAML 的 --- 分隔符)可以绕过只允许追加的检查,而以 ++ 开头的新增行会被丢弃。现在只在第一个 @@ hunk 标记之前跳过文件头,这类删除会被拒绝,这类新增会被保留。不含此类行的 diff 解析结果不变。 --- inferencex-e2e/infx/matrix/plan.py | 9 +++++++-- .../infx/tests/matrix/test_process_changelog.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index a8cada4904..9fa14013bb 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -46,8 +46,13 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: raise RuntimeError(f"git diff of {filepath} failed: {result.stderr.strip()}") added_lines = [] + in_hunk = False for line in result.stdout.split("\n"): - if line.startswith("-") and not line.startswith("---"): + if line.startswith("@@"): + in_hunk = True + elif not in_hunk: + continue + elif line.startswith("-"): deleted_content = line[1:] if deleted_content.strip(): raise ValueError( @@ -55,7 +60,7 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: f"Only additions to the changelog are permitted. " f"Found deleted line: {deleted_content}" ) - elif line.startswith("+") and not line.startswith("+++"): + elif line.startswith("+"): added_lines.append(line[1:]) return "\n".join(added_lines) diff --git a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py index 8f969f8317..5a9a550705 100644 --- a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py +++ b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py @@ -844,3 +844,16 @@ def test_added_lines_report_a_failing_git_diff(commit_changelog, tmp_path, monke with pytest.raises(RuntimeError, match="fatal: diff exploded"): process_changelog.get_added_lines(base, head, "perf-changelog.yaml") + + +def test_added_lines_reject_deleting_a_line_that_starts_with_dashes(commit_changelog): + base = commit_changelog("- a\n---\n- b\n") + head = commit_changelog("- a\n- b\n- c\n") + with pytest.raises(ValueError, match="Deletions are not allowed.*Found deleted line: ---"): + process_changelog.get_added_lines(base, head, "perf-changelog.yaml") + + +def test_added_lines_keep_an_added_line_that_starts_with_pluses(commit_changelog): + base = commit_changelog("- a\n") + head = commit_changelog("- a\n++ b\n") + assert process_changelog.get_added_lines(base, head, "perf-changelog.yaml") == "++ b" From 8fd3dfdf4c51b4a61bea5cb3e0752a0f96c49bdc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:57:41 -0500 Subject: [PATCH 043/203] fix(klaud): skip unreadable ownership records instead of aborting recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recovery read every owner's planner run and ownership artifact outside the per-candidate error handling. A family claim whose planner run was deleted, or an ownership artifact that failed to download or parse, raised out of recover and stopped reconciliation of every other candidate, the abandoned-work sweep and the blocked-families output. Such a family claim now keeps its family blocked and is reported as needing inspection, and such an artifact is skipped with a warning. Passes where every owner reads cleanly behave as before. 中文:恢复流程在逐个候选的错误处理之外读取每个所有者的规划运行和所有权构件。若某个配置族声明对应的规划运行已被删除,或某个所有权构件下载或解析失败,异常会直接抛出 recover,导致其他候选的收尾、遗弃工作清理以及被阻塞配置族的输出全部中止。现在这类配置族声明会继续阻塞其配置族并标记为需要检查,这类构件则被跳过并给出警告。所有所有者都能正常读取时,行为保持不变。 --- inferencex-e2e/infx/klaud/lifecycle.py | 48 ++++++++++++------ .../infx/tests/klaud/test_klaud_github.py | 49 +++++++++++++++++++ 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index b51f2cde12..e191f777a4 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -851,22 +851,43 @@ def recover(max_resumes: int) -> None: for artifact in inventory: if artifact["expired"]: continue - parent = github.read(repository, f"actions/runs/{artifact['workflow_run']['id']}") - if ( - parent["head_branch"] != "main" - or parent["path"] != ".github/workflows/klaud-plan.yml" - or parent["event"] not in ("schedule", "workflow_dispatch") - ): + try: + parent = github.read(repository, f"actions/runs/{artifact['workflow_run']['id']}") + if ( + parent["head_branch"] != "main" + or parent["path"] != ".github/workflows/klaud-plan.yml" + or parent["event"] not in ("schedule", "workflow_dispatch") + ): + continue + with tempfile.TemporaryDirectory(prefix="klaud-ownership-") as temp: + github.download_json(repository, artifact, Path(temp)) + owner = Ownership.model_validate_json((Path(temp) / "ownership.json").read_text()) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + print(f"::warning::Klaud: ownership artifact {artifact['id']} is unreadable; skipped.") continue - with tempfile.TemporaryDirectory(prefix="klaud-ownership-") as temp: - github.download_json(repository, artifact, Path(temp)) - owner = Ownership.model_validate_json((Path(temp) / "ownership.json").read_text()) if owner.run_id != parent["id"]: raise VerificationError("Ownership parent mismatch") owners.append((owner, artifact)) - parents = { - owner.run_id: github.read(repository, f"actions/runs/{owner.run_id}") for owner, _ in owners - } + results = {} + blocked = set() + live = set() + parents = {} + for owner, _ in owners: + if owner.run_id in parents: + continue + try: + parents[owner.run_id] = github.read(repository, f"actions/runs/{owner.run_id}") + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + for candidate in owner.candidates: + results[(owner.run_id, candidate.id)] = False + blocked.add(candidate.family) + summarize( + repository, + candidate, + owner.run_id, + "needs inspection / 需要检查 (state unavailable or invalid)", + ) + owners = [pair for pair in owners if pair[0].run_id in parents] if any( parent["head_branch"] != "main" or parent["path"] != ".github/workflows/klaud-plan.yml" @@ -874,9 +895,6 @@ def recover(max_resumes: int) -> None: for parent in parents.values() ): raise VerificationError("Untrusted ownership parent") - results = {} - blocked = set() - live = set() approved: list[tuple[Session, dict]] = [] for owner, _ in sorted(owners, key=lambda pair: pair[0].run_id, reverse=True): parent = parents[owner.run_id] diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 9a2ddc22f8..4a38fa2ab0 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -828,6 +828,55 @@ def test_recovery_keeps_the_pr_claim_and_lease_of_a_rerun_final_sweep(monkeypatc "sweep). [Parent run / 父运行](https://github.com/example/project/actions/runs/42)\n\n") +@pytest.mark.parametrize("stale", ["deleted-claim-parent", "corrupt-artifact"]) +def test_recovery_skips_an_unreadable_owner_and_reconciles_the_others( + monkeypatch, tmp_path, capsys, stale +): + world = ended_session_sweep(monkeypatch, jobs=["success", "cancelled"]) + other = OwnedCandidate(id="5" * 16 + "-" + "6" * 16, + family="configs/amd-master.yaml:other", base="a" * 40) + owners = [Ownership(run_id=42, candidates=[world.session.candidate])] + if stale == "deleted-claim-parent": + owners.append(Ownership(run_id=77, candidates=[other])) + monkeypatch.setattr(claims, "family_owners", lambda _r: owners) + serve, read = github.items, github.read + + def items(repository, path, key=None): + if path.startswith("actions/artifacts?") and stale == "corrupt-artifact": + return [{"id": 5, "expired": False, "size_in_bytes": 10, "workflow_run": {"id": 42}}] + return serve(repository, path, key) + + def gone(repository, path): + if path == "actions/runs/77": + raise subprocess.CalledProcessError(1, ["gh", "api"]) + return read(repository, path) + + def corrupt(*_args): + raise github.VerificationError("Invalid artifact archive") + + monkeypatch.setattr(github, "items", items) + monkeypatch.setattr(github, "read", gone) + monkeypatch.setattr(github, "download_json", corrupt) + + summary, _ = recovery_pass(monkeypatch, tmp_path) + + assert world.writes == [("POST", "actions/runs/9/rerun-failed-jobs")] + rerun = (f"Klaud `{world.session.candidate.id}`: pending / 等待后续收尾 (Reran preempted " + "final sweep). [Parent run / 父运行](https://github.com/example/project/actions/runs/42)" + "\n\n") + if stale == "deleted-claim-parent": + assert json.loads((tmp_path / "klaud-recovery.json").read_text()) == [ + "configs/amd-master.yaml:other", FAMILY] + assert summary.read_text() == ( + f"Klaud `{other.id}`: needs inspection / 需要检查 (state unavailable or invalid). " + "[Parent run / 父运行](https://github.com/example/project/actions/runs/77)\n\n" + rerun) + else: + assert json.loads((tmp_path / "klaud-recovery.json").read_text()) == [FAMILY] + assert summary.read_text() == rerun + assert "::warning::Klaud: ownership artifact 5 is unreadable; skipped.\n" in ( + capsys.readouterr().out) + + def test_recovery_warns_why_a_passed_final_sweep_failed_validation(monkeypatch, capsys): world = ended_session_sweep(monkeypatch, attempt=2, conclusion="success", artifacts=[{ "id": 1, "name": "klaud-sweep-manifest", "expired": False, From 7231c6dd94a00d2f4c6bff1a79762fe21ca797ae Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:00:07 -0500 Subject: [PATCH 044/203] fix(klaud): name baseline request-error notes by their concurrency row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR-body baseline table labels rows by bare concurrency when every point shares one shape, and its result and retirement notes then say c4. The request-error note appended the bare label instead, so it read "4: 2 request errors." under a table whose row is 4. Only baselines with the concurrency layout and a point with request errors change, and their note now reads "c4: 2 request errors." like the attempt tables do. 中文:当所有点形状相同时,PR 正文中的基线表按纯并发数标注各行,其结果和退役说明写作 c4。请求错误说明却直接使用了裸标签,于是显示为“4: 2 request errors.”。只有采用并发布局且存在请求错误的基线会变化,其说明现在与尝试表一致,写作“c4: 2 request errors.”。 --- inferencex-e2e/infx/klaud/reporting.py | 2 +- .../infx/tests/klaud/test_klaud_github.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index dc6bd20296..daccb33a99 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -563,7 +563,7 @@ def baseline_table(baseline: Baseline, *, context: bool = True) -> str: reason = f"{number(point.values.request_errors)} request error" + ( "s" if point.values.request_errors != 1 else "" ) - issues.setdefault(reason, []).append(label) + issues.setdefault(reason, []).append(shown) return "\n\n".join( part for part in ( diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 4a38fa2ab0..916bfabc53 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -2746,6 +2746,23 @@ def test_agentx_deltas_compare_only_the_same_replay_corpus(old, new, cell, note) assert notes == ([f"**Note:** All rows: Δ N/A: {note}."] if note else []) +def test_baseline_notes_name_concurrency_rows_like_their_table(): + points = [ + agentx_point("cc", output_tps_gpu=10.0, request_errors=2), + agentx_point("cc", output_tps_gpu=10.0).model_copy(update={ + "key": "e" * 64, "label": "AgentX c8 TP8 EP1 eeeeee", "conc": 8}), + ] # fmt: skip + baseline = reporting.Baseline( + family=f"configs/nvidia-master.yaml:{KEY}", date="2026-09-19", image="example/image:1", + goal=reporting.Prose(en="Update the image.", zh="更新镜像。"), sources=[], points=points, + ) # fmt: skip + + text = reporting.baseline_table(baseline) + + assert "| 4 | N/A | 10 | N/A | N/A |" in text + assert text.endswith("**Note:** c4: 2 request errors.") + + def eval_entry(tp): return { "model-prefix": "dsr1", "runner": "cluster:fixture-a", "framework": "sglang", From 2caf65d1c1663a2b33c27c94b53179aee1a25133 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:00:49 -0500 Subject: [PATCH 045/203] fix(klaud): fail source reads of a directory with a verification error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contents API returns a list when the requested path is a directory, and file_at called .get on it. The resulting AttributeError escaped every Klaud error handler, so a candidate head whose recipe reference names a directory aborted the whole recovery pass and printed a traceback from check-final. file_at now raises its fixed "Source data unavailable or too large" error for any non-file response. File reads are unchanged. 中文:当请求路径是目录时,contents API 返回的是列表,而 file_at 直接对其调用 .get。由此产生的 AttributeError 会越过 Klaud 的所有错误处理,因此若候选提交中的 recipe 引用指向目录,整个恢复流程会中止,check-final 也会打印回溯信息。现在对任何非文件响应,file_at 都会抛出固定的“Source data unavailable or too large”错误。文件读取不受影响。 --- inferencex-e2e/infx/klaud/github.py | 3 ++- .../infx/tests/klaud/test_klaud_github.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/klaud/github.py b/inferencex-e2e/infx/klaud/github.py index bdb5ef5b2e..b7770ff27b 100644 --- a/inferencex-e2e/infx/klaud/github.py +++ b/inferencex-e2e/infx/klaud/github.py @@ -43,7 +43,8 @@ def file_at(repository: str, head: str, path: str) -> bytes: raise VerificationError("Expected an immutable source revision") result = read(repository, f"contents/{quote(path, safe='/')}?ref={head}") if ( - result.get("type") != "file" + not isinstance(result, dict) + or result.get("type") != "file" or result.get("encoding") != "base64" or result["size"] > 4 * 1024 * 1024 ): diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 916bfabc53..6bedc8358a 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -65,6 +65,24 @@ def run(args, **kwargs): assert github.write("example/project", "git/refs/heads/claim", "DELETE") == {} +@pytest.mark.parametrize(("contents", "data"), [ + ({"type": "file", "encoding": "base64", "size": 3, "content": "YWJj\n"}, b"abc"), + ([{"type": "file", "name": "recipe.yaml", "path": "recipes/recipe.yaml"}], None), +], ids=["file", "directory"]) # fmt: skip +def test_source_reads_return_file_bytes_or_a_fixed_failure(monkeypatch, contents, data): + def run(args, **_kwargs): + assert args[4] == f"repos/example/project/contents/recipes?ref={'a' * 40}" + return subprocess.CompletedProcess(args, 0, json.dumps(contents), "") + + monkeypatch.setattr(github.subprocess, "run", run) + + if data is None: + with pytest.raises(github.VerificationError, match="Source data unavailable or too large"): + github.file_at("example/project", "a" * 40, "recipes") + else: + assert github.file_at("example/project", "a" * 40, "recipes") == data + + @pytest.mark.parametrize( "failure,reason", [ From 2c25ca62d57a156e63a74409000972be45021de7 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:01:39 -0500 Subject: [PATCH 046/203] fix(klaud): report a missing candidate PR instead of crashing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report and check-final commands indexed the session's PR list directly. Run before the candidate PR exists, they raised an IndexError that no Klaud handler catches, so the agent saw a traceback instead of the fixed error line every other failure prints. Both now fail with "Candidate PR not found" and exit 1. Sessions with a PR are unchanged. 中文:report 和 check-final 命令直接按下标读取会话的 PR 列表。若在候选 PR 创建之前运行,它们会抛出 Klaud 任何处理器都不捕获的 IndexError,代理看到的是回溯信息,而不是其他失败都会输出的固定错误行。现在两者都会以“Candidate PR not found”失败并返回 1。已有 PR 的会话不受影响。 --- inferencex-e2e/infx/klaud/__main__.py | 5 ++++- inferencex-e2e/infx/klaud/reporting.py | 5 ++++- .../infx/tests/klaud/test_klaud_github.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/klaud/__main__.py b/inferencex-e2e/infx/klaud/__main__.py index 54cbc447a4..84d336cb41 100644 --- a/inferencex-e2e/infx/klaud/__main__.py +++ b/inferencex-e2e/infx/klaud/__main__.py @@ -1195,7 +1195,10 @@ def main() -> int: from .validation import canonical_matrix, check_matrix session = current_session() - pull = session.pulls()[0] + pulls = session.pulls() + if not pulls: + raise VerificationError("Candidate PR not found") + pull = pulls[0] head = pull["head"]["sha"] matrix = json.loads(args.matrix_file.read_text()) canonical = canonical_matrix(session.repository, head, session.candidate.family) diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index daccb33a99..d00dfb510d 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -781,7 +781,10 @@ def check_reported_datasets(record: Attempt) -> None: def publish(session: Session, record: Baseline | Attempt) -> None: - pull = session.pulls()[0] + pulls = session.pulls() + if not pulls: + raise VerificationError("Candidate PR not found") + pull = pulls[0] session.refresh(pull) if isinstance(record, Baseline): if record.family != session.candidate.family: diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 6bedc8358a..dfddd07f66 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -240,6 +240,23 @@ def test_finish_requires_specific_baseline_reason_before_touching_session( ) +@pytest.mark.parametrize("command", ["report", "check-final"]) +def test_pr_commands_fail_cleanly_before_the_candidate_pr_exists( + tmp_path, monkeypatch, capfd, command +): + record = tmp_path / "record.json" + record.write_text(reporting.Baseline( + family=FAMILY, date="2026-09-19", image="example/image:1", + goal=reporting.Prose(en="Update the image.", zh="更新镜像。"), sources=[], points=[], + ).model_dump_json(by_alias=True)) # fmt: skip + options = (["--kind", "baseline", "--file"] if command == "report" else ["--matrix-file"]) + monkeypatch.setattr(sys, "argv", ["klaud", command, *options, str(record)]) + monkeypatch.setattr(lifecycle, "current_session", lambda: SimpleNamespace(pulls=lambda: [])) + + assert klaud.main() == 1 + assert capfd.readouterr().out == "::error::Klaud: Candidate PR not found.\n" + + def test_cooldown_skips_recent_blocks_across_bases_and_retries_capacity_deferrals( monkeypatch, ): From 9f47d99aad69f0ba77ff692282e46bceac176c2d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:02:48 -0500 Subject: [PATCH 047/203] fix(klaud): let finish continue when a run ends before its cancel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishing a failed or deferred candidate cancels each owned run that was still running when listed. A run that ended in between makes GitHub refuse the cancel, and that error failed the whole finish with "State unavailable" even though nothing was left to stop. When a cancel request fails, finish now reads the run again and continues only if it has ended. A cancel that fails while the run is still active still fails finish. 中文:结束失败或延期的候选时,会取消列出时仍在运行的每个自有运行。若某个运行在此期间已结束,GitHub 会拒绝取消请求,这个错误会让整个 finish 以“State unavailable”失败,尽管已无需停止任何内容。现在取消请求失败时,finish 会重新读取该运行,仅在其已结束时继续。若运行仍在进行而取消失败,finish 仍会失败。 --- inferencex-e2e/infx/klaud/lifecycle.py | 7 ++- .../infx/tests/klaud/test_klaud_github.py | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index e191f777a4..8a0fa2df75 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -426,7 +426,12 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: self.check_parent() if pull: self.refresh(pull) - github.write(self.repository, f"actions/runs/{run['id']}/cancel", "POST") + try: + github.write(self.repository, f"actions/runs/{run['id']}/cancel", "POST") + except (OSError, subprocess.SubprocessError): + # GitHub refuses to cancel a run that ended after it was listed. + if not terminal(github.read(self.repository, f"actions/runs/{run['id']}")): + raise if any(not terminal(run) for run in self.runs()): raise PendingCleanup( "Cancellation requested; wait for terminal jobs then finish again" diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index dfddd07f66..8217a9c86e 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -2403,6 +2403,49 @@ def finish_report(monkeypatch, outcome, comments, recovering=False): return report.split("\n-->\n", 1)[1] +@pytest.mark.parametrize("ended", [True, False], ids=["ended-meanwhile", "still-running"]) +def test_finish_tolerates_only_a_cancel_refused_because_the_run_already_ended( + monkeypatch, ended +): + session = lifecycle.Session( + "example/project", {"id": 42}, OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40) + ) + pull = {"number": 7, "head": {"sha": "c" * 40}, "merged_at": None, "draft": True, + "state": "closed", "labels": []} + run = {"id": 9, "event": "pull_request", "head_sha": "c" * 40, "run_attempt": 1, + "created_at": "2026-10-15T00:00:00Z", "status": "in_progress", "conclusion": None} + finished = {**run, "status": "completed", "conclusion": "success"} + current, writes = [run], [] + + def write(_repository, path, method, _payload=None): + writes.append((method, path)) + if path == "actions/runs/9/cancel": + if ended: + current[0] = finished + raise subprocess.CalledProcessError(1, ["gh", "api"]) + return {} + + monkeypatch.setattr(session, "pulls", lambda: [pull]) + monkeypatch.setattr(session, "runs", lambda: list(current)) + monkeypatch.setattr(session, "refresh", lambda _pull: pull) + monkeypatch.setattr(claims, "release_family", lambda *_: None) + monkeypatch.setattr(github, "items", lambda *_args: []) + monkeypatch.setattr(github, "read", lambda _repository, path: ( + current[0] if path == "actions/runs/9" else pytest.fail(path))) + monkeypatch.setattr(github, "write", write) + outcome = CandidateOutcome(outcome="failed", phase="targeted", pull_request=7, run_ids=[9], + repairs_used=0) + + if ended: + assert session.finish(outcome).run_ids == [9] + assert writes == [("POST", "issues/7/comments"), ("POST", "actions/runs/9/cancel"), + ("POST", "issues/7/comments")] + else: + with pytest.raises(subprocess.CalledProcessError): + session.finish(outcome) + assert writes == [("POST", "issues/7/comments"), ("POST", "actions/runs/9/cancel")] + + @pytest.mark.parametrize(("labels", "accepted"), [ (["full-sweep-fail-fast"], True), (["full-sweep-enabled"], False), From aca70bc5366b6d58cc0887535af3c453d1191b5c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:06:23 -0500 Subject: [PATCH 048/203] perf(klaud): reuse session listings until the next write or ownership check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recovering or finishing session listed the same PRs, resume runs, owned runs, comments, branch refs and candidate jobs again on nearly every step. Each Session now keeps these listings until any GitHub write or its next ownership check, which still reads the parent run and resume runs fresh before every write. One candidate recovered through cleanup and finish in the test fakes went from 85 to 67 GitHub reads, with the same writes and summary. Nothing a write or ownership check would observe changes. 中文:恢复或结束中的会话几乎每一步都会重新列出相同的 PR、恢复运行、自有运行、评论、分支引用和候选作业。现在每个 Session 会保留这些列表,直到发生任何 GitHub 写入或下一次所有权检查,而该检查在每次写入前仍会重新读取父运行和恢复运行。在测试桩中,一个候选经过清理和结束的完整恢复流程,GitHub 读取次数从 85 次降至 67 次,写入和摘要保持不变。写入或所有权检查所能观察到的内容均未改变。 --- inferencex-e2e/infx/klaud/github.py | 6 +++++ inferencex-e2e/infx/klaud/lifecycle.py | 31 +++++++++++++++----------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/inferencex-e2e/infx/klaud/github.py b/inferencex-e2e/infx/klaud/github.py index b7770ff27b..0a8d55d8e2 100644 --- a/inferencex-e2e/infx/klaud/github.py +++ b/inferencex-e2e/infx/klaud/github.py @@ -27,7 +27,13 @@ def items(repository: str, path: str, key: str | None = None) -> list[dict]: raise VerificationError(str(error)) from error +# Every write bumps this, so listings cached before it are read again. +writes = 0 + + def write(repository: str, path: str, method: str, payload: dict | None = None) -> dict: + global writes # noqa: PLW0603 + writes += 1 return github.api(repository, path, method=method, data=payload) diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index 8a0fa2df75..ee35965088 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -95,6 +95,14 @@ def __init__( self.marker = f"", 1)[0]] + english, chinese = main.split("
") + assert "Results continue in numbered report comments." in english + assert "结果见后续编号的报告评论。" in chinese + + def eval_entry(tp): return { "model-prefix": "dsr1", "runner": "cluster:fixture-a", "framework": "sglang", From 45969e451101d1133f98e2e392651efad9da3ad7 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:18:32 -0500 Subject: [PATCH 126/203] fix(klaud): translate an unknown repair count in closing reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The closing report printed the English word "unknown" in its Chinese section whenever the outcome had no repair count. Recovery always closes a session with an unknown count, so every PR that recovery closed showed untranslated text to Chinese readers. Only outcomes without a repair count change; their Chinese line now reads 未知. 中文:只要结果中没有修复次数,收尾报告的中文部分就会显示英文单词 "unknown"。恢复流程关闭会话时修复次数总是未知,因此由恢复流程关闭的每个 PR 都会向中文读者显示未翻译的文本。只有没有修复次数的结果会改变,其中文行现在显示为未知。 --- inferencex-e2e/infx/klaud/lifecycle.py | 3 ++- inferencex-e2e/infx/tests/klaud/test_klaud_github.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index 71d20cf3f6..75df9c5c62 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -499,6 +499,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: or "—" ) repairs = outcome.repairs_used if outcome.repairs_used is not None else "unknown" + repairs_zh = outcome.repairs_used if outcome.repairs_used is not None else "未知" reason_en = f" · Reason: {outcome.reason_code}" if outcome.reason_code else "" reason_zh = f" · 原因:{outcome.reason_code}" if outcome.reason_code else "" why = ( @@ -521,7 +522,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: if proof else "PR closed; branch deleted for retry." ), - f"**{outcome.outcome}**{reason_zh} · 修复次数:{repairs} · 运行:{links} \n{why_zh}所有自有运行均已结束。" + f"**{outcome.outcome}**{reason_zh} · 修复次数:{repairs_zh} · 运行:{links} \n{why_zh}所有自有运行均已结束。" + ( "完整 sweep 已验证;已就绪,等待审查。" if proof diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 782da42c41..4dd9807ee5 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -2435,7 +2435,7 @@ def attempt_comment(number, en, zh, *, author="Klaud-Cold", parent=42, candidate f"\nRendered.")} -def finish_report(monkeypatch, outcome, comments, recovering=False): +def finish_report(monkeypatch, outcome, comments, recovering=False, repairs=0): session = lifecycle.Session( "example/project", {"id": 42}, OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40), recovering=recovering, @@ -2461,7 +2461,7 @@ def finish_report(monkeypatch, outcome, comments, recovering=False): session.finish(CandidateOutcome( outcome=outcome, phase="final-sweep" if validated else "targeted", pull_request=7, - run_ids=[9], repairs_used=0, + run_ids=[9], repairs_used=repairs, )) [report] = [body for body in posted if body.startswith(session.marker)] @@ -2594,6 +2594,14 @@ def test_closing_report_states_why_from_the_latest_own_attempt( ) +def test_closing_report_translates_an_unknown_repair_count(monkeypatch): + report = finish_report(monkeypatch, "uncertain", [], recovering=True, repairs=None) + english, chinese = report.split("中文") + + assert "· Repairs: unknown ·" in english + assert "· 修复次数:未知 ·" in chinese + + def test_every_candidate_pr_body_says_how_to_stop_the_candidate(monkeypatch): body = reporting.render_body(frozen_dsr1_baseline(monkeypatch)) english, chinese = body.split("中文") From c5035c1ad24fced8c82c2cf7a443c4a7fda7590c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:38:56 -0500 Subject: [PATCH 127/203] fix(changelog): report a tagged non-mapping entry as invalid YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The duplicate-key loader the changelog tools use unpacked every node tagged as a mapping, so an entry such as `- !!map config-a` raised a bare ValueError from tuple unpacking instead of PyYAML's ConstructorError. The validators then crashed with a traceback instead of reporting the file as invalid YAML. The loader now rejects a non-mapping node with SafeLoader's own message; well-formed changelogs load as before. 中文:changelog 工具使用的拒绝重复键加载器会对每个被标记为映射的节点进行解包,因此像 `- !!map config-a` 这样的条目会因元组解包抛出裸 ValueError,而不是 PyYAML 的 ConstructorError,校验器随即以堆栈跟踪崩溃,而不是报告文件不是有效的 YAML。现在加载器会 用 SafeLoader 自身的信息拒绝非映射节点;格式正确的 changelog 加载结果不变。 --- .../infx/tests/workflows/test_validate_perf_changelog.py | 5 +++++ inferencex-e2e/infx/yamlio.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py b/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py index 73521ba264..1b2519d4b2 100644 --- a/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py +++ b/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py @@ -82,6 +82,11 @@ def test_parse_changelog_rejects_duplicate_mapping_keys() -> None: parse_changelog(raw, "test changelog") +def test_parse_changelog_reports_a_tagged_non_mapping_entry_as_invalid_yaml() -> None: + with pytest.raises(ChangelogValidationError, match="expected a mapping node, but found scalar"): + parse_changelog(b"- !!map config-a\n", "test changelog") + + def test_compare_entries_allows_appended_pr_entry() -> None: base = [entry("config-a")] added = entry("config-b", "XXX") diff --git a/inferencex-e2e/infx/yamlio.py b/inferencex-e2e/infx/yamlio.py index d0e3dd1b09..9179cb6904 100644 --- a/inferencex-e2e/infx/yamlio.py +++ b/inferencex-e2e/infx/yamlio.py @@ -171,10 +171,14 @@ class UniqueKeyLoader(yaml.SafeLoader): def construct_unique_mapping( loader: UniqueKeyLoader, - node: yaml.MappingNode, + node: yaml.Node, deep: bool = False, ) -> dict[Any, Any]: """Construct a mapping while rejecting keys that would overwrite values.""" + if not isinstance(node, yaml.MappingNode): + raise ConstructorError( + None, None, f"expected a mapping node, but found {node.id}", node.start_mark + ) mapping: dict[Any, Any] = {} for key_node, value_node in node.value: key = loader.construct_object(key_node, deep=deep) From 3da2f26c8df70774a055697041d0762f4a316c02 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:25:52 -0500 Subject: [PATCH 128/203] fix(bench): name every physical GPU in multi-node sweep result files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-node fixed-sequence sweep named each result with prefill and decode GPU counts of workers times TP only, while the result processor and the AgentX path count TP x PP x PCP (x DWDP for prefill) per worker. A fixed-sequence row with prefill PP, PCP or DWDP above 1 would get too few GPUs in its filename, inflating per-GPU throughput and failing power GPU-count checks. 中文:多节点固定序列 sweep 在结果文件名中只按 worker 数乘以 TP 计算 prefill 和 decode 的 GPU 数, 而结果处理器和 AgentX 路径按每个 worker 的 TP x PP x PCP(prefill 另乘 DWDP)计数。 当固定序列配置的 prefill PP、PCP 或 DWDP 大于 1 时,文件名中的 GPU 数偏少, 会夸大每 GPU 吞吐量并导致功耗 GPU 数校验失败。 --- inferencex-e2e/infx/bench/fixed_seq.py | 12 ++++++-- .../infx/tests/bench/test_fixed_seq.py | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/bench/fixed_seq.py b/inferencex-e2e/infx/bench/fixed_seq.py index 15704cfe11..c856255c7c 100644 --- a/inferencex-e2e/infx/bench/fixed_seq.py +++ b/inferencex-e2e/infx/bench/fixed_seq.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import math from dataclasses import dataclass from pathlib import Path @@ -120,13 +121,18 @@ def srt_sweep(args: argparse.Namespace) -> int: """Every ``CONC_LIST`` point of an srt-slurm multi-node job; srt-slurm owns the servers.""" values = env.require( "ISL", "OSL", "RANDOM_RANGE_RATIO", "SRT_FRONTEND_HOST", "SRT_FRONTEND_PORT", - "CONC_LIST", "PREFILL_NUM_WORKERS", "PREFILL_TP", "DECODE_NUM_WORKERS", "DECODE_TP", + "CONC_LIST", "PREFILL_NUM_WORKERS", "PREFILL_TP", "PREFILL_PP_SIZE", "PREFILL_PCP_SIZE", + "PREFILL_DWDP_SIZE", "DECODE_NUM_WORKERS", "DECODE_TP", "DECODE_PP_SIZE", "DECODE_PCP_SIZE", ) # fmt: skip isl = env.positive_int("ISL") osl = env.positive_int("OSL") base_url = f"http://{values['SRT_FRONTEND_HOST']}:{env.positive_int('SRT_FRONTEND_PORT')}" - ctx = env.non_negative_int("PREFILL_NUM_WORKERS") * env.positive_int("PREFILL_TP") - gen = env.non_negative_int("DECODE_NUM_WORKERS") * env.positive_int("DECODE_TP") + ctx = env.non_negative_int("PREFILL_NUM_WORKERS") * math.prod( + env.positive_int(f"PREFILL_{size}") for size in ("TP", "PP_SIZE", "PCP_SIZE", "DWDP_SIZE") + ) + gen = env.non_negative_int("DECODE_NUM_WORKERS") * math.prod( + env.positive_int(f"DECODE_{size}") for size in ("TP", "PP_SIZE", "PCP_SIZE") + ) concs = [env.parse_positive_int("CONC_LIST", word) for word in values["CONC_LIST"].split()] windows = env.optional("SRT_MEASUREMENT_WINDOW_DIR") model = served_model(base_url) diff --git a/inferencex-e2e/infx/tests/bench/test_fixed_seq.py b/inferencex-e2e/infx/tests/bench/test_fixed_seq.py index 9919f20b5c..1765ea4ce7 100644 --- a/inferencex-e2e/infx/tests/bench/test_fixed_seq.py +++ b/inferencex-e2e/infx/tests/bench/test_fixed_seq.py @@ -151,8 +151,13 @@ def sweep_env( "CONC_LIST": "4 16", "PREFILL_NUM_WORKERS": "1", "PREFILL_TP": "2", + "PREFILL_PP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_DWDP_SIZE": "1", "DECODE_NUM_WORKERS": "2", "DECODE_TP": "4", + "DECODE_PP_SIZE": "1", + "DECODE_PCP_SIZE": "1", "TOKENIZER": None, "SRT_MEASUREMENT_WINDOW_DIR": None, **overrides, @@ -340,6 +345,30 @@ def test_sweep_stops_at_the_first_failing_point(tmp_path, tools, http_server, mo assert [path.name for path in point_dir.iterdir()] == ["results_concurrency_4_gpus_4_ctx_4_gen_0.json"] +@pytest.mark.parametrize( + ("topology", "name"), + [ + ( + {"PREFILL_TP": "2", "PREFILL_PP_SIZE": "2", "DECODE_TP": "4", "DECODE_PCP_SIZE": "2"}, + "results_concurrency_4_gpus_20_ctx_4_gen_16.json", + ), + ( + {"PREFILL_NUM_WORKERS": "3", "PREFILL_TP": "1", "PREFILL_DWDP_SIZE": "4"}, + "results_concurrency_4_gpus_20_ctx_12_gen_8.json", + ), + ], +) +def test_sweep_names_every_physical_gpu_of_each_worker( + tmp_path, tools, http_server, monkeypatch, topology, name +): + url = http_server(frontend("served/model-id")) + use_env(monkeypatch, sweep_env(tmp_path, tools, url, CONC_LIST="4", **topology)) + + assert bench(["fixed-seq", "srt-sweep", "--logs-dir", str(tmp_path / "logs")]) == 0 + point_dir = tmp_path / "logs" / "sa-bench_isl_1024_osl_128" + assert [path.name for path in point_dir.iterdir()] == [name] + + def test_sweep_without_a_served_model_runs_nothing( tmp_path, tools, http_server, monkeypatch, capsys ): From 94314a739c36aa5e803a3850eb3286b828463ec3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:35:03 -0500 Subject: [PATCH 129/203] fix(matrix): reject plan rows for sequence lengths no sweep job runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan grouped a fixed-sequence row under its isl/osl name, falling back to "_" for pairs other than 1k1k and 8k1k, but run-sweep.yml only dispatches the 1k1k, 8k1k and agentic groups. A master config with any other sequence length therefore planned rows that no job ran, and the sweep still succeeded. Planning now fails and names the row and the supported groups; 1k1k, 8k1k and agentic rows are unchanged. 中文:计划按 isl/osl 名称对固定序列行分组,对 1k1k 和 8k1k 以外的组合回退为 "_",但 run-sweep.yml 只分派 1k1k、8k1k 和 agentic 分组。因此主配置中任何其他序列长度都会生成没有任何任务运行的行,而扫描仍显示成功。现在计划会失败并指出该行以及受支持的分组;1k1k、8k1k 和 agentic 行不受影响。 --- inferencex-e2e/infx/matrix/plan.py | 18 ++++++++++++------ .../tests/matrix/test_process_changelog.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index f5deb44504..5d53aada72 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -15,7 +15,7 @@ drop_app_colliding_evals, freeze_config_value, generate_config_matrix, - seq_len_to_str, + seq_len_itos, trim_conc, ) from .revision import GENERATOR, snapshot @@ -431,11 +431,17 @@ def generate_current( for result in all_benchmark_results: result["recipe-fingerprint"] = recipe_fingerprint(result, root) node_type = "multi_node" if result.get("prefill") is not None else "single_node" - scenario = ( - "agentic" - if result.get("scenario-type") == "agentic-coding" - else seq_len_to_str(result["isl"], result["osl"]) - ) + if result.get("scenario-type") == "agentic-coding": + scenario = "agentic" + else: + scenario = seq_len_itos.get((result["isl"], result["osl"])) + # run-sweep.yml dispatches only these groups; any other would never run. + if scenario is None: + raise ValueError( + f"{result['exp-name']}: fixed-sequence isl={result['isl']} " + f"osl={result['osl']} has no sweep job; supported: " + f"{', '.join(seq_len_itos.values())}" + ) final_results[node_type][scenario].append(result) all_eval_results = [ diff --git a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py index 55f3dd8c8e..8909b1389a 100644 --- a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py +++ b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py @@ -679,6 +679,16 @@ def test_generation_failure_does_not_publish_a_partial_matrix(planning_repo, cha assert '\"single_node\":' not in captured.out +@pytest.mark.parametrize("key", ["single", "multi"]) +def test_plan_rejects_a_sequence_length_no_sweep_job_runs(planning_repo, changelog_run, key): + root, master, _ = planning_repo + master[key]["scenarios"]["fixed-seq-len"][0].update(isl=1024, osl=8192) + (root / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master, sort_keys=False)) + + with pytest.raises(ValueError, match=r"isl=1024 osl=8192 .*1k1k, 8k1k"): + changelog_run([{"config-keys": [key], "scenario-type": ["fixed-seq-len"], "no-evals": True}]) + + def test_a_master_config_key_defined_twice_is_rejected(planning_repo, changelog_run): root, master, _ = planning_repo shadow = {"single": {**master["single"], "image": "example/image:shadow"}} From 780acc83439cc5206524b77e590873a3a3d8e8e2 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:37:36 -0500 Subject: [PATCH 130/203] fix(matrix): trim single-node bundle points to one row per deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generator CLI now fingerprints rows before eval selection, and trim_conc grouped rows by every field except concurrency and eval settings, so the fingerprint kept apart points of one single-node bundle whose variant is chosen per CONC. With --trim-conc (the trim-conc input of e2e-tests.yml) those deployments kept every concurrency instead of the lowest; on the current configs 1490 rows remained where 773 are expected. Trimming now ignores the fingerprint, as changelog plans already did by trimming before fingerprinting; the kept row keeps its own fingerprint. 中文:生成器 CLI 现在在评测选择之前为各行计算指纹,而 trim_conc 按并发与评测设置以外的所有字段分组,因此按 CONC 选择变体的单节点 bundle 的各个点会因指纹不同而无法合并。使用 --trim-conc(e2e-tests.yml 的 trim-conc 输入)时,这些部署保留了所有并发而不是最低并发;在当前配置下剩余 1490 行,而预期为 773 行。现在裁剪时忽略指纹,与变更日志计划先裁剪后计算指纹的行为一致;保留的行仍带有其自身的指纹。 --- inferencex-e2e/infx/matrix/generate.py | 2 ++ .../infx/tests/matrix/test_generate_sweep_configs.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 7bedd382b5..d8edf6c4d8 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -58,6 +58,8 @@ def trim_conc(entries: list[dict]) -> list[dict]: ignored_fields = { "conc", "exp-name", + # Single-node bundles bind one variant per CONC, so the fingerprint varies with it. + Fields.RECIPE_FINGERPRINT.value, "run-eval", "eval-only", "eval-conc", diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index 241deb9917..4095ad274f 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -2295,6 +2295,16 @@ def test_trim_conc_updates_multinode_dispatch_concurrency(self): assert result[0]['eval-conc'] == 4 assert result[0]['run-eval'] is True + def test_trim_conc_merges_concurrencies_whose_recipe_variants_differ(self): + # A single-node bundle selects one variant per CONC, so each point has its own fingerprint. + shape = {'tp': 8, 'model': 'm', 'srt-recipe': 'agentic.yaml'} + rows = [ + {**shape, 'conc': 8, 'recipe-fingerprint': 'b' * 64}, + {**shape, 'conc': 2, 'recipe-fingerprint': 'a' * 64}, + ] + + assert trim_conc(rows) == [{**shape, 'conc': 2, 'recipe-fingerprint': 'a' * 64}] + def test_trim_conc_never_merges_a_standalone_eval_into_another_suite(self): shape = {'tp': 8, 'model': 'm'} vendor = {**shape, 'conc': 16, 'run-eval': True, From 3e6c2ca6c48aae0d7e67bb2541d80061751e5599 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 04:41:39 -0500 Subject: [PATCH 131/203] docs(configs): list the fixed-sequence lengths PR sweeps run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The config guide gave 1024/8192 as an example sequence length, a pair that run-sweep.yml never dispatched and that the changelog planner now rejects. The guide now names the two supported pairs, 1024/1024 and 8192/1024, and states that any other pair fails planning. 中文:配置指南曾以 1024/8192 作为序列长度示例,但 run-sweep.yml 从不分派该组合,而变更日志计划器现在会拒绝它。指南现在列出两个受支持的组合 1024/1024 和 8192/1024,并说明任何其他组合都会导致计划失败。 --- inferencex-e2e/configs/CONFIGS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index 1eb2e42c2a..da1be8d4d5 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -98,8 +98,9 @@ The below list describes what each field is: scheduling. - `scenarios`: A dictionary of benchmark scenario types. At least one must be specified. Currently supported: - `fixed-seq-len`: Fixed input/output sequence length benchmarks. Each entry must have: - - `isl`: An integer representing the input sequence length, e.g., `1024` - - `osl`: An integer representing the output sequence length, e.g., `8192` + - `isl`: An integer representing the input sequence length, e.g., `8192` + - `osl`: An integer representing the output sequence length, e.g., `1024` + - PR sweeps run only `isl`/`osl` pairs 1024/1024 and 8192/1024; the changelog planner rejects any other pair. - `search-space`: A list of configurations to run with respective `isl` and `osl`, each entry must be a dict with the following fields: - `tp`: A positive integer representing the tensor parallelism level that the configuration will be served at. - `conc-start`: An integer representing the starting level of concurrency e.g., `4` From 000a10b42e6fbe5cbc847fcd05eec85f0dea8c7a Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:22:28 -0500 Subject: [PATCH 132/203] fix(yaml): match PyYAML where adversarial testing found libyaml differing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A grammar fuzzer, exhaustive token sequences and threaded probes found inputs where infx.yamlio returned what yaml.load would not. libyaml accepts an empty explicit key in a flow collection followed by ']', ',' or ':', which PyYAML rejects. Str subclasses, iterated or closed files, streams that need a read size and files that fail to decode late raised different errors. Constructors or methods added after the first load were ignored, and a table change racing a load could leave a stale twin. A cached int ignored a lower int digit limit. Such documents and loaders now go to PyYAML, only exact str and seekable TextIOWrapper or StringIO streams take the libyaml path, and on any fallback the stream is rewound and handed to yaml.load. The twin parses with copies of the tables and is rebuilt, with the cache cleared, when the tables or any attribute of the loader's classes change, and the cache key includes the digit limit. What remains is documented on load, namely RecursionError within a few frames of the limit, and PyYAML internals patched before import or mutated in place. 中文:语法模糊测试、穷举记号序列和多线程探测发现了 infx.yamlio 返回与 yaml.load 不同结果 的输入。libyaml 接受流式集合中后跟 ']'、',' 或 ':' 的空显式键,而 PyYAML 会拒绝;str 子类、已迭代或已关闭的文件、需要指定读取长度的流以及在靠后位置才解码失败的文件会抛出不同 的错误;首次加载后添加的构造器或方法被忽略,与加载并发的查找表变更可能留下过期的孪生类; 缓存的整数忽略了更低的整数位数限制。现在这些文档和加载器交给 PyYAML,只有精确的 str 以及 可定位的 TextIOWrapper 或 StringIO 流走 libyaml 路径,任何回退都会把流倒回原位置再交给 yaml.load。孪生类使用查找表的副本解析,当查找表或加载器各个类的任何属性发生变化时会重建 并清空缓存;缓存键包含位数限制。剩余差异已在 load 的文档中说明:距离递归上限几帧以内的 RecursionError,以及在导入前被修补或被原地修改的 PyYAML 内部实现。 --- inferencex-e2e/infx/tests/test_yamlio.py | 129 +++++++++++++++++++ inferencex-e2e/infx/yamlio.py | 152 +++++++++++++++++------ 2 files changed, 243 insertions(+), 38 deletions(-) diff --git a/inferencex-e2e/infx/tests/test_yamlio.py b/inferencex-e2e/infx/tests/test_yamlio.py index 6f2967cbec..64a336ddd4 100644 --- a/inferencex-e2e/infx/tests/test_yamlio.py +++ b/inferencex-e2e/infx/tests/test_yamlio.py @@ -1,3 +1,6 @@ +import io +import sys + import pytest import yaml @@ -12,6 +15,7 @@ ("a: 1\n\ufeff\nb: 2\n", "could not find expected ':'"), ("a: |#\n x\n", "expected chomping or indentation indicators"), ("[!!str, a]\n", "could not determine a constructor for the tag"), + ("[?]]\n", "expected '', but found ']'"), ], ids=[ "tab-in-plain-scalar", @@ -19,6 +23,7 @@ "inner-byte-order-mark", "comment-on-block-scalar-header", "tag-before-flow-comma", + "token-after-empty-flow-key", ], ) def test_documents_only_libyaml_accepts_raise_pyyamls_error(text, message): @@ -53,6 +58,130 @@ def test_stream_errors_name_the_file_with_pyyamls_wording(tmp_path): ) +def test_a_file_that_fails_to_decode_late_raises_pyyamls_earlier_syntax_error(tmp_path): + path = tmp_path / "bad.yaml" + path.write_bytes(b"a: b: c\n" + b"# padding\n" * 2000 + b"x: \xff\n") + + with path.open(encoding="utf-8") as stream, pytest.raises(yaml.YAMLError) as error: + yamlio.safe_load(stream) + + assert "mapping values are not allowed here" in str(error.value) + + +def test_a_file_iterated_before_loading_loads_its_remaining_lines(tmp_path): + path = tmp_path / "x.yaml" + path.write_text("skip: 1\na: 1\n") + + with path.open() as stream: + next(stream) + assert yamlio.safe_load(stream) == {"a": 1} + + +def test_a_closed_file_raises_reads_closed_file_error(tmp_path): + path = tmp_path / "x.yaml" + path.write_text("a: 1\n") + with path.open() as stream: + pass + + with pytest.raises(ValueError, match=r"^I/O operation on closed file\.$"): + yamlio.safe_load(stream) + + +def test_text_streams_that_need_a_read_size_load(): + class Chunks(io.TextIOBase): + def __init__(self) -> None: + self.left = "a: 1\n" + + def read(self, size: int) -> str: + chunk, self.left = self.left[:size], self.left[size:] + return chunk + + assert yamlio.safe_load(Chunks()) == {"a": 1} + + +def test_str_subclasses_load_like_str(): + class Text(str): + __slots__ = () + + assert yamlio.safe_load(Text("a: 1\n")) == {"a": 1} + + +def test_a_metaclass_table_property_is_ignored_as_pyyaml_ignores_it(): + class Meta(type): + @property + def yaml_constructors(cls): + return { + **yaml.SafeLoader.yaml_constructors, + "tag:yaml.org,2002:int": lambda _, node: f"META:{node.value}", + } + + class Loader(yaml.SafeLoader, metaclass=Meta): + pass + + assert yamlio.load("n: 1\n", Loader) == {"n": 1} + + +def test_a_replaced_constructor_that_refuses_comparison_applies(): + class Fussy: + __hash__ = object.__hash__ + + def __init__(self, label): + self.label = label + + def __eq__(self, other): + raise TypeError("ambiguous comparison") + + def __call__(self, _, node): + return f"{self.label}:{node.value}" + + class Loader(yaml.SafeLoader): + pass + + Loader.add_constructor("tag:yaml.org,2002:int", Fussy("OLD")) + first = yamlio.load("n: 1\n", Loader) + Loader.add_constructor("tag:yaml.org,2002:int", Fussy("NEW")) + + assert (first, yamlio.load("n: 1\n", Loader)) == ({"n": "OLD:1"}, {"n": "NEW:1"}) + + +def test_methods_patched_onto_a_loader_after_a_load_apply_to_the_same_text(): + class Loader(yaml.SafeLoader): + pass + + def construct_mapping(self, node, deep=False): + return {**yaml.SafeLoader.construct_mapping(self, node, deep), "patched": True} + + first = yamlio.load("a: 1\n", Loader) + Loader.construct_mapping = construct_mapping + + assert (first, yamlio.load("a: 1\n", Loader)) == ({"a": 1}, {"a": 1, "patched": True}) + + +def test_constructors_added_after_a_load_apply_to_the_same_text(): + class Loader(yaml.SafeLoader): + pass + + first = yamlio.load("n: 12\n", Loader) + Loader.add_constructor("tag:yaml.org,2002:int", lambda _, node: f"INT {node.value}") + + assert (first, yamlio.load("n: 12\n", Loader)) == ({"n": 12}, {"n": "INT 12"}) + + +def test_the_int_digit_limit_applies_to_a_text_loaded_under_a_higher_limit(): + text = "n: " + "1" * 4400 + "\n" + limit = sys.get_int_max_str_digits() + sys.set_int_max_str_digits(5000) + try: + yamlio.safe_load(text) + finally: + sys.set_int_max_str_digits(4300) + try: + with pytest.raises(ValueError, match="Exceeds the limit"): + yamlio.safe_load(text) + finally: + sys.set_int_max_str_digits(limit) + + def test_custom_loader_constructors_apply(): with pytest.raises(yaml.YAMLError, match="found duplicate key 'a'"): yamlio.load("x: 1\ny: {a: 1, a: 2}\n", yamlio.UniqueKeyLoader) diff --git a/inferencex-e2e/infx/yamlio.py b/inferencex-e2e/infx/yamlio.py index 9179cb6904..93ec47480a 100644 --- a/inferencex-e2e/infx/yamlio.py +++ b/inferencex-e2e/infx/yamlio.py @@ -6,8 +6,8 @@ import io import os import re +import sys from collections.abc import Iterator -from contextlib import suppress from typing import IO, Any import yaml @@ -61,25 +61,78 @@ _PYYAML = object() -class _NamedText(io.StringIO): - """Text that reports the original stream's name in PyYAML error marks.""" +# Each entry is (twin, watched classes, their attributes, the loader's tables) as of the build. +_TWINS: dict[type, tuple[type, tuple[type, ...], tuple[dict[str, Any], ...], tuple[Any, ...]]] = {} +# libyaml skips the token after an empty explicit key in a flow collection; PyYAML rejects it. +_AFTER_EMPTY_FLOW_KEY = (yaml.ValueToken, yaml.FlowEntryToken, yaml.FlowSequenceEndToken) - def __init__(self, text: str, name: str) -> None: - super().__init__(text) - self.name = name + +def _class_state(classes: tuple[type, ...]) -> tuple[dict[str, Any], ...]: + return tuple( + {key: value for key, value in dict(vars(cls)).items() if key not in _TABLES} + for cls in classes + ) + + +_PYYAML_CLASSES = tuple( + dict.fromkeys( + cls + for loader in (yaml.SafeLoader, _C_SAFE_LOADER) + if loader is not None + for cls in loader.__mro__ + if cls is not object + ) +) +_PYYAML_STATE = _class_state(_PYYAML_CLASSES) + + +def _table_state(loader: type) -> tuple[dict[Any, Any], ...]: + return tuple( + {key: list(value) if isinstance(value, list) else value for key, value in table.items()} + for table in (dict(getattr(loader, name)) for name in _TABLES) + ) -@functools.cache def _fast_loader(loader: type) -> type | None: - """libyaml twin of a SafeLoader subclass that only customizes its lookup tables.""" - if _C_SAFE_LOADER is None or not issubclass(loader, yaml.SafeLoader): + """libyaml twin of a SafeLoader subclass that only customizes its lookup tables. + + The twin parses with a copy of the tables and is rebuilt, with the parse cache cleared, + once ``add_constructor`` and friends change them or an attribute of any class involved + changes. Patched PyYAML classes, metaclasses and attributes that cannot be compared leave + every load to PyYAML. + """ + if _C_SAFE_LOADER is None or type(loader) is not type: return None + if not issubclass(loader, yaml.SafeLoader): + return None + known = _TWINS.get(loader) mro = loader.__mro__ - if any(set(vars(cls)) - _CLASS_ATTRIBUTES for cls in mro[: mro.index(yaml.SafeLoader)]): + try: + if known is not None: + twin, classes, attributes, tables = known + if all( + saved == vars(cls) for saved, cls in zip(attributes, classes, strict=True) + ) and all( + saved == getattr(loader, name) for saved, name in zip(tables, _TABLES, strict=True) + ): + return twin + if ( + any(set(vars(cls)) - _CLASS_ATTRIBUTES for cls in mro[: mro.index(yaml.SafeLoader)]) + or _class_state(_PYYAML_CLASSES) != _PYYAML_STATE + ): + return None + classes = tuple(dict.fromkeys((*mro[:-1], *_PYYAML_CLASSES))) + attributes = tuple(dict(vars(cls)) for cls in classes) + tables = _table_state(loader) + except Exception: # noqa: BLE001 return None - return type( - f"Fast{loader.__name__}", (_C_SAFE_LOADER,), {t: getattr(loader, t) for t in _TABLES} + if known is not None: + _cached_libyaml_load.cache_clear() + twin = type( + f"Fast{loader.__name__}", (_C_SAFE_LOADER,), dict(zip(_TABLES, tables, strict=True)) ) + _TWINS[loader] = (twin, classes, attributes, tables) + return twin def _may_nest_deeply(text: str) -> bool: @@ -97,6 +150,7 @@ def _needs_pyyaml(text: str, fast: type) -> bool: if "?" not in text and "!" not in text and not _may_nest_deeply(text): return False depth = flow = 0 + previous: type | None = None for token in yaml.scan(text, Loader=fast): kind = type(token) if kind in _OPENING_TOKENS: @@ -105,11 +159,14 @@ def _needs_pyyaml(text: str, fast: type) -> bool: return True elif kind in _CLOSING_TOKENS: depth -= 1 - elif kind is yaml.TagToken or ( - flow and kind is yaml.ScalarToken and token.plain and "?" in token.value + if ( + kind is yaml.TagToken + or (flow and kind is yaml.ScalarToken and token.plain and "?" in token.value) + or (flow and previous is yaml.KeyToken and kind in _AFTER_EMPTY_FLOW_KEY) ): return True flow += _FLOW_TOKENS.get(kind, 0) + previous = kind return False @@ -121,43 +178,62 @@ def _libyaml_load(text: str, fast: type) -> Any: if _needs_pyyaml(text, fast): return _PYYAML return yaml.load(text, Loader=fast) # noqa: S506 - except yaml.YAMLError: + except Exception: # noqa: BLE001 return _PYYAML @functools.lru_cache(maxsize=512) -def _cached_libyaml_load(text: str, fast: type) -> Any: +def _cached_libyaml_load(text: str, fast: type, int_digits: int) -> Any: # noqa: ARG001 """Planner runs bind the same recipes and shared blocks once per matrix row.""" return _libyaml_load(text, fast) -def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: - """Return exactly what ``yaml.load(stream, Loader=loader)`` returns or raises. +def _fast_value(text: str, fast: type) -> Any: + if len(text) > _CACHED_TEXT_CHARACTERS: + return _libyaml_load(text, fast) + value = _cached_libyaml_load(text, fast, sys.get_int_max_str_digits()) + if value is _PYYAML: + return _PYYAML + try: + return fastcopy.deepcopy(value) + except RecursionError: + # Chains of aliases can nest values far deeper than the document's own syntax. + return _PYYAML + - Documents libyaml accepts are parsed by libyaml, which builds the same nodes as PyYAML; - PyYAML reparses everything else, so errors and their marks are PyYAML's own. Texts up to - 128 KiB are parsed once per process and returned as fresh deep copies. - ``INFX_PURE_YAML=1`` turns libyaml off. +def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: + """Return exactly what ``yaml.load(stream, Loader=loader)`` returns or raises, except + within a few frames of the interpreter's recursion limit, where either one can raise + RecursionError while the other returns. + + A ``str`` or a seekable ``io.TextIOWrapper``/``io.StringIO`` whose text libyaml accepts is + parsed by libyaml, which builds the same nodes as PyYAML; everything else, including every + error, goes to PyYAML from the stream's original position. Texts up to 128 KiB are parsed + once per process and returned as fresh deep copies. ``INFX_PURE_YAML=1`` turns libyaml off. + PyYAML internals patched before this module is imported, or mutated in place (such as + ``Scanner.ESCAPE_REPLACEMENTS``), are not detected. Each loader class is kept for the life + of the process. """ fast = _fast_loader(loader) - if fast is None or not isinstance(stream, str | io.TextIOBase): - return yaml.load(stream, Loader=loader) # noqa: S506 - if isinstance(stream, str): - text, fallback = stream, stream - else: - text = stream.read() - fallback = _NamedText(text, getattr(stream, "name", "")) - if len(text) <= _CACHED_TEXT_CHARACTERS: - value = _cached_libyaml_load(text, fast) - if value is not _PYYAML: - # Chains of aliases can nest values far deeper than the document's own syntax. - with suppress(RecursionError): - return fastcopy.deepcopy(value) - else: - value = _libyaml_load(text, fast) + if fast is not None and type(stream) is str: + value = _fast_value(stream, fast) if value is not _PYYAML: return value - return yaml.load(fallback, Loader=loader) # noqa: S506 + elif fast is not None and type(stream) in (io.TextIOWrapper, io.StringIO): + try: + # Iterating a file disables tell(), and a closed one fails differently from read(). + start = stream.tell() if stream.seekable() else None + except (OSError, ValueError): + start = None + if start is not None: + try: + value = _fast_value(stream.read(), fast) + except (OSError, ValueError): + value = _PYYAML + if value is not _PYYAML: + return value + stream.seek(start) + return yaml.load(stream, Loader=loader) # noqa: S506 def safe_load(stream: str | IO[str]) -> Any: From d7f79064aab6a8a2c0e7985b22d2bc1d9927d970 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:22:18 -0500 Subject: [PATCH 133/203] fix(launch): record no Slurm end time while the allocation still runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When sacct cannot answer, the launcher reads the allocation from scontrol, whose EndTime for a running job is the start plus the time limit. A finished container step is checked while its allocation is still held, so job_event.json recorded slurm_ended_at up to a day in the future. Projected start and end times of unfinished jobs are now dropped. 中文:当 sacct 无法应答时,启动器会改用 scontrol 读取分配信息,而运行中作业的 EndTime 是开始时间加上时间限制。容器步骤结束时分配仍被持有,因此 job_event.json 会记录一个可能晚一天的 slurm_ended_at。现在会丢弃未结束作业的预计开始和结束时间。 --- .../infx/launch/backends/slurm/cli.py | 7 +++++-- .../infx/tests/launch/test_slurm_cli.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/launch/backends/slurm/cli.py b/inferencex-e2e/infx/launch/backends/slurm/cli.py index dfc9f30e84..9e32cd7e7d 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/cli.py +++ b/inferencex-e2e/infx/launch/backends/slurm/cli.py @@ -252,9 +252,12 @@ def observe(job: Job) -> JobStatus: fields.get(name, "") for name in ("JobState", "ExitCode", "NodeList", "SubmitTime", "StartTime", "EndTime") ) - nodes = nodes.strip() + nodes, state = nodes.strip(), state.strip() + # scontrol projects StartTime for pending jobs and EndTime from the time limit until the end. + started = "" if state == "PENDING" else started + ended = "" if state in {"PENDING", "CONFIGURING", "RUNNING"} else ended return replace( - job_status(state.strip(), exit_code.strip()), + job_status(state, exit_code.strip()), nodes=None if nodes in {"", "None assigned", "(null)"} else nodes, submitted_at=_time(submitted), started_at=_time(started), diff --git a/inferencex-e2e/infx/tests/launch/test_slurm_cli.py b/inferencex-e2e/infx/tests/launch/test_slurm_cli.py index 24096caf26..4cf2e25b8d 100644 --- a/inferencex-e2e/infx/tests/launch/test_slurm_cli.py +++ b/inferencex-e2e/infx/tests/launch/test_slurm_cli.py @@ -257,6 +257,23 @@ def test_a_container_step_on_a_lost_allocation_is_recorded_as_retriable(fake_bin assert (error.type, error.retriable) == (kind, retriable) +def test_a_running_allocation_records_no_end_from_the_controllers_projection(fake_bin): + fake_bin("sacct", "echo 'sacct: error: Problem talking to the database' >&2; exit 1") + # A running job's EndTime is its start plus the time limit, not when it ended. + fake_bin("scontrol", "echo 'JobId=42 JobState=RUNNING ExitCode=0:0 NodeList=gpu01 " + "SubmitTime=1791547200 StartTime=1791547260 EndTime=1791633660'") + step = subprocess.Popen(["bash", "-c", "exit 0"]) + step.wait() + backend = slurm_backend() + + backend.state(slurm.SlurmJob("42", step=step)) + + record = backend.life.event.finish(0) + assert (record.slurm_started_at, record.slurm_ended_at, record.queue_s) == ( + datetime(2026, 10, 9, 12, 1, tzinfo=UTC), None, 60.0, + ) + + def test_srun_rejects_env_value_that_would_split_export(): with pytest.raises(ValueError, match="UCX_NET_DEVICES"): srun_argv(None, ["true"], container=ContainerSpec(image="i", env={"UCX_NET_DEVICES": "a,b"})) From 58137827602074655b09dcf1cca052a29f750ac7 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:37:12 -0500 Subject: [PATCH 134/203] fix(results): publish null for a non-numeric eval standard error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lm-eval writes the string "N/A" as the stderr of any metric computed from one sample, which an eval-limit=1 smoke run produces. The collector copied that string into the score_se, em_strict_se and em_flexible_se fields of agg_eval rows, where consumers expect a number or null. Collected rows now keep only finite non-negative standard errors and publish null otherwise. 中文:lm-eval 在指标只由一个样本计算时,会把 stderr 写成字符串 "N/A",eval-limit=1 的冒烟运行就会出现这种情况。收集器原先把该字符串原样写入 agg_eval 行的 score_se、em_strict_se 和 em_flexible_se 字段,而下游期望这些字段是数字或 null。现在收集的行只保留有限且非负的标准误,其余情况发布为 null。 --- inferencex-e2e/infx/results/evals.py | 13 +++++++++++ .../results/test_collect_eval_results.py | 23 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/inferencex-e2e/infx/results/evals.py b/inferencex-e2e/infx/results/evals.py index 35ab90a0d2..3c1f56bef7 100644 --- a/inferencex-e2e/infx/results/evals.py +++ b/inferencex-e2e/infx/results/evals.py @@ -130,6 +130,15 @@ def is_valid_effective_count(value: object) -> bool: ) +def _is_standard_error(value: object) -> bool: + return ( + not isinstance(value, bool) + and isinstance(value, (int, float)) + and math.isfinite(value) + and value >= 0 + ) + + def metric_family(name: str) -> str | None: """Classify a filter name or metric key; strict takes precedence.""" if "strict" in name: @@ -336,6 +345,10 @@ def build_rows( "type": "InvalidPrimaryScore", "message": f"invalid primary score: {score!r}", } + for name in _SCORE_NAMES: + # lm-eval reports a one-sample metric's stderr as the string "N/A". + if not _is_standard_error(metrics[f"{name}_se"]): + metrics[f"{name}_se"] = None rows.append(build_row(meta, metrics)) return rows diff --git a/inferencex-e2e/infx/tests/results/test_collect_eval_results.py b/inferencex-e2e/infx/tests/results/test_collect_eval_results.py index 44396ffc09..a6c10d01b0 100644 --- a/inferencex-e2e/infx/tests/results/test_collect_eval_results.py +++ b/inferencex-e2e/infx/tests/results/test_collect_eval_results.py @@ -168,6 +168,29 @@ def test_invalid_primary_does_not_fall_back_to_valid_secondary(tmp_path: Path, s } +def test_one_sample_stderr_sentinel_is_published_as_null(tmp_path: Path) -> None: + # lm-eval writes the string "N/A" as the stderr of a metric with one sample. + (tmp_path / "meta_env.json").write_text("{}") + (tmp_path / "results.json").write_text(json.dumps({ + "lm_eval_version": "test", + "results": {"gsm8k": { + "exact_match,strict-match": 1.0, "exact_match_stderr,strict-match": "N/A", + "exact_match,flexible-extract": 0.0, "exact_match_stderr,flexible-extract": "N/A", + }}, + "configs": {"gsm8k": {"filter_list": [ + {"name": "strict-match"}, {"name": "flexible-extract"}, + ]}}, + "n-samples": {"gsm8k": {"original": 1319, "effective": 1}}, + })) + + [row] = collect_eval_rows(tmp_path) + + assert (row["score"], row["em_strict"], row["em_flexible"]) == (1.0, 1.0, 0.0) + assert (row["score_se"], row["em_strict_se"], row["em_flexible_se"]) == (None, None, None) + assert row["infrastructure_success"] is True + + + @pytest.mark.parametrize("error,expected", [ (None, None), ("", {"type": "IntegrationError", "message": ""}), From 03ecb1f5789e0f806de78eab430446383beefc5f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:54:31 -0500 Subject: [PATCH 135/203] fix(workflows): keep polling merge checks after PyGithub gives up on server errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyGithub retries a 5xx or rate-limited GET itself and then raises requests' RetryError, which check polling did not treat as transient. A GitHub 5xx burst lasting a few seconds therefore aborted the merge helper after it had pushed the sync commit, instead of retrying until the check timeout. RetryError is now retried like other network failures, and only polls that hit an exhausted PyGithub retry change behavior. 中文:PyGithub 会自行重试 5xx 或被限流的 GET 请求,用尽后抛出 requests 的 RetryError,而检查轮询没有把它当作暂时性错误。因此 GitHub 持续几秒的 5xx 就会在合并辅助工具推送同步提交之后将其终止,而不是一直重试到检查超时。现在 RetryError 会像其他网络故障一样被重试,只有遇到 PyGithub 重试耗尽的轮询行为会改变。 --- .../tests/workflows/test_merge_with_reuse.py | 59 +++++++++++++++++++ .../infx/workflows/merge_with_reuse.py | 4 +- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py index 966a017d4c..3ba4626198 100644 --- a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py +++ b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py @@ -2,8 +2,12 @@ from __future__ import annotations +import json import os import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -21,6 +25,7 @@ _latest_statuses, _MergeState, _poll_pr_head, + _make_github, _resolve_token, _retry_delay, find_eligible_run, @@ -388,6 +393,60 @@ def test_later_skipped_run_does_not_replace_verdict(self): assert wait_for_check("a" * 40, "check-changelog", gh_repo, timeout=60) == 0 + def test_server_errors_past_client_retries_are_retried(self, monkeypatch): + """PyGithub's own 5xx retries end in requests' RetryError, which must not abort.""" + sha = "a" * 40 + state = {"healthy": False, "check_polls": 0} + + class GitHub(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_GET(self): + base = f"http://127.0.0.1:{self.server.server_port}/repos/owner/repo" + path = self.path.partition("?")[0] + if path == "/repos/owner/repo": + status, body = 200, {"full_name": "owner/repo", "url": base} + elif path == f"/repos/owner/repo/commits/{sha}": + status, body = 200, {"sha": sha, "url": f"{base}/commits/{sha}"} + elif path == f"/repos/owner/repo/commits/{sha}/check-runs": + state["check_polls"] += 1 + status, body = (200, {"total_count": 1, "check_runs": [{ + "id": 1, "name": "check-changelog", "status": "completed", + "conclusion": "success", "started_at": "2024-01-01T00:00:00Z", + "details_url": None, + }]}) if state["healthy"] else (502, {"message": "Server Error"}) + else: + status, body = 404, {"message": "Not Found"} + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = ThreadingHTTPServer(("127.0.0.1", 0), GitHub) + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}) + thread.start() + monkeypatch.setenv("GITHUB_API_URL", f"http://127.0.0.1:{server.server_port}") + # PyGithub throttles with time.sleep too, so only the helper's own sleep heals GitHub. + monkeypatch.setattr( + "infx.workflows.merge_with_reuse.time", + SimpleNamespace( + monotonic=time.monotonic, sleep=lambda _seconds: state.update(healthy=True) + ), + ) + try: + gh_repo = _make_github("token").get_repo("owner/repo") + result = wait_for_check(sha, "check-changelog", gh_repo, timeout=60) + finally: + server.shutdown() + thread.join() + server.server_close() + + assert result == 0 + assert state["check_polls"] > 1 + diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index 9714d39eb8..162e0078e9 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -40,12 +40,14 @@ from github import Github, GithubException, RateLimitExceededException - # PyGithub surfaces network failures as requests exceptions, which are not builtin ones. + # PyGithub surfaces network failures as requests exceptions, which are not builtin ones, + # and raises RetryError once its own 5xx and rate-limit retries run out. _NETWORK_ERRORS: tuple[type[Exception], ...] = ( ConnectionError, TimeoutError, requests.ConnectionError, requests.Timeout, + requests.exceptions.RetryError, ) except ImportError: gitpython = None # type: ignore[assignment] From 41bae0073e78ca6a8848147356f443b268fcb302 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:00:42 -0500 Subject: [PATCH 136/203] chore(workflows): authenticate the merge helper with PyGithub's Auth.Token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge helper built its client with login_or_token, which the pinned PyGithub deprecates and warns about on every run. It now passes auth=Auth.Token(token), which sends the same Authorization header. 中文:合并辅助工具原先用 login_or_token 创建客户端,锁定版本的 PyGithub 已弃用该参数, 每次运行都会发出警告。现在改为传入 auth=Auth.Token(token),发送的 Authorization 头 保持不变。 --- inferencex-e2e/infx/workflows/merge_with_reuse.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index 162e0078e9..698085c033 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -38,7 +38,7 @@ import git as gitpython import requests - from github import Github, GithubException, RateLimitExceededException + from github import Auth, Github, GithubException, RateLimitExceededException # PyGithub surfaces network failures as requests exceptions, which are not builtin ones, # and raises RetryError once its own 5xx and rate-limit retries run out. @@ -52,6 +52,7 @@ except ImportError: gitpython = None # type: ignore[assignment] Github = None # type: ignore[assignment, misc] + Auth = None # type: ignore[assignment] _NETWORK_ERRORS = (ConnectionError, TimeoutError) class GithubException(Exception): # type: ignore[no-redef] # noqa: N818 @@ -155,7 +156,7 @@ def _resolve_token() -> str: def _make_github(token: str) -> Github: """Build a PyGithub ``Github`` client, respecting ``GITHUB_API_URL``.""" base_url = os.environ.get("GITHUB_API_URL") or "https://api.github.com" - return Github(login_or_token=token, base_url=base_url) + return Github(auth=Auth.Token(token), base_url=base_url) _TRANSIENT_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504}) From 665673f814d78c4fd9b470f0eec78c350ee07aaa Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:56:48 -0500 Subject: [PATCH 137/203] fix(launch): exit 128 plus the signal when a launch step dies by signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host step such as make setup, the recipe binder or srtctl apply that was killed by a signal handed its negative returncode straight to sys.exit. The launch then exited 256 minus the signal, for example 247 after SIGKILL, and job_event.json recorded -9. The launch now reports the shell-style status 128 plus the signal that its docstring promises. 中文: 当 make setup、recipe binder 或 srtctl apply 等主机步骤被信号杀死时,其负的 returncode 会被直接传给 sys.exit。启动器因此以 256 减去信号值退出(例如 SIGKILL 后为 247),job_event.json 中也记录为 -9。现在启动器按其文档承诺,报告 shell 风格的退出状态,即 128 加信号值。 --- inferencex-e2e/infx/launch/__main__.py | 3 ++- inferencex-e2e/infx/tests/launch/test_srt_driver.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/launch/__main__.py b/inferencex-e2e/infx/launch/__main__.py index 19d6a63557..f6398710fc 100644 --- a/inferencex-e2e/infx/launch/__main__.py +++ b/inferencex-e2e/infx/launch/__main__.py @@ -14,6 +14,7 @@ import yaml +from infx.bench.proc import status from infx.clusters import Cluster, resolve_cluster from infx.launch import drivers from infx.launch.backends import BACKENDS, backend_class @@ -28,7 +29,7 @@ def launch(cluster: Cluster, request: LaunchRequest, event: JobEventBuilder | No """Run ``request`` on ``cluster``; a launch error is one ``ERROR:`` line, not a traceback.""" with Lifecycle(event) as life: try: - rc = drivers.run(cluster, request, life) + rc = status(drivers.run(cluster, request, life)) except (LaunchError, BackendError, RequestError) as error: life.event.error(error) rc = 1 diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index 0c81a7e905..acbfde1779 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -722,6 +722,19 @@ def test_a_setup_failure_without_a_bad_archive_is_not_retried(harness): assert (error["stage"], error["type"], error["evidence"]) == ("prepare", "SetupFailed", "srt-setup.log") +def test_a_setup_step_killed_by_a_signal_exits_128_plus_the_signal(harness): + fakes = Path(harness.env["PATH"].split(os.pathsep)[0]) + (fakes / "make").write_text("#!/bin/bash\nkill -KILL $$\n") + env = lane_env( + harness, "h200-dgxc", MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", + MODEL="deepseek-ai/DeepSeek-R1-0528", + ) # fmt: skip + result = launch(env, harness.config, harness.workspace) + assert result.returncode == 137, result.stderr[-4000:] + error = job_event(harness.workspace)["error"] + assert (error["type"], error["exit_code"]) == ("SetupFailed", 137) + + def test_a_failed_uv_download_fails_the_uv_install(harness): fakes = Path(harness.env["PATH"].split(os.pathsep)[0]) (fakes / "uv").unlink() From aa4f3fce8810ea9b427de92f0c4bfa1d7b8e3122 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:57:26 -0500 Subject: [PATCH 138/203] fix(srt-slurm): exit 128 plus the signal when srtctl apply dies by signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The golden acceptance wrapper returned srtctl's negative returncode from main, so a srtctl killed by SIGKILL made the wrapper exit 247 instead of 137. The launcher records that code as the submission failure, so the signal was misreported. The wrapper now returns the shell-style status 128 plus the signal. 中文: golden acceptance 包装器从 main 直接返回 srtctl 的负 returncode,因此 srtctl 被 SIGKILL 杀死时包装器以 247 而不是 137 退出。启动器会把该代码记录为提交失败,导致信号被错误报告。现在包装器返回 shell 风格的退出状态,即 128 加信号值。 --- .../infx/srt_slurm/synthetic_acceptance.py | 3 ++- .../tests/srt_slurm/test_synthetic_acceptance.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 8bbb11b03b..51607eca11 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -16,6 +16,7 @@ import yaml from infx import yamlio +from infx.bench.proc import status from infx.golden_al_distribution import GOLDEN_DIR, golden_length from infx.srt_slurm.variants import ZIP_VARIANT, without_nulls @@ -372,7 +373,7 @@ def main(argv: list[str] | None = None) -> int: for plan in plans: result = subprocess.run(plan.command, check=False) if result.returncode: - return result.returncode + return status(result.returncode) return 0 diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index e7991fa53e..e687fc922c 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -646,7 +646,12 @@ def test_caller_json_is_merged_before_golden_selection(tmp_path: Path, golden_di } -def test_cli_forwards_options_and_submission_failure(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("ending", "returncode"), [("sys.exit(7)", 7), ("os.kill(os.getpid(), signal.SIGKILL)", 137)] +) +def test_cli_forwards_options_and_submission_failure( + tmp_path: Path, ending: str, returncode: int +) -> None: recipe = tmp_path / "recipe.yaml" recipe.write_text( yaml.safe_dump( @@ -655,7 +660,8 @@ def test_cli_forwards_options_and_submission_failure(tmp_path: Path) -> None: ) binary = tmp_path / "srtctl" binary.write_text( - f"#!{sys.executable}\nimport json,sys\nprint(json.dumps(sys.argv[1:]))\nsys.exit(7)\n" + f"#!{sys.executable}\nimport json,os,signal,sys\n" + f"print(json.dumps(sys.argv[1:]), flush=True)\n{ending}\n" ) binary.chmod(0o755) record = tmp_path / "acceptance.json" @@ -676,7 +682,7 @@ def test_cli_forwards_options_and_submission_failure(tmp_path: Path) -> None: capture_output=True, text=True, ) - assert result.returncode == 7, result.stderr + assert result.returncode == returncode, result.stderr assert json.loads(record.read_text()) == {"golden_acceptance_length": None} argv = json.loads(result.stdout) assert argv[:5] == ["apply", "-f", str(recipe), "--tags", "a b"] From 44a5977935198f8174ce1ea42b1a69e1ff796cdc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:59:16 -0500 Subject: [PATCH 139/203] fix(launch): exit with the wrapped launch's code on the srt-batch path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A point that re-runs inside an sbatch allocation exited 1 whenever the batch job failed, even though Slurm accounting held the wrapped launch's own exit code. A srtctl submission failure of 7 or an interrupted launch's 143 therefore surfaced as 1 in the exit status and in job_event.json. The batch path now returns the job's exit code, as the script driver already does. 中文: 在 sbatch 分配中重新运行的点只要批处理作业失败就以 1 退出,尽管 Slurm 记账中保存了被包装启动器自身的退出码。因此 srtctl 提交失败的 7 或被中断启动器的 143 在退出状态和 job_event.json 中都显示为 1。现在 batch 路径与 script 驱动一致,返回作业的退出码。 --- inferencex-e2e/infx/launch/drivers/srt/__init__.py | 4 ++-- inferencex-e2e/infx/tests/launch/fake_slurm.py | 3 ++- inferencex-e2e/infx/tests/launch/test_srt_driver.py | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index 65227bb907..5b4ae0e292 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -151,9 +151,9 @@ def run_batch(launch: Launch) -> int: backend.stream_logs(job) except BackendError: return 1 - succeeded = backend.state(job).succeeded + status = backend.state(job) event.adopt_reentry() - return 0 if succeeded else 1 + return 0 if status.succeeded else status.exit_code or 1 def run_multinode(launch: Launch) -> int: diff --git a/inferencex-e2e/infx/tests/launch/fake_slurm.py b/inferencex-e2e/infx/tests/launch/fake_slurm.py index 15a31174c4..ad49e8f722 100644 --- a/inferencex-e2e/infx/tests/launch/fake_slurm.py +++ b/inferencex-e2e/infx/tests/launch/fake_slurm.py @@ -52,7 +52,8 @@ """, "sacct": r""" state="${FAKE_STATE:-COMPLETED|0:0}" -if [[ " $* " == *" 4242 "* && "$(cat "$FAKE_LOG_DIR/batch-rc" 2>/dev/null)" != 0 ]]; then state="FAILED|1:0"; fi +rc="$(cat "$FAKE_LOG_DIR/batch-rc" 2>/dev/null)" +if [[ " $* " == *" 4242 "* && -n "$rc" && "$rc" != 0 ]]; then state="FAILED|$rc:0"; fi echo "$state" """, "scontrol": "exit 1", diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index acbfde1779..01a1d4216d 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -610,6 +610,15 @@ def test_a_failed_batch_wrapped_point_reports_the_wrapped_failure_once(harness): assert (result.stdout + result.stderr).count("::error") == 1 +def test_a_batch_wrapped_point_exits_with_the_wrapped_launchs_code(harness): + env = batch_env(harness, FAKE_SRTCTL_RC="7") + result = launch(env, harness.config, harness.workspace) + assert (harness.logs / "batch-rc").read_text() == "7" + assert result.returncode == 7, result.stderr[-4000:] + error = job_event(harness.workspace)["error"] + assert (error["type"], error["exit_code"]) == ("SubmitFailed", 7) + + def test_tilert_uses_upstream_submission_and_prepared_weights(harness): env = lane_env( harness, "mi355x-amds", MODEL_PREFIX="glm5.3", PRECISION="fp8", FRAMEWORK="tilert", From 294be8c3700ae834c67a54f38567eb8f03e7c6bc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:02:28 -0500 Subject: [PATCH 140/203] fix(matrix): exit 128 plus the signal when a revision's tool dies by signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python -m infx.matrix.revision passed the planner's or generator's returncode straight to SystemExit, so a tool killed by SIGKILL made the command exit 247 instead of 137. It now maps a death by signal N to 128 + N, as the launcher does. 中文:python -m infx.matrix.revision 把规划器或生成器的返回码直接传给 SystemExit, 因此被 SIGKILL 终止的工具会让命令以 247 而不是 137 退出。现在与启动器一样,把因信号 N 终止映射为 128 + N。 --- inferencex-e2e/infx/matrix/revision.py | 5 ++++- inferencex-e2e/infx/tests/matrix/test_revision.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/matrix/revision.py b/inferencex-e2e/infx/matrix/revision.py index 0a5a943759..3a0b57a71c 100644 --- a/inferencex-e2e/infx/matrix/revision.py +++ b/inferencex-e2e/infx/matrix/revision.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Self +from infx.bench.proc import status from infx.config import MASTER_CONFIGS, git_repository_root, project_root @@ -349,7 +350,9 @@ def main(argv: Sequence[str] | None = None) -> None: command, env = revision.invocation(TOOLS[options.tool], options.args) except ValueError as error: parser.error(str(error)) - raise SystemExit(subprocess.run(command, cwd=revision.root, env=env, check=False).returncode) + raise SystemExit( + status(subprocess.run(command, cwd=revision.root, env=env, check=False).returncode) + ) if __name__ == "__main__": diff --git a/inferencex-e2e/infx/tests/matrix/test_revision.py b/inferencex-e2e/infx/tests/matrix/test_revision.py index afe56f8d2f..c17d1ff8a4 100644 --- a/inferencex-e2e/infx/tests/matrix/test_revision.py +++ b/inferencex-e2e/infx/tests/matrix/test_revision.py @@ -271,6 +271,8 @@ def test_snapshot_runs_pre_root_move_revisions_with_their_own_config_directory(t print(json.dumps({{"planner": NAME, "args": sys.argv[1:], "cwd": os.getcwd(), "root": os.environ["INFERENCEX_REPOSITORY_ROOT"], "env": dict(os.environ)}})) print("planner diagnostics", file=sys.stderr) +if "--kill" in sys.argv: + os.kill(os.getpid(), 9) sys.exit(int(sys.argv[sys.argv.index("--exit") + 1]) if "--exit" in sys.argv else 0) """ @@ -318,6 +320,14 @@ def test_cli_runs_the_checkouts_own_planner_and_forwards_its_output_and_status(t assert result.stderr == "planner diagnostics\n" +def test_cli_exits_128_plus_the_signal_that_killed_the_tool(tmp_path): + checkout = planner_checkout(tmp_path / "checkout", "module") + + result = run_cli(tmp_path, "plan", str(checkout), "--kill") + + assert result.returncode == 137, result.stderr + + def test_cli_runs_the_tool_without_the_callers_credentials(tmp_path): checkout = planner_checkout(tmp_path / "checkout", "module") caller = { From 17555dbb7d9cfd4afcf1d942e6b0b4e4717f1daf Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:58:31 -0500 Subject: [PATCH 141/203] fix(bench): stream a dying server's last log lines before failing the wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The readiness wait read the server log, then probed the health URL, then checked the server process. vLLM binds its port before the engine starts, so each probe can block for its full timeout while startup runs. A server that crashed in that window wrote its traceback after the read, and the wait failed without ever showing it. The wait now reads the log once more after it sees the process gone. 中文:就绪等待会先读取服务器日志,再探测健康检查 URL,最后检查服务器进程。vLLM 在引擎启动前就绑定端口,因此启动期间每次探测都可能阻塞到超时。服务器若在这段时间内崩溃,其回溯信息写在读取之后,等待失败时从未显示出来。现在等待在发现进程已退出后会再读取一次日志。 --- inferencex-e2e/infx/bench/server.py | 30 ++++++++++++------- .../infx/tests/bench/test_server.py | 17 +++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/inferencex-e2e/infx/bench/server.py b/inferencex-e2e/infx/bench/server.py index 6c29a2ffd1..58d739d002 100644 --- a/inferencex-e2e/infx/bench/server.py +++ b/inferencex-e2e/infx/bench/server.py @@ -66,22 +66,30 @@ def wait_ready( offset = 0 inode: int | None = None decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + + def stream() -> None: + nonlocal offset, inode + if log is None or not log.exists(): + return + with log.open("rb") as source: + stat = os.fstat(source.fileno()) + if stat.st_ino != inode or stat.st_size < offset: + offset, inode = 0, stat.st_ino + decoder.reset() + source.seek(offset) + chunk = source.read() + offset += len(chunk) + sys.stdout.write(decoder.decode(chunk)) + sys.stdout.flush() + while True: - if log is not None and log.exists(): - with log.open("rb") as stream: - stat = os.fstat(stream.fileno()) - if stat.st_ino != inode or stat.st_size < offset: - offset, inode = 0, stat.st_ino - decoder.reset() - stream.seek(offset) - chunk = stream.read() - offset += len(chunk) - sys.stdout.write(decoder.decode(chunk)) - sys.stdout.flush() + stream() status = http_status(url) if status is not None and status < 400: return if pid is not None and not process_alive(pid): + # Its last words may have landed during the probe. + stream() raise NotReadyError(f"process {pid} died before {url} became ready") time.sleep(poll_s) diff --git a/inferencex-e2e/infx/tests/bench/test_server.py b/inferencex-e2e/infx/tests/bench/test_server.py index 3644297a0c..72cb8261e2 100644 --- a/inferencex-e2e/infx/tests/bench/test_server.py +++ b/inferencex-e2e/infx/tests/bench/test_server.py @@ -98,6 +98,23 @@ def test_wait_fails_once_the_server_process_is_gone(): assert result.stderr == f"ERROR: process {dead.pid} died before {url} became ready\n" +def test_wait_streams_what_a_dying_server_wrote_before_failing(http_server, tmp_path, capsys): + log = tmp_path / "server.log" + log.write_text("loading\n") + dead = subprocess.Popen(["true"]) + dead.wait() + + def respond(_method: str, _path: str) -> tuple[int, object]: + with log.open("a") as out: + out.write("CUDA out of memory\n") + return 503, {} + + with pytest.raises(server.NotReadyError): + server.wait_ready(f"{http_server(respond)}/health", pid=dead.pid, log=log, poll_s=0.01) + + assert capsys.readouterr().out == "loading\nCUDA out of memory\n" + + @pytest.mark.parametrize( ("models", "health", "stabilization_s", "waiting"), [(["m"], 503, 0, False), (["other"], 200, 1, True)], From d8d5b73005459627078cf70ec8b12d2e5de12acd Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:53:19 -0500 Subject: [PATCH 142/203] docs(evals): describe single-node eval groups and BFCL full-suite task names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-node fixed-sequence evals group rows by model, runner, framework, precision, spec-decoding and DP attention, and pick the highest and median concurrency of 16 or above. TP does not split those groups. BFCL full suites name their per-category tasks after the suite, and the validator reads the declared acc metric without an override. 中文:单节点固定序列评测按模型、运行器、框架、精度、投机解码和 DP attention 分组,并在 16 及以上的并发中选择最高值和中位数,TP 不会拆分分组。BFCL 完整套件的分类任务以套件名作前缀,评分校验直接读取声明的 acc 指标,无需覆盖参数。 --- inferencex-e2e/infx/evals/EVALS.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/infx/evals/EVALS.md b/inferencex-e2e/infx/evals/EVALS.md index 88d0c63020..33d643a60d 100644 --- a/inferencex-e2e/infx/evals/EVALS.md +++ b/inferencex-e2e/infx/evals/EVALS.md @@ -7,9 +7,10 @@ from throughput. Selection lives in `mark_eval_entries()` in ## Selection -- **Fixed-sequence, single-node:** 8k1k only, at the highest and median - concurrency for every model, runner, framework, precision, TP, and decoding - configuration. +- **Fixed-sequence, single-node:** 8k1k only. Each group of model, runner, + framework, precision, spec-decoding, and dp-attn runs evals at the highest + and the median of its concurrencies of 16 or above. Every row at those concurrencies + gets an eval, and TP/EP do not split groups. - **Fixed-sequence, multi-node:** 8k1k only, with one job per parallelism topology at its highest eligible concurrency. Rows differing only by concurrency share a topology. @@ -461,8 +462,8 @@ mismatched parser/chat template. staging, the workflow upload, the collector, and the score validator. It projects the four-case aggregate as task `bfcl_smoke` and the four one-case diagnostic tasks shown above. Every row uses lm-eval-compatible `acc,none` (plus -`acc_stderr,none`); BFCL workflows therefore validate with metric prefix -`acc,` rather than the default exact-match prefix. +`acc_stderr,none`), and every task config declares the `acc` metric, so the +score validator selects `acc,` without a metric override. The `bfcl_smoke` and four `bfcl_` thresholds are `0.0`, so all five scores remain diagnostic. Dependency, endpoint, timeout, malformed-output, @@ -502,10 +503,11 @@ adapter rejects missing, unexpected, or duplicate result IDs and score headers whose counts or accuracy do not reconcile with the selected corpus. The compatibility result publishes `bfcl_vllm_minimax_m3` or -`bfcl_vllm_kimi` as the aggregate task and preserves per-category -`bfcl_` tasks. Kimi also publishes a combined `bfcl_multi_turn` -task. Full-suite thresholds are `0.0`; they are diagnostic until repeated -hardware runs establish model, precision, and backend baselines. A completed +`bfcl_vllm_kimi` as the aggregate task, plus one `_` task per +category, such as `bfcl_vllm_kimi_simple_python`. Kimi also publishes a +combined `bfcl_vllm_kimi_multi_turn` task. Full-suite thresholds are `0.0`, so +they stay diagnostic until repeated hardware runs establish model, precision, +and backend baselines. A completed zero-score run therefore passes threshold validation, while dependency, transport, timeout, malformed-output, and integration failures still fail. From eed9ed160091440732860717c1367a6765a48e79 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:53:48 -0500 Subject: [PATCH 143/203] docs(matrix): say what --all-evals selects and that it emits eval jobs only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generator help claimed --all-evals covered every fixed-sequence config and added eval-only jobs next to throughput. It expands evals to eligible 8k1k fixed-sequence and agentic rows and emits only the eval-only jobs, as --evals-only --all-evals does. 中文:生成器帮助曾声称 --all-evals 覆盖所有固定序列配置,并在吞吐任务之外附加仅评测任务。实际上它把评测扩展到符合条件的 8k1k 固定序列和 agentic 行,并且只输出仅评测任务,与 --evals-only --all-evals 相同。 --- inferencex-e2e/infx/matrix/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index d8edf6c4d8..4deea17513 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1567,8 +1567,8 @@ def main() -> list[dict]: "--all-evals", action="store_true", help=( - "Expand eval selection to every generated fixed-sequence config. " - "Can be combined with --evals-only; used alone, it also emits eval-only jobs." + "Expand eval selection to every eligible 8k1k fixed-sequence and agentic config " + "and emit only those eval-only jobs, with or without --evals-only." ), ) parent_parser.add_argument( From 5b7466455dce644e018511f1b903ac3f095d067c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:54:56 -0500 Subject: [PATCH 144/203] docs(golden-al): describe the SGLang collector, dflash lookup and srt driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The golden AL guide named an apply_srt_recipe helper that no longer exists and described only vLLM collectors, while speedbench-al.yml also runs an SGLang collector chosen by its framework input. The lookup method list also left out dflash, which the Muse Spark curve uses. The Chinese README is updated to match. 中文:黄金 AL 指南提到了已不存在的 apply_srt_recipe 辅助函数,并且只描述了 vLLM 收集脚本,而 speedbench-al.yml 还会根据 framework 输入运行 SGLang 收集脚本。lookup 的方法列表也遗漏了 Muse Spark 曲线使用的 dflash。中文 README 已同步更新。 --- .../infx/golden_al_distribution/README.md | 16 ++++++++-------- .../infx/golden_al_distribution/README_zh.md | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/inferencex-e2e/infx/golden_al_distribution/README.md b/inferencex-e2e/infx/golden_al_distribution/README.md index 59c4f752b5..ed38aede28 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README.md +++ b/inferencex-e2e/infx/golden_al_distribution/README.md @@ -24,8 +24,8 @@ AL is workload-dependent: a draft model's predictions are easier to accept in so Under the AgentX Guidelines, each model, thinking mode, and draft length has one committed golden AL. Once synthetic acceptance is enabled for a benchmark scenario, a submission may choose any supported draft length, but it may not substitute a different acceptance target. Different models keep their own SPEED-Bench-derived curves. All submissions evaluating the same model and mode use the same curve. -For Qwen3.5 srt-slurm launches, `apply_srt_recipe` uses the -`infx/srt_slurm/synthetic_acceptance.py` connector to select the committed curve +For Qwen3.5 srt-slurm launches, the srt driver runs the +`infx/srt_slurm/synthetic_acceptance.py` connector, which selects the committed curve from the generation role's `speculative-num-steps` and `THINKING_MODE` after caller overrides. The verification token is not an additional MTP step. For SGLang MTP synthetic throughput, the connector rejects a generation role whose @@ -85,11 +85,11 @@ This policy follows the same broad principle as MLPerf Inference: prescribe the The push-button [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.yml) workflow, introduced in [InferenceX#1650](https://github.com/SemiAnalysisAI/InferenceX/pull/1650) and extended to additional MTP and EAGLE3 models in [InferenceX#1706](https://github.com/SemiAnalysisAI/InferenceX/pull/1706), performs the following process. It superseded the early manually assembled reference in [InferenceX#1592](https://github.com/SemiAnalysisAI/InferenceX/pull/1592), making the exact commands, logs, outputs, and generated YAML auditable from one run. -1. A maintainer dispatches the workflow with a model, model prefix, vLLM image, draft lengths (normally 1–8), thinking modes, `category=coding`, and `output-len=4096`. -2. The workflow launches the model on a B300 runner and selects the matching collector under [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench). -3. For every `(thinking mode, draft length)` cell, the collector starts a clean vLLM server with real MTP or EAGLE3 decoding and the model's production sampling/chat-template settings. -4. The collector snapshots vLLM's cumulative accepted-token and verification-draft counters, runs every prompt in the SPEED-Bench Qualitative `coding` category through `vllm bench serve`, and snapshots the counters again. -5. It computes the mean acceptance length as: +1. A maintainer dispatches the workflow with a model, model prefix, serving framework (`vllm` or `sglang`), image, draft lengths (normally 1–8), thinking modes, `category=coding`, and `output-len=4096`. +2. The workflow launches the model on a B300 runner and selects the matching `_fp4_b300_.sh` collector under [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench). +3. For every `(thinking mode, draft length)` cell, the collector starts a clean vLLM or SGLang server with real speculative decoding and the model's production sampling/chat-template settings. +4. A vLLM collector snapshots vLLM's cumulative accepted-token and verification-draft counters, runs every prompt in the SPEED-Bench Qualitative `coding` category through `vllm bench serve`, and snapshots the counters again. The SGLang collector runs the same prompts through `sglang.bench_serving` and records the `accept_length` it reports. +5. A vLLM collector computes the mean acceptance length as: ```text AL = 1 + (delta accepted draft tokens / delta verification drafts) @@ -166,7 +166,7 @@ python -m infx.golden_al_distribution lookup qwen3.5 mtp 3 # 3.39 (thin python -m infx.golden_al_distribution lookup kimik3 dspark 7 --draft-sample-method probabilistic --json ``` -`lookup` takes the InferenceX model prefix, the engine's method name (`mtp`, `eagle`, `nextn`, `eagle3`, `dspark`) and `num_speculative_tokens`. `--draft-model` selects the MiniMax GQA curve, and `--draft-sample-method` chooses between the two Kimi DSpark curves. A missing curve or cell exits non-zero, exactly as a launch would fail. +`lookup` takes the InferenceX model prefix, the engine's method name (`mtp`, `eagle`, `nextn`, `eagle3`, `dspark`, `dflash`) and `num_speculative_tokens`. `--draft-model` selects the MiniMax GQA curve, and `--draft-sample-method` chooses between the two Kimi DSpark curves. A missing curve or cell exits non-zero, exactly as a launch would fail. ## Primary references diff --git a/inferencex-e2e/infx/golden_al_distribution/README_zh.md b/inferencex-e2e/infx/golden_al_distribution/README_zh.md index 8e01779b71..cf11c2d808 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README_zh.md +++ b/inferencex-e2e/infx/golden_al_distribution/README_zh.md @@ -24,7 +24,7 @@ AL 取决于工作负载:草稿模型的预测在某些领域比其他领域 根据 AgentX 指南,每个模型、思考模式和草稿长度都有一个已提交的黄金 AL。当某个基准场景启用合成接受后,提交可以选择任意受支持的草稿长度,但不能替换为其他接受目标。不同模型保留各自基于 SPEED-Bench 测得的曲线。所有评估同一模型和模式的提交都使用同一条曲线。 -对于 Qwen3.5 的 srt-slurm 启动,`apply_srt_recipe` 通过 `infx/srt_slurm/synthetic_acceptance.py` 连接器,在应用调用方覆盖设置后,根据生成角色的 `speculative-num-steps` 和 `THINKING_MODE` 选择已提交的曲线。验证 token 不计为额外的 MTP 步数。对于采用合成接受的 SGLang MTP 吞吐测试,如果生成角色缺少 `speculative-num-draft-tokens`,或者其值不是等于 `speculative-num-steps + 1` 的整数,连接器会拒绝启动。无需手动设置接受长度。仅评测运行会移除合成设置并使用真实验证。 +对于 Qwen3.5 的 srt-slurm 启动,srt 驱动会运行 `infx/srt_slurm/synthetic_acceptance.py` 连接器,在应用调用方覆盖设置后,根据生成角色的 `speculative-num-steps` 和 `THINKING_MODE` 选择已提交的曲线。验证 token 不计为额外的 MTP 步数。对于采用合成接受的 SGLang MTP 吞吐测试,如果生成角色缺少 `speculative-num-draft-tokens`,或者其值不是等于 `speculative-num-steps + 1` 的整数,连接器会拒绝启动。无需手动设置接受长度。仅评测运行会移除合成设置并使用真实验证。 vLLM 通过合成拒绝采样支持这一策略。例如,EAGLE3 运行可以通过 `synthetic_acceptance_length` 注入所选 YAML 值: @@ -78,11 +78,11 @@ python -m atom.entrypoints.openai_server \ 一键触发的 [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.yml) 工作流最初由 [InferenceX#1650](https://github.com/SemiAnalysisAI/InferenceX/pull/1650) 引入,随后在 [InferenceX#1706](https://github.com/SemiAnalysisAI/InferenceX/pull/1706) 中扩展到更多 MTP 和 EAGLE3 模型。它取代了 [InferenceX#1592](https://github.com/SemiAnalysisAI/InferenceX/pull/1592) 中早期手工整理的参考值,使精确命令、日志、输出和生成的 YAML 都可以从同一次运行中审计。其流程如下: -1. 维护者触发工作流,指定模型、模型前缀、vLLM 镜像、草稿长度(通常为 1–8)、思考模式、`category=coding` 和 `output-len=4096`。 -2. 工作流在 B300 runner 上启动模型,并选择 [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench) 下对应的收集脚本。 -3. 对每个“思考模式 × 草稿长度”组合,收集脚本使用真实 MTP 或 EAGLE3 解码以及该模型的生产采样和聊天模板设置,启动一个干净的 vLLM 服务。 -4. 收集脚本读取 vLLM 累计的已接受 token 和验证草稿计数器,通过 `vllm bench serve` 运行 SPEED-Bench Qualitative `coding` 类别中的全部提示词,然后再次读取计数器。 -5. 按以下公式计算平均接受长度: +1. 维护者触发工作流,指定模型、模型前缀、服务框架(`vllm` 或 `sglang`)、镜像、草稿长度(通常为 1–8)、思考模式、`category=coding` 和 `output-len=4096`。 +2. 工作流在 B300 runner 上启动模型,并选择 [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench) 下对应的 `_fp4_b300_.sh` 收集脚本。 +3. 对每个“思考模式 × 草稿长度”组合,收集脚本使用真实投机解码以及该模型的生产采样和聊天模板设置,启动一个干净的 vLLM 或 SGLang 服务。 +4. vLLM 收集脚本读取 vLLM 累计的已接受 token 和验证草稿计数器,通过 `vllm bench serve` 运行 SPEED-Bench Qualitative `coding` 类别中的全部提示词,然后再次读取计数器。SGLang 收集脚本通过 `sglang.bench_serving` 运行相同的提示词,并记录其报告的 `accept_length`。 +5. vLLM 收集脚本按以下公式计算平均接受长度: ```text AL = 1 + (delta accepted draft tokens / delta verification drafts) @@ -159,7 +159,7 @@ python -m infx.golden_al_distribution lookup qwen3.5 mtp 3 # 3.39(默 python -m infx.golden_al_distribution lookup kimik3 dspark 7 --draft-sample-method probabilistic --json ``` -`lookup` 接受 InferenceX 模型前缀、引擎的方法名(`mtp`、`eagle`、`nextn`、`eagle3`、`dspark`)以及 `num_speculative_tokens`。`--draft-model` 用于选择 MiniMax GQA 曲线,`--draft-sample-method` 用于在两条 Kimi DSpark 曲线之间选择。缺少曲线或数值时以非零状态退出,与启动时的失败方式一致。 +`lookup` 接受 InferenceX 模型前缀、引擎的方法名(`mtp`、`eagle`、`nextn`、`eagle3`、`dspark`、`dflash`)以及 `num_speculative_tokens`。`--draft-model` 用于选择 MiniMax GQA 曲线,`--draft-sample-method` 用于在两条 Kimi DSpark 曲线之间选择。缺少曲线或数值时以非零状态退出,与启动时的失败方式一致。 ## 主要参考资料 From c1db578d4928ddfac025bebabc798aa67a369ddc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:55:10 -0500 Subject: [PATCH 145/203] docs(golden-al): list dflash among the lookup help's methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lookup help listed the speculative methods without dflash, although the Muse Spark curve resolves through it and the srt connector passes it for SGLang DFlash roles. The help now names every method that selects a committed curve. 中文:lookup 帮助列出的投机方法中缺少 dflash,而 Muse Spark 曲线正是通过它解析,srt 连接器也会为 SGLang DFlash 角色传入该方法。帮助现在列出所有能选中已提交曲线的方法。 --- inferencex-e2e/infx/golden_al_distribution/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/golden_al_distribution/__main__.py b/inferencex-e2e/infx/golden_al_distribution/__main__.py index 670013fff0..363bfb5bcf 100644 --- a/inferencex-e2e/infx/golden_al_distribution/__main__.py +++ b/inferencex-e2e/infx/golden_al_distribution/__main__.py @@ -128,7 +128,9 @@ def main(argv: list[str] | None = None) -> int: "lookup", help="the AL synthetic acceptance applies for a model prefix and method" ) lookup_parser.add_argument("model", help="InferenceX model prefix, e.g. qwen3.5, dsv4, kimik3") - lookup_parser.add_argument("method", help="spec method: mtp, eagle, nextn, eagle3, dspark") + lookup_parser.add_argument( + "method", help="spec method: mtp, eagle, nextn, eagle3, dspark, dflash" + ) lookup_parser.add_argument("tokens", type=int, help="num_speculative_tokens") lookup_parser.add_argument("--thinking", choices=THINKING_MODES, default="thinking_on") lookup_parser.add_argument("--draft-model", help="draft checkpoint (selects MiniMax GQA)") From 696c8b0b3dd8299fd935899a589ea75184f541ef Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:55:47 -0500 Subject: [PATCH 146/203] docs(datasets): stop saying the weka build's --limit forces top sampling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build help said --limit implies --sampling top, but the build forwards whichever --sampling mode is given, and random sampling with --limit is supported. The help now says --limit keeps the first sessions in the chosen sampling order. 中文:构建脚本帮助曾称 --limit 会强制使用 --sampling top,但构建会原样转发所给的 --sampling 模式,并且支持随机采样配合 --limit。帮助现在说明 --limit 按所选采样顺序保留前若干个会话。 --- inferencex-e2e/infx/datasets/build_weka_hf_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index 1946cbe085..454cf8c502 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -102,7 +102,7 @@ def parse_args() -> argparse.Namespace: "--limit", type=int, default=None, - help="Cap session count (for smoke tests). Implies --sampling top.", + help="Cap session count (smoke tests), taking the first sessions in --sampling order.", ) p.add_argument("--sampling", choices=("top", "recent", "random"), default="top") p.add_argument( From 71b1fbf97691c81629dc5f5ec6820ba63cbe277e Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:56:02 -0500 Subject: [PATCH 147/203] docs(datasets): say the sampler hashes random seeds locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sampler help described --seed as a server-side md5 salt, but random ordering is now computed in Python after the phase-2 filters. The help now names the md5 of session id and seed without claiming Postgres computes it. 中文:采样器帮助把 --seed 描述为服务端 md5 的盐值,但随机排序现在在第二阶段过滤之后由 Python 计算。帮助现在说明按会话 id 与种子的 md5 排序,不再声称由 Postgres 计算。 --- inferencex-e2e/infx/datasets/sample_proxy_traces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/datasets/sample_proxy_traces.py b/inferencex-e2e/infx/datasets/sample_proxy_traces.py index f2589e3d6d..9e3e22d988 100644 --- a/inferencex-e2e/infx/datasets/sample_proxy_traces.py +++ b/inferencex-e2e/infx/datasets/sample_proxy_traces.py @@ -263,7 +263,7 @@ def parse_args() -> argparse.Namespace: "--seed", type=int, default=None, - help="Salt for --sampling random (server-side md5). Required when --sampling random.", + help="Salt for --sampling random (md5 of session_id + seed). Required when --sampling random.", ) p.add_argument( "--db-url", From 02115220ed843721b2b6d503805b307b8a15c0ef Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:56:21 -0500 Subject: [PATCH 148/203] docs(datasets): say the weka converter writes traces into --out-dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The converter docstring said traces land in /../, but it writes each trace directly under --out-dir as given. The docstring now matches the path the converter writes. 中文:转换器文档字符串称轨迹写入 /../,但它实际直接写入所给的 --out-dir。文档字符串现在与转换器写入的路径一致。 --- inferencex-e2e/infx/datasets/proxy_to_weka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/datasets/proxy_to_weka.py b/inferencex-e2e/infx/datasets/proxy_to_weka.py index 1e5436d78e..d5fff5a814 100644 --- a/inferencex-e2e/infx/datasets/proxy_to_weka.py +++ b/inferencex-e2e/infx/datasets/proxy_to_weka.py @@ -1,7 +1,7 @@ """Convert flat per-session JSONL dumps into weka-format trace JSON. Reads /.jsonl produced by `sample_proxy_traces.py` -and writes /..//.json in the v1 weka trace +and writes /.json in the v1 weka trace format consumed by the kv-cache-tester replayer (see agentx-harness: src/aiperf/dataset/loader/weka_trace_models.py). From 9561540228aafe5dda9b4aa5126c828a3c679626 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:57:52 -0500 Subject: [PATCH 149/203] docs(bench-serving): show a client command that uses the random dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module usage ran the client with --dataset-name sharegpt, which argparse rejects because random is the only dataset the client supports. It also pointed at a launch_tgi_server.sh script this repository does not ship. The example now passes the random dataset and its length flags. 中文:模块用法示例使用 --dataset-name sharegpt 运行客户端,但客户端只支持 random 数据集,argparse 会拒绝该参数。示例还引用了本仓库并不存在的 launch_tgi_server.sh 脚本。示例现在使用 random 数据集及其长度参数。 --- .../infx/bench_serving/benchmark_serving.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/infx/bench_serving/benchmark_serving.py b/inferencex-e2e/infx/bench_serving/benchmark_serving.py index 9b2a0aa2f7..fec4874a9d 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_serving.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_serving.py @@ -1,21 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 r"""Benchmark online serving throughput. -On the server side, run one of the following commands: - vLLM OpenAI API server +On the server side, start an OpenAI-compatible server, for example: vllm serve \ --swap-space 16 \ --disable-log-requests - (TGI backend) - ./launch_tgi_server.sh - On the client side, run: python -m infx.bench_serving.benchmark_serving \ --backend \ --model \ - --dataset-name sharegpt \ - --dataset-path \ + --dataset-name random \ + --random-input-len \ + --random-output-len \ --request-rate \ # By default is inf --num-prompts # By default is 1000 From 1cc7ea6f482b6bf11da2de7088682928113488c9 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:02:04 -0500 Subject: [PATCH 150/203] docs(matrix): describe how the concurrency and parallelism caps rewrite sweeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-sweep help called --min-conc, --max-conc, --max-tp and --max-ep plain filters. For fixed-sequence rows the concurrency bounds also move a single-node range to the bound and run a config at --max-conc when nothing is left, --max-ep caps EP instead of dropping rows, and agentic rows ignore --max-tp and --max-ep. The help now says so. 中文:full-sweep 帮助把 --min-conc、--max-conc、--max-tp 和 --max-ep 描述为单纯的过滤条件。对固定序列行而言,并发上下限还会把单节点范围移到边界值,并在没有剩余并发时以 --max-conc 运行该配置。--max-ep 会把 EP 截断而不是丢弃行,agentic 行则忽略 --max-tp 和 --max-ep。帮助现在如实说明这些行为。 --- inferencex-e2e/infx/matrix/generate.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 4deea17513..46d0927b5f 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1650,25 +1650,31 @@ def main() -> list[dict]: "--min-conc", type=int, required=False, - help="Minimum concurrency value to include (filters out lower concurrency values)", + help=( + "Drop concurrencies below this value. A single-node fixed-sequence " + "conc-start/conc-end range that begins lower starts at this value instead." + ), ) full_sweep_parser.add_argument( "--max-conc", type=int, required=False, - help="Maximum concurrency value to include (filters out higher concurrency values)", + help=( + "Drop concurrencies above this value. A single-node fixed-sequence range that " + "ends higher ends at it, and a fixed-sequence config with nothing left runs at it." + ), ) full_sweep_parser.add_argument( "--max-tp", type=int, required=False, - help="Maximum tensor parallelism value to include (single-node only)", + help="Drop single-node fixed-sequence configs whose TP exceeds this value", ) full_sweep_parser.add_argument( "--max-ep", type=int, required=False, - help="Maximum expert parallelism value to include (single-node only)", + help="Cap the EP of single-node fixed-sequence configs at this value", ) full_sweep_parser.add_argument( "--single-node", From 50941597b947b43a52f998a02b5cbdfe15f5964f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 05:38:33 -0500 Subject: [PATCH 151/203] fix(launch): skip PyTorch-format sidecars when staging srt results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit copy_fixed_sequence_results globs results_concurrency_*.json, which also matches the results_concurrency_*.pytorch.json sidecar benchmark_serving writes when SAVE_TO_PYTORCH_BENCHMARK_FORMAT is set. The sidecar's name has no "_gen_N.json", so for a disaggregated point _point_fields returns an empty decode count and staging fails, and for an aggregate point the sidecar is renamed to the point's own name and overwrites its result. The stager now skips the sidecar, as the result reader and the job event already do. 中文:暂存 srt 结果时跳过 PyTorch 格式的附属文件。 copy_fixed_sequence_results 匹配 results_concurrency_*.json,这也会匹配 benchmark_serving 在设置 SAVE_TO_PYTORCH_BENCHMARK_FORMAT 时写出的 results_concurrency_*.pytorch.json 附属文件。附属文件名中没有 "_gen_N.json", 因此对于分离式测点,_point_fields 返回空的 decode 数量,暂存失败;对于聚合式 测点,附属文件被重命名为该测点自身的文件名并覆盖其结果。暂存现在会跳过该附属 文件,与结果读取器和 job event 的做法一致。 --- inferencex-e2e/infx/launch/artifacts.py | 3 ++- inferencex-e2e/infx/tests/launch/test_launch_artifacts.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/launch/artifacts.py b/inferencex-e2e/infx/launch/artifacts.py index 555ef2dd9f..44b7c88a8a 100644 --- a/inferencex-e2e/infx/launch/artifacts.py +++ b/inferencex-e2e/infx/launch/artifacts.py @@ -85,7 +85,8 @@ def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename _say(f"Processing result subdirectory: {subdir}") config_name = subdir.name for result_file in sorted(subdir.rglob("results_concurrency_*.json")): - if not result_file.is_file(): + # benchmark_serving's PyTorch-format sidecar shares the point's name prefix. + if not result_file.is_file() or result_file.name.endswith(".pytorch.json"): continue concurrency, gpus, ctx, gen = _point_fields(result_file.name) _say( diff --git a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py index e82f1b37dc..5396239af0 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py @@ -25,7 +25,9 @@ def test_fixed_sequence_results_use_bounded_point_names(tmp_path): point = logs / "cfg_isl1024_osl128" (point / "nested").mkdir(parents=True) (point / "results_concurrency_4_gpus_8.json").write_text("agg") + (point / "results_concurrency_4_gpus_8.pytorch.json").write_text("agg sidecar") (point / "nested" / "results_concurrency_16_gpus_12_ctx_2_gen_3.json").write_text("disagg") + (point / "nested" / "results_concurrency_16_gpus_12_ctx_2_gen_3.pytorch.json").write_text("[]") (logs / "unrelated").mkdir() (logs / "unrelated" / "results_concurrency_1_gpus_1.json").write_text("ignored") From 89693343ee79a73dd67b5550af09635cbdb21ff7 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:13:54 -0500 Subject: [PATCH 152/203] fix(matrix): reject test-config concurrencies no selected config generates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-config --conc documents that each value must exist in the config's conc-range or conc-list, but a value outside it was silently dropped. An e2e dispatch or eval campaign shard with such a value produced fewer rows, or an empty matrix that skipped every job and passed green. The generator now fails and names the concurrencies no selected row uses. 中文:test-config --conc 的说明要求每个值都必须出现在配置的 conc-range 或 conc-list 中,但超出范围的值会被静默丢弃。带有这种值的 e2e 调度或评测分片会生成更少的行,或生成空矩阵,从而跳过所有任务并显示通过。现在生成器会报错,并列出没有任何所选行使用的并发值。 --- inferencex-e2e/infx/matrix/generate.py | 13 ++++++++++++- .../tests/matrix/test_generate_sweep_configs.py | 16 ++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 46d0927b5f..88f700fa2c 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1290,7 +1290,7 @@ def _expand_selected_configs( root: Path | None = None, ) -> list[dict]: resolved_keys = expand_config_keys(config_keys, all_config_data.keys()) - return _expand_configs( + rows = _expand_configs( (all_config_data[key] for key in resolved_keys), runner_data or {}, runner_node_filter=runner_node_filter, @@ -1299,6 +1299,17 @@ def _expand_selected_configs( concurrencies=concurrencies, root=root, ) + if concurrencies: + generated = set() + for row in rows: + conc = row[Fields.CONC.value] + generated.update(conc if isinstance(conc, list) else [conc]) + if missing := [conc for conc in concurrencies if conc not in generated]: + raise ValueError( + "--conc value(s) not generated by the selected configs: " + + ", ".join(map(str, missing)) + ) + return rows def _expand_configs( diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index 4095ad274f..c4a05feed1 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -2663,18 +2663,22 @@ def test_point_order_and_input_preservation( assert [(e["runner"], e["conc"]) for e in entries] == expected assert config == original - @pytest.mark.parametrize(("full_filters", "exact_filters", "expected"), [ - ({}, {}, [3, 6, 10]), - ({"min_conc": 5, "max_conc": 9}, {"conc": [6, 9]}, [6]), - ({"max_conc": 2}, {"conc": [2]}, []), - ({"min_conc": 11}, {"conc": [11]}, []), + @pytest.mark.parametrize(("full_filters", "exact_filters", "expected", "rejected"), [ + ({}, {}, [3, 6, 10], None), + ({"min_conc": 5, "max_conc": 9}, {"conc": [6]}, [6], None), + ({"max_conc": 2}, {"conc": [2]}, [], "2"), + ({"min_conc": 11}, {"conc": [6, 11]}, [], "11"), ]) def test_range_boundaries( self, agentic_config, sample_runner_config, generate_agentic_sweep, - agentic_mode, full_filters, exact_filters, expected, + agentic_mode, full_filters, exact_filters, expected, rejected, ): config, benchmark = agentic_config benchmark.update({"conc-start": 3, "conc-end": 10}) + if agentic_mode == "test-config" and rejected is not None: + with pytest.raises(ValueError, match=f"not generated by the selected configs: {rejected}$"): + generate_agentic_sweep(config, sample_runner_config, **exact_filters) + return filters = full_filters if agentic_mode == "full-sweep" else exact_filters entries = generate_agentic_sweep(config, sample_runner_config, **filters) points = [entry["conc"] for entry in entries] From 4dd3b6281ee309506c095e98c8de1c96dedbb95c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:15:08 -0500 Subject: [PATCH 153/203] fix(matrix): reject full-sweep filter values that no config has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit full-sweep checked --runner-type against the runner inventory but took any --model-prefix, --precision or --framework value. A typo such as --framework trtllm (master configs say trt or dynamo-trt) or --precision FP8 filtered out every config, and the e2e dispatch passed with no jobs. Each value must now match at least one loaded master config. 中文:full-sweep 会对照 runner 清单检查 --runner-type,但对 --model-prefix、--precision 和 --framework 的任意值都直接接受。像 --framework trtllm(主配置写的是 trt 或 dynamo-trt)或 --precision FP8 这样的拼写错误会过滤掉所有配置,e2e 调度在没有任何任务的情况下显示通过。现在每个值都必须匹配至少一个已加载的主配置。 --- inferencex-e2e/infx/matrix/generate.py | 13 ++++ .../matrix/test_generate_sweep_configs.py | 66 ++++++++++--------- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 88f700fa2c..0ae80db1d6 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1173,6 +1173,19 @@ def expand_full_sweep( f"Invalid runner type(s): {invalid_runners}. " f"Valid runner types are: {', '.join(sorted(valid_runner_types))}" ) + precisions = {config[Fields.PRECISION.value] for config in master_config.values()} + frameworks = {config[Fields.FRAMEWORK.value] for config in master_config.values()} + for label, requested, exists in ( + ( + "model prefix", + options.model_prefix, + lambda p: any(k.startswith(f"{p}-") for k in master_config), + ), + ("precision", options.precision, precisions.__contains__), + ("framework", options.framework, frameworks.__contains__), + ): + if missing := [value for value in requested or () if not exists(value)]: + raise ValueError(f"No config has {label}: {', '.join(missing)}") seq_filter = {seq_len_stoi[sl] for sl in options.seq_lens} if options.seq_lens else None configs = ( diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index c4a05feed1..03c7a16995 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -1264,7 +1264,7 @@ def test_filter_by_model_prefix(self, sample_single_node_config, sample_runner_c sample_single_node_config["dsr1flash-fp8-mi300x-sglang"] = {**entry, "model-prefix": "dsr1flash"} selected = {} - for prefixes in (["dsr1"], ["dsr1flash"], ["dsr1", "dsr1flash"], ["nonexistent"]): + for prefixes in (["dsr1"], ["dsr1flash"], ["dsr1", "dsr1flash"]): full_sweep_args_single_node.model_prefix = prefixes result = generate_full_sweep( full_sweep_args_single_node, @@ -1277,42 +1277,44 @@ def test_filter_by_model_prefix(self, sample_single_node_config, sample_runner_c ("dsr1",): ["dsr1"], ("dsr1flash",): ["dsr1flash"], ("dsr1", "dsr1flash"): ["dsr1", "dsr1flash"], - ("nonexistent",): [], } - def test_filter_by_precision(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): - full_sweep_args_single_node.precision = ["fp8"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) > 0 + @pytest.mark.parametrize("field", ["precision", "framework"]) + def test_filter_by_config_field( + self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node, field, + ): + entry = sample_single_node_config["dsr1-fp8-mi300x-sglang"] + sample_single_node_config["dsr1-fp4-mi300x-vllm"] = { + **entry, "precision": "fp4", "framework": "vllm", + } - full_sweep_args_single_node.precision = ["fp4"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) == 0 + selected = {} + for value in ("fp8", "fp4") if field == "precision" else ("sglang", "vllm"): + setattr(full_sweep_args_single_node, field, [value]) + result = generate_full_sweep( + full_sweep_args_single_node, sample_single_node_config, sample_runner_config, + ) + selected[value] = {row["precision"] for row in result} - def test_filter_by_framework(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): - full_sweep_args_single_node.framework = ["sglang"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) > 0 + if field == "precision": + assert selected == {"fp8": {"fp8"}, "fp4": {"fp4"}} + else: + assert selected == {"sglang": {"fp8"}, "vllm": {"fp4"}} - full_sweep_args_single_node.framework = ["vllm"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) == 0 + @pytest.mark.parametrize(("filters", "message"), [ + ({"model_prefix": ["dsr1", "deepseek"]}, "No config has model prefix: deepseek"), + ({"precision": ["FP8"]}, "No config has precision: FP8"), + ({"framework": ["trtllm"]}, "No config has framework: trtllm"), + ]) + def test_filter_value_no_config_has_is_rejected( + self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node, + filters, message, + ): + vars(full_sweep_args_single_node).update(filters) + with pytest.raises(ValueError, match=f"^{message}$"): + generate_full_sweep( + full_sweep_args_single_node, sample_single_node_config, sample_runner_config, + ) def test_filter_by_runner_type(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): full_sweep_args_single_node.runner_type = ["mi300x"] From 574a5257eee70554a38628aada335cc499fa2b05 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:18:38 -0500 Subject: [PATCH 154/203] fix(workflows): reject misspelled keys in the CI priority policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_policy checked only the version, and scoring read every adjustment with .get(), so a misspelled key such as multinode (the master-config spelling) for multi-node silently scored as zero. Every CI matrix job then queued at the wrong priority without any error. Unknown top-level and adjustment keys now fail the policy load. 中文:load_policy 只检查版本,评分时用 .get() 读取每项调整,因此像把 multi-node 拼成 multinode(主配置中的写法)这样的错误键会被静默按零分处理。随后每个 CI 矩阵任务都会以错误的优先级排队,且没有任何报错。现在未知的顶层键和调整项键会使策略加载失败。 --- .../infx/tests/workflows/test_ci_priority.py | 15 ++++++++++++++ inferencex-e2e/infx/workflows/ci_priority.py | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/inferencex-e2e/infx/tests/workflows/test_ci_priority.py b/inferencex-e2e/infx/tests/workflows/test_ci_priority.py index d0f8b78460..3c0a8541ca 100644 --- a/inferencex-e2e/infx/tests/workflows/test_ci_priority.py +++ b/inferencex-e2e/infx/tests/workflows/test_ci_priority.py @@ -1,11 +1,13 @@ from decimal import Decimal import pytest +import yaml from infx.workflows.ci_priority import ( PriorityContext, annotate_jobs, calculate_priority, + load_policy, queue_token, ) @@ -32,6 +34,19 @@ def policy(): } +@pytest.mark.parametrize(("misspell", "unknown"), [ + (lambda p: p["adjustments"].update({"multinode": p["adjustments"].pop("multi-node")}), + "adjustments.multinode"), + (lambda p: p.update({"patchwork_score": p.pop("patchwork-score")}), "patchwork_score"), +]) +def test_load_policy_rejects_misspelled_keys(policy, tmp_path, misspell, unknown): + misspell(policy) + path = tmp_path / "ci-priority.yaml" + path.write_text(yaml.safe_dump(policy)) + with pytest.raises(ValueError, match=f"^Unknown CI priority policy keys: {unknown}$"): + load_policy(path) + + def test_combined_signals_add_their_configured_weights(policy): baseline = { "runner": "h100", diff --git a/inferencex-e2e/infx/workflows/ci_priority.py b/inferencex-e2e/infx/workflows/ci_priority.py index fa588e397b..85ce5f05e0 100755 --- a/inferencex-e2e/infx/workflows/ci_priority.py +++ b/inferencex-e2e/infx/workflows/ci_priority.py @@ -17,6 +17,21 @@ from infx.config import repository_root SCORE_QUANTUM = Decimal("0.001") +POLICY_KEYS = frozenset({"version", "base-score", "adjustments", "patchwork-score"}) +ADJUSTMENT_KEYS = frozenset( + { + "additional-node", + "event", + "multi-node", + "agentic", + "eval-only", + "checklist-complete", + "precision", + "spec-decoding", + "framework-prefix", + "model-prefix", + } +) @dataclass(frozen=True) @@ -35,6 +50,11 @@ def load_policy(path: str | Path) -> dict[str, Any]: policy = yaml.safe_load(policy_file) if policy.get("version") != 1: raise ValueError(f"Unsupported CI priority policy version: {policy.get('version')}") + unknown = sorted(set(policy) - POLICY_KEYS) + sorted( + f"adjustments.{key}" for key in set(policy.get("adjustments") or {}) - ADJUSTMENT_KEYS + ) + if unknown: + raise ValueError(f"Unknown CI priority policy keys: {', '.join(unknown)}") return policy From 8be2d08f09cf221a6cf694b688881d7398a24698 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:19:18 -0500 Subject: [PATCH 155/203] fix(evals): reject unknown sections in the eval thresholds file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thresholds file with default and a misspelled models section, such as model:, loaded without error and dropped every per-model threshold. Evals for those models were then gated by the looser default and passed scores the stricter per-model bound would fail. Sections other than default and models now fail the thresholds load. 中文:如果阈值文件包含 default 和一个拼错的 models 段(例如 model:),会正常加载并丢弃所有按模型设置的阈值。这些模型的评测随后按更宽松的默认阈值判定,通过了本应被更严格的模型阈值拒绝的分数。现在除 default 和 models 之外的段会使阈值加载失败。 --- inferencex-e2e/infx/evals/validate_scores.py | 4 ++++ inferencex-e2e/infx/tests/evals/test_batched_eval.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index a6f1ea0dca..e586720fa6 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -38,6 +38,10 @@ def load_config(path: str) -> dict: raise ValueError("thresholds config must be a mapping") if "default" not in cfg and "models" not in cfg: cfg = {"default": cfg} + if unknown := sorted(set(cfg) - {"default", "models"}, key=str): + raise ValueError( + f"thresholds config has unknown section(s): {', '.join(map(str, unknown))}" + ) sections = {name: {} if cfg.get(name) is None else cfg[name] for name in ("default", "models")} for name, section in sections.items(): if not isinstance(section, dict): diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index 50dd93b237..ea6888496e 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -298,6 +298,10 @@ def test_validate_scores_fails_when_thresholds_cannot_be_loaded( "default:\n gsm8k: 0.8\nmodels:\n dsr1:\n gsm8k: '0.9'\n", "thresholds config models.dsr1.gsm8k must be a finite number, got '0.9'", ), + ( + "default:\n gsm8k: 0.8\nmodel:\n dsr1:\n gsm8k: 0.9\n", + "thresholds config has unknown section(s): model", + ), ], ) def test_validate_scores_handles_empty_or_malformed_threshold_sections( From ddbdf5d3bf85b80b8d631bc5cf3a67ac5409542c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:20:12 -0500 Subject: [PATCH 156/203] fix(datasets): reject --dwbug-min-peak without the dynamic-workflow filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The weka dataset builder forwards --dwbug-min-peak to the sampler only with --exclude-dynamic-workflow-bug, so a build given just the threshold kept every buggy session without saying so. The builder now fails when the threshold is passed without the filter it configures. 中文:weka 数据集构建器只有在指定 --exclude-dynamic-workflow-bug 时才会把 --dwbug-min-peak 传给采样器,因此只给出阈值的构建会保留所有受该 bug 影响的会话,且没有任何提示。现在如果传入阈值却没有启用它所配置的过滤器,构建器会报错。 --- .../infx/datasets/build_weka_hf_dataset.py | 5 ++++- .../tests/datasets/test_build_weka_hf_dataset.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index 454cf8c502..e0619274cd 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -146,7 +146,10 @@ def parse_args() -> argparse.Namespace: help="Build payloads but don't push to HF.", ) - return p.parse_args() + args = p.parse_args() + if args.dwbug_min_peak is not None and not args.exclude_dynamic_workflow_bug: + p.error("--dwbug-min-peak requires --exclude-dynamic-workflow-bug") + return args def _run(cmd: list, **kw: Any) -> None: diff --git a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py index 08aff3bb0a..dc0d4df1fe 100644 --- a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py @@ -156,6 +156,19 @@ def test_random_sampling_forwards_seed_to_sampler_and_readme(build, tmp_path: Pa assert "--seed 7" in readme +def test_dwbug_min_peak_without_the_filter_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + argv = ["build", "--repo-base", "org/base", "--work-dir", str(tmp_path), "--dwbug-min-peak", "2"] + monkeypatch.setattr(sys, "argv", argv) + + with pytest.raises(SystemExit) as exit_info: + build_weka_hf_dataset.parse_args() + + assert exit_info.value.code == 2 + assert "--dwbug-min-peak requires --exclude-dynamic-workflow-bug" in capsys.readouterr().err + + def _request(t: float, api_time: float, tokens: int = 1) -> dict: return { "t": t, From b87ed870525bac80d7d379cddc2eeca0b8ebb93b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:13:32 -0500 Subject: [PATCH 157/203] fix(results): compare a sweep result only with its own model's baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compare-results step looked up main's baseline with c.model LIKE prefix plus a wildcard, so a dsv4 result also matched dsv41flash configs and took whichever row was newer. The database stores the model prefix itself as the model key, so the lookup now requires an exact match. Results whose prefix no other model extends compare as before. 中文:compare-results 步骤用 c.model LIKE 前缀加通配符来查找 main 上的基线,因此 dsv4 的结果也会匹配 dsv41flash 的配置,并取其中较新的一行。数据库直接以模型前缀作为模型键,所以查找现在要求完全匹配。前缀没有被其他模型延长的结果,比较方式保持不变。 --- .../infx/results/compare_results.py | 2 +- .../tests/results/test_compare_results.py | 86 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 inferencex-e2e/infx/tests/results/test_compare_results.py diff --git a/inferencex-e2e/infx/results/compare_results.py b/inferencex-e2e/infx/results/compare_results.py index ad49479b8a..989fc90d4c 100644 --- a/inferencex-e2e/infx/results/compare_results.py +++ b/inferencex-e2e/infx/results/compare_results.py @@ -102,7 +102,7 @@ def build_config_params(result: dict) -> dict: JOIN workflow_runs wr ON wr.id = br.workflow_run_id WHERE c.hardware = %(hardware)s AND c.framework = %(framework)s - AND c.model LIKE %(model)s || '%%' + AND c.model = %(model)s AND c.precision = %(precision)s AND c.spec_method = %(spec_method)s AND c.disagg = %(disagg)s diff --git a/inferencex-e2e/infx/tests/results/test_compare_results.py b/inferencex-e2e/infx/tests/results/test_compare_results.py new file mode 100644 index 0000000000..6244137d62 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/test_compare_results.py @@ -0,0 +1,86 @@ +"""The PR summary compares each fixed-sequence result with main's latest row of its config.""" + +import json +import re +import sqlite3 +import sys + +import infx.results.compare_results as compare + +SCHEMA = """ +CREATE TABLE configs ( + id INTEGER PRIMARY KEY, hardware TEXT, framework TEXT, model TEXT, precision TEXT, + spec_method TEXT, disagg BOOLEAN, is_multinode BOOLEAN, prefill_tp INTEGER, + prefill_ep INTEGER, prefill_dp_attention BOOLEAN, decode_tp INTEGER, decode_ep INTEGER, + decode_dp_attention BOOLEAN +); +CREATE TABLE workflow_runs (id INTEGER PRIMARY KEY, head_branch TEXT); +CREATE TABLE benchmark_results ( + config_id INTEGER, workflow_run_id INTEGER, metrics TEXT, isl INTEGER, osl INTEGER, + conc INTEGER, date TEXT, error TEXT +); +INSERT INTO workflow_runs VALUES (1, 'main'); +""" + + +class Database: + """An in-memory SQL database behind psycopg2's pyformat parameters.""" + + def __init__(self, configs: list[tuple[str, str, str, float]]): + self.connection = sqlite3.connect(":memory:") + self.connection.executescript(SCHEMA) + for index, (model, hardware, date, tput) in enumerate(configs, start=1): + self.connection.execute( + "INSERT INTO configs VALUES (?, ?, 'sglang', ?, 'fp4', 'none', 0, 0, 8, 1, 0, 8, 1, 0)", + (index, hardware, model), + ) + self.connection.execute( + "INSERT INTO benchmark_results VALUES (?, 1, ?, 8192, 1024, 4, ?, NULL)", + (index, json.dumps({"tput_per_gpu": tput}), date), + ) + + def cursor(self): + return self + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, query: str, params: dict) -> None: + query = re.sub(r"%\((\w+)\)s", r":\1", query).replace("%%", "%") + self.rows = self.connection.execute(query, params).fetchall() + + def fetchone(self): + return self.rows[0] if self.rows else None + + def close(self) -> None: + self.connection.close() + + +def summary(tmp_path, monkeypatch, capsys, database: Database, **result) -> str: + row = { + "hw": "b200", "infmax_model_prefix": "dsv4", "model": "deepseek-ai/DeepSeek-V4-Pro", + "framework": "sglang", "precision": "fp4", "spec_decoding": "none", "disagg": False, + "is_multinode": False, "tp": 8, "ep": 1, "dp_attention": False, "isl": 8192, + "osl": 1024, "conc": 4, "tput_per_gpu": 110.0, **result, + } # fmt: skip + (tmp_path / "agg.json").write_text(json.dumps([row])) + monkeypatch.setenv("DATABASE_URL", "postgresql://unused") + monkeypatch.setattr(compare.psycopg2, "connect", lambda url: database) + monkeypatch.setattr(sys, "argv", ["compare_results", str(tmp_path)]) + compare.main() + return capsys.readouterr().out + + +def test_a_model_prefix_is_not_compared_with_a_longer_prefix_it_begins( + tmp_path, monkeypatch, capsys +): + database = Database( + [("dsv4", "b200", "2026-10-01", 100.0), ("dsv41flash", "b200", "2026-10-20", 400.0)] + ) + + out = summary(tmp_path, monkeypatch, capsys, database) + + assert r"\text{+10.00}$ (+10.0%)" in out From b5e5044776feac67e4e61b92bb47662d2418836f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:36 -0500 Subject: [PATCH 158/203] fix(results): look up compare baselines under the GPU a runner label names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Results carry their runner label as hw, such as cluster:b200-nscale or gb300-nv, but ingestion stores the GPU before the first dash as the hardware key. compare-results only stripped -multinode, -trt and -disagg, so every result from a cluster label or a vendor-suffixed label found no baseline. The lookup and the table's Hardware column now use the same GPU key ingestion stores. 中文:结果中的 hw 字段是 runner 标签,例如 cluster:b200-nscale 或 gb300-nv,而入库时以第一个短横线之前的 GPU 名作为 hardware 键。compare-results 只去掉 -multinode、-trt 和 -disagg 后缀,因此来自 cluster 标签或带厂商后缀标签的结果都找不到基线。现在查找和表格的 Hardware 列都使用与入库相同的 GPU 键。 --- inferencex-e2e/infx/results/compare_results.py | 4 ++-- .../infx/tests/results/test_compare_results.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/infx/results/compare_results.py b/inferencex-e2e/infx/results/compare_results.py index 989fc90d4c..1f49bb1c14 100644 --- a/inferencex-e2e/infx/results/compare_results.py +++ b/inferencex-e2e/infx/results/compare_results.py @@ -44,8 +44,8 @@ def compute_delta_str( def extract_hardware(runner: str) -> str: - """Strip suffixes like -multinode, -trt, -disagg from runner to get hardware name.""" - return re.split(r"-(multinode|trt|disagg)$", runner)[0].lower() + """The GPU of a runner label such as cluster:b200-nscale, the key ingestion stores.""" + return runner.lower().rpartition(":")[2].split("-", 1)[0] def build_config_params(result: dict) -> dict: diff --git a/inferencex-e2e/infx/tests/results/test_compare_results.py b/inferencex-e2e/infx/tests/results/test_compare_results.py index 6244137d62..e18471aef7 100644 --- a/inferencex-e2e/infx/tests/results/test_compare_results.py +++ b/inferencex-e2e/infx/tests/results/test_compare_results.py @@ -5,6 +5,8 @@ import sqlite3 import sys +import pytest + import infx.results.compare_results as compare SCHEMA = """ @@ -31,7 +33,8 @@ def __init__(self, configs: list[tuple[str, str, str, float]]): self.connection.executescript(SCHEMA) for index, (model, hardware, date, tput) in enumerate(configs, start=1): self.connection.execute( - "INSERT INTO configs VALUES (?, ?, 'sglang', ?, 'fp4', 'none', 0, 0, 8, 1, 0, 8, 1, 0)", + "INSERT INTO configs VALUES" + " (?, ?, 'sglang', ?, 'fp4', 'none', 0, 0, 8, 1, 0, 8, 1, 0)", (index, hardware, model), ) self.connection.execute( @@ -84,3 +87,16 @@ def test_a_model_prefix_is_not_compared_with_a_longer_prefix_it_begins( out = summary(tmp_path, monkeypatch, capsys, database) assert r"\text{+10.00}$ (+10.0%)" in out + + +@pytest.mark.parametrize( + ("runner", "hardware"), [("cluster:b200-nscale", "b200"), ("gb300-nv", "gb300")] +) +def test_a_runner_label_finds_the_baseline_of_its_gpu( + tmp_path, monkeypatch, capsys, runner, hardware +): + database = Database([("dsv4", hardware, "2026-10-01", 100.0)]) + + out = summary(tmp_path, monkeypatch, capsys, database, hw=runner) + + assert r"\text{+10.00}$ (+10.0%)" in out From 02ea3c1674d3b2c3f534dd6ca9fc17d554abf89b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:27:57 -0500 Subject: [PATCH 159/203] fix(results): look up sglang-disagg compare baselines as mori-sglang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit InferenceX-app ingestion stores sglang-disagg results under the framework mori-sglang (and atom-disagg under mooncake-atom). compare-results queried the raw framework, so the mi355x sglang-disagg sweeps never found a baseline on main. The lookup now uses the name the database holds. 中文:InferenceX-app 入库时把 sglang-disagg 的结果存为 mori-sglang 框架(atom-disagg 存为 mooncake-atom)。compare-results 用原始框架名查询,因此 mi355x 上 sglang-disagg 的扫描从来找不到 main 上的基线。现在查找使用数据库中保存的名称。 --- .../infx/results/compare_results.py | 11 +++++++++- .../tests/results/test_compare_results.py | 20 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/infx/results/compare_results.py b/inferencex-e2e/infx/results/compare_results.py index 1f49bb1c14..5c2a90306f 100644 --- a/inferencex-e2e/infx/results/compare_results.py +++ b/inferencex-e2e/infx/results/compare_results.py @@ -43,6 +43,15 @@ def compute_delta_str( return colorize_delta(delta, pct, higher_is_better, fmt) +# InferenceX-app ingestion stores these frameworks under their canonical names. +FRAMEWORK_ALIASES = { + "atom-disagg": "mooncake-atom", + "sglang-disagg": "mori-sglang", + "trtllm": "trt", + "dynamo-trtllm": "dynamo-trt", +} + + def extract_hardware(runner: str) -> str: """The GPU of a runner label such as cluster:b200-nscale, the key ingestion stores.""" return runner.lower().rpartition(":")[2].split("-", 1)[0] @@ -53,7 +62,7 @@ def build_config_params(result: dict) -> dict: is_multinode = result.get("is_multinode", False) hw = extract_hardware(result["hw"]) model = result["infmax_model_prefix"].lower() - framework = result["framework"].lower() + framework = FRAMEWORK_ALIASES.get(result["framework"].lower(), result["framework"].lower()) precision = result["precision"].lower() spec_method = result.get("spec_decoding", "none").lower() disagg = parse_bool(result.get("disagg", False)) diff --git a/inferencex-e2e/infx/tests/results/test_compare_results.py b/inferencex-e2e/infx/tests/results/test_compare_results.py index e18471aef7..3fcbd9a055 100644 --- a/inferencex-e2e/infx/tests/results/test_compare_results.py +++ b/inferencex-e2e/infx/tests/results/test_compare_results.py @@ -28,14 +28,13 @@ class Database: """An in-memory SQL database behind psycopg2's pyformat parameters.""" - def __init__(self, configs: list[tuple[str, str, str, float]]): + def __init__(self, configs: list[tuple[str, str, str, float]], framework: str = "sglang"): self.connection = sqlite3.connect(":memory:") self.connection.executescript(SCHEMA) for index, (model, hardware, date, tput) in enumerate(configs, start=1): self.connection.execute( - "INSERT INTO configs VALUES" - " (?, ?, 'sglang', ?, 'fp4', 'none', 0, 0, 8, 1, 0, 8, 1, 0)", - (index, hardware, model), + "INSERT INTO configs VALUES (?, ?, ?, ?, 'fp4', 'none', 0, 0, 8, 1, 0, 8, 1, 0)", + (index, hardware, framework, model), ) self.connection.execute( "INSERT INTO benchmark_results VALUES (?, 1, ?, 8192, 1024, 4, ?, NULL)", @@ -100,3 +99,16 @@ def test_a_runner_label_finds_the_baseline_of_its_gpu( out = summary(tmp_path, monkeypatch, capsys, database, hw=runner) assert r"\text{+10.00}$ (+10.0%)" in out + + +def test_sglang_disagg_finds_the_baseline_ingestion_stored_as_mori_sglang( + tmp_path, monkeypatch, capsys +): + database = Database([("dsr1", "mi355x", "2026-10-01", 100.0)], framework="mori-sglang") + + out = summary( + tmp_path, monkeypatch, capsys, database, + hw="cluster:mi355x-amds", infmax_model_prefix="dsr1", framework="sglang-disagg", + ) # fmt: skip + + assert r"\text{+10.00}$ (+10.0%)" in out From 1db25427dda5b2eeae17fafc455b2c7752cbae0c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:21:35 -0500 Subject: [PATCH 160/203] fix(klaud): name sglang-disagg and atom-disagg families by their published frameworks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public API serves sglang-disagg and atom-disagg rows under mori-sglang and mooncake-atom, but Klaud built live families and baseline identities from the raw config framework. The mi355x sglang-disagg and atom-disagg AgentX families therefore never matched a catalog row or a published baseline point. Klaud and compare-results now share one alias table in infx.results.identity, limited to the frameworks configs use. 中文:公开 API 把 sglang-disagg 和 atom-disagg 的数据行分别以 mori-sglang 和 mooncake-atom 提供,但 Klaud 用配置中的原始框架名构建 live family 和基线身份。因此 mi355x 上 sglang-disagg 和 atom-disagg 的 AgentX family 从未匹配到目录行或已发布的基线点。现在 Klaud 与 compare-results 共用 infx.results.identity 中的同一张别名表,只包含配置实际使用的框架。 --- inferencex-e2e/infx/klaud/__main__.py | 3 +- inferencex-e2e/infx/klaud/reporting.py | 3 +- .../infx/results/compare_results.py | 13 ++----- inferencex-e2e/infx/results/identity.py | 15 ++++++++ .../infx/tests/klaud/test_klaud_github.py | 35 +++++++++++++++++++ 5 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 inferencex-e2e/infx/results/identity.py diff --git a/inferencex-e2e/infx/klaud/__main__.py b/inferencex-e2e/infx/klaud/__main__.py index 837b98c3fa..362913c34e 100644 --- a/inferencex-e2e/infx/klaud/__main__.py +++ b/inferencex-e2e/infx/klaud/__main__.py @@ -89,6 +89,7 @@ def live_families(root: Path) -> dict[tuple, dict[str, tuple[frozenset[str], int from infx.config import project_root from infx.matrix.generate import _hardware_family, generate_test_config_sweep from infx.matrix.validation import load_config_files, load_runner_file + from infx.results.identity import public_framework from .validation import in_scope @@ -122,7 +123,7 @@ def live_families(root: Path) -> dict[tuple, dict[str, tuple[frozenset[str], int row = { "model": entry["model-prefix"], "hardware": _hardware_family(entry["runner"]), - "framework": entry["framework"], + "framework": public_framework(entry["framework"]), "precision": entry["precision"], "spec_method": entry["spec-decoding"], "disagg": entry.get("disagg", False), diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index 61f2a27d3a..c15081a93a 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -935,6 +935,7 @@ def publish_final( def public_point(entry: dict) -> dict: """Project generated settings onto the public BenchmarkRow identity (not metrics).""" from infx.matrix.generate import _hardware_family + from infx.results.identity import public_framework from .models import normalized_image @@ -943,7 +944,7 @@ def public_point(entry: dict) -> dict: point = { "model": entry["model-prefix"], "hardware": _hardware_family(entry["runner"]), - "framework": entry["framework"], + "framework": public_framework(entry["framework"]), "precision": entry["precision"], "spec_method": entry["spec-decoding"], "disagg": entry.get("disagg", False), diff --git a/inferencex-e2e/infx/results/compare_results.py b/inferencex-e2e/infx/results/compare_results.py index 5c2a90306f..a38e3619ab 100644 --- a/inferencex-e2e/infx/results/compare_results.py +++ b/inferencex-e2e/infx/results/compare_results.py @@ -7,6 +7,8 @@ import psycopg2 from tabulate import tabulate +from infx.results.identity import public_framework + def parse_bool(value: object) -> bool: if isinstance(value, bool): @@ -43,15 +45,6 @@ def compute_delta_str( return colorize_delta(delta, pct, higher_is_better, fmt) -# InferenceX-app ingestion stores these frameworks under their canonical names. -FRAMEWORK_ALIASES = { - "atom-disagg": "mooncake-atom", - "sglang-disagg": "mori-sglang", - "trtllm": "trt", - "dynamo-trtllm": "dynamo-trt", -} - - def extract_hardware(runner: str) -> str: """The GPU of a runner label such as cluster:b200-nscale, the key ingestion stores.""" return runner.lower().rpartition(":")[2].split("-", 1)[0] @@ -62,7 +55,7 @@ def build_config_params(result: dict) -> dict: is_multinode = result.get("is_multinode", False) hw = extract_hardware(result["hw"]) model = result["infmax_model_prefix"].lower() - framework = FRAMEWORK_ALIASES.get(result["framework"].lower(), result["framework"].lower()) + framework = public_framework(result["framework"]) precision = result["precision"].lower() spec_method = result.get("spec_decoding", "none").lower() disagg = parse_bool(result.get("disagg", False)) diff --git a/inferencex-e2e/infx/results/identity.py b/inferencex-e2e/infx/results/identity.py new file mode 100644 index 0000000000..666dd65b07 --- /dev/null +++ b/inferencex-e2e/infx/results/identity.py @@ -0,0 +1,15 @@ +"""Benchmark identity names as InferenceX-app ingestion stores them.""" + +from __future__ import annotations + +# InferenceX-app FRAMEWORK_ALIASES: ingestion stores these config frameworks under other names. +FRAMEWORK_ALIASES = { + "atom-disagg": "mooncake-atom", + "sglang-disagg": "mori-sglang", +} + + +def public_framework(framework: str) -> str: + """The framework name the database and public API hold for a config's ``framework``.""" + framework = framework.lower() + return FRAMEWORK_ALIASES.get(framework, framework) diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 4dd9807ee5..1430b826ab 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -1845,6 +1845,26 @@ def test_multinode_identity_matches_published_aggregate_encodings( assert reporting._matches_public_point(row, reporting.public_point(entry)) is matches +def test_sglang_disagg_identity_matches_its_published_mori_sglang_rows(): + worker = {"num-worker": 1, "tp": 4, "ep": 1, "dp-attn": False} + entry = { + "model-prefix": "dsv4", "runner": "cluster:mi355x-amds", "framework": "sglang-disagg", + "precision": "fp4", "spec-decoding": "draft_model", "disagg": True, + "scenario-type": "agentic-coding", "kv-offloading": "dram", "conc": 16, + "image": "example/image:1", "prefill": worker, "decode": worker, + } # fmt: skip + row = { + "model": "dsv4", "hardware": "mi355x", "framework": "mori-sglang", "precision": "fp4", + "spec_method": "draft_model", "disagg": True, "is_multinode": True, + "benchmark_type": "agentic_traces", "isl": None, "osl": None, "offload_mode": "on", + "conc": 16, "image": "example/image:1", "prefill_tp": 4, "prefill_ep": 1, + "prefill_dp_attention": False, "prefill_num_workers": 1, "decode_tp": 4, + "decode_ep": 1, "decode_dp_attention": False, "decode_num_workers": 1, + } # fmt: skip + + assert reporting._matches_public_point(row, reporting.public_point(entry)) is True + + @pytest.fixture def family_configs(): master = { @@ -2762,6 +2782,21 @@ def test_live_families_preserve_public_identity_after_move(tmp_path, family_conf } +def test_live_families_name_atom_disagg_as_its_published_mooncake_atom(tmp_path, family_configs): + master, runners = family_configs + master["fixture"]["framework"] = "atom-disagg" + configs = tmp_path / "configs" + configs.mkdir() + (configs / "amd-master.yaml").write_text(yaml.safe_dump(master)) + (configs / "runners.yaml").write_text(yaml.safe_dump(runners)) + write_fixture_recipe(tmp_path) + + families = klaud.live_families(tmp_path) + + assert [key[2] for key in families] == ["mooncake-atom"] + + + def test_historical_families_ignore_leftover_nested_results(tmp_path, family_configs): master, runners = family_configs configs = tmp_path / "configs" From 5d52eeb395af8485592c3c1a94454e43e77b3540 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:18:42 -0500 Subject: [PATCH 161/203] fix(matrix): list changed points in generation order when names and concurrencies tie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit infx.matrix.changed took the added and removed points from a set difference of point identities and then sorted them by experiment name and concurrency only. Points that share both, such as the tp4 and tp8 rows of dsr1-fp8-mi355x-sglang at conc 32, came out in an order that changed with PYTHONHASHSEED. The report now walks each matrix in generation order, so ties keep the order the generator produced them in. 中文:infx.matrix.changed 通过点标识的集合差得到新增和删除的点,然后只按实验名称和并发数排序。名称和并发数都相同的点(例如 dsr1-fp8-mi355x-sglang 在 conc 32 下的 tp4 和 tp8 行)的输出顺序会随 PYTHONHASHSEED 变化。现在报告按生成顺序遍历每个矩阵,因此并列的点保持生成器给出的顺序。 --- inferencex-e2e/infx/matrix/changed.py | 4 ++-- inferencex-e2e/infx/tests/matrix/test_changed.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index b26f38bc83..47db409ca6 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -81,8 +81,8 @@ def compare(base: Matrix, head: Matrix) -> list[dict]: changed = [] for key in _keys(base, head): before, after = base.get(key, ({}, []))[0], head.get(key, ({}, []))[0] - added = _sorted(after[point] for point in after.keys() - before.keys()) - removed = _sorted(before[point] for point in before.keys() - after.keys()) + added = _sorted(point for point_id, point in after.items() if point_id not in before) + removed = _sorted(point for point_id, point in before.items() if point_id not in after) if added or removed: changed.append( { diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index d6a75bb497..14a09273ed 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -100,6 +100,16 @@ def matrix(*tps: int) -> changed.Matrix: ] == [(2, 2, [2], [8], [None, None])] # fmt: skip +def test_points_sharing_a_name_and_concurrency_keep_generation_order(tmp_path): + tps = [8, 4, 2, 1, 16, 32, 64, 128] + rows = [{"exp-name": "dsr1_8k1k", "tp": tp, "ep": 1, "conc": [32]} for tp in tps] + head = {"key": changed._points(rows, tmp_path)} + + [change] = changed.compare({}, head) + + assert [point["tp"] for point in change["added"]] == tps + + def test_a_head_config_error_wins_over_a_base_that_cannot_generate(tmp_path, monkeypatch): (tmp_path / "configs").mkdir() (tmp_path / "configs/nvidia-master.yaml").write_text("key: [unclosed\n") From 0641f4f28d487b1bd0e194c8ce465d8d507e21c0 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:19:47 -0500 Subject: [PATCH 162/203] fix(results): aggregate benchmark results in artifact path order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collect-results workflow built agg_bmk.json in whatever order the runner's filesystem listed the downloaded artifact directories. The published results_bmk artifact, and the comparison table that keeps its order for rows with the same model, hardware, framework, sequence lengths and concurrency, therefore listed tp4 and tp8 points in arbitrary order. The collector now reads result files in sorted path order. 中文:collect-results 工作流按运行器文件系统列出已下载产物目录的顺序生成 agg_bmk.json。因此发布的 results_bmk 产物,以及对模型、硬件、框架、序列长度和并发数都相同的行保持该顺序的对比表,会以任意顺序列出 tp4 和 tp8 的点。现在收集器按排序后的路径顺序读取结果文件。 --- inferencex-e2e/infx/results/collect_results.py | 2 +- .../infx/tests/test_module_entrypoints.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/results/collect_results.py b/inferencex-e2e/infx/results/collect_results.py index e30faf7972..8c748dae73 100644 --- a/inferencex-e2e/infx/results/collect_results.py +++ b/inferencex-e2e/infx/results/collect_results.py @@ -8,7 +8,7 @@ def main() -> None: exp_name = sys.argv[2] agg_results = [] - for result_path in results_dir.rglob("*.json"): + for result_path in sorted(results_dir.rglob("*.json")): with open(result_path) as f: result = json.load(f) agg_results.append(result) diff --git a/inferencex-e2e/infx/tests/test_module_entrypoints.py b/inferencex-e2e/infx/tests/test_module_entrypoints.py index 93edc5b367..ceca5fbe02 100644 --- a/inferencex-e2e/infx/tests/test_module_entrypoints.py +++ b/inferencex-e2e/infx/tests/test_module_entrypoints.py @@ -44,6 +44,20 @@ def test_collector_preserves_nested_json_and_empty_inputs(invoke, tmp_path, payl assert json.loads((tmp_path / "agg_test.json").read_text()) == expected +def test_collector_orders_rows_by_artifact_path(invoke, tmp_path): + for name in ("tp8_conc64", "tp4_conc4", "tp8_conc4", "tp2_conc16", "tp4_conc64", "tp1_conc1"): + artifact = tmp_path / "inputs" / f"bmk_dsr1_1k1k_{name}" + artifact.mkdir(parents=True) + (artifact / f"dsr1_1k1k_{name}.json").write_text(json.dumps({"point": name})) + + result = invoke("infx.results.collect_results", "inputs", "test") + + assert result.returncode == 0, result.stderr + assert [row["point"] for row in json.loads((tmp_path / "agg_test.json").read_text())] == [ + "tp1_conc1", "tp2_conc16", "tp4_conc4", "tp4_conc64", "tp8_conc4", "tp8_conc64", + ] + + @pytest.mark.parametrize("published", [None, b'[{"previous": true}]\n']) def test_collector_does_not_publish_partial_output_on_invalid_json(invoke, tmp_path, published): inputs = tmp_path / "inputs" From 0bfa06679c2ad52f2b7f46d330ed7770b2333908 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:20:22 -0500 Subject: [PATCH 163/203] fix(results): collect eval artifacts in directory name order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collect-evals workflow wrote agg_eval_.json in the order the runner's filesystem listed the downloaded eval artifact directories. The published eval_results_all rows, and the step-summary rows whose sort key ties (for example the same configuration with and without DP attention), came out in arbitrary order. Eval sets are now read in sorted directory order. 中文:collect-evals 工作流按运行器文件系统列出已下载评测产物目录的顺序写入 agg_eval_.json。因此发布的 eval_results_all 各行,以及排序键相同的步骤摘要行(例如同一配置开启和关闭 DP attention),会以任意顺序出现。现在按排序后的目录顺序读取评测集。 --- inferencex-e2e/infx/results/collect_eval_results.py | 4 +++- .../infx/tests/results/test_collect_eval_results.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/results/collect_eval_results.py b/inferencex-e2e/infx/results/collect_eval_results.py index 334cff83cc..e11d02d7dd 100644 --- a/inferencex-e2e/infx/results/collect_eval_results.py +++ b/inferencex-e2e/infx/results/collect_eval_results.py @@ -62,7 +62,9 @@ def find_eval_sets(root: Path) -> list[Path]: try: if (root / "meta_env.json").exists(): out.append(root) - out.extend(d for d in root.iterdir() if d.is_dir() and (d / "meta_env.json").exists()) + out.extend( + d for d in sorted(root.iterdir()) if d.is_dir() and (d / "meta_env.json").exists() + ) except Exception: # noqa: BLE001, S110 pass return out diff --git a/inferencex-e2e/infx/tests/results/test_collect_eval_results.py b/inferencex-e2e/infx/tests/results/test_collect_eval_results.py index a6c10d01b0..393df8c53f 100644 --- a/inferencex-e2e/infx/tests/results/test_collect_eval_results.py +++ b/inferencex-e2e/infx/tests/results/test_collect_eval_results.py @@ -406,6 +406,19 @@ def test_collector_cli_runs_outside_checkout(tmp_path: Path) -> None: assert rows[0]["source"] == str(result) +def test_collect_eval_rows_reads_eval_sets_in_name_order(tmp_path: Path) -> None: + scores = {"tp8_dpa": 0.8, "tp4": 0.4, "tp8": 0.81, "tp2": 0.2, "tp4_dpa": 0.41, "tp1": 0.1} + for name, score in scores.items(): + artifact = tmp_path / f"eval_dsr1_8k1k_{name}_conc64" + artifact.mkdir() + (artifact / "meta_env.json").write_text('{"conc": 64}') + _write_lm_eval_result(artifact / "results.json", score) + + rows = collect_eval_rows(tmp_path) + + assert [row["score"] for row in rows] == [0.1, 0.2, 0.4, 0.41, 0.81, 0.8] + + def test_collect_eval_rows_expands_batched_concurrencies( tmp_path: Path, ) -> None: From 98d00d69ce22b0e067e1cd40ff3482f0072854b9 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:33:20 -0500 Subject: [PATCH 164/203] fix(klaud): report fixed-sequence request errors from the benchmark outcome MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final Klaud reports read request errors for fixed-sequence points from an errors key that no result producer writes. Every passed point therefore showed request errors unavailable, and real failed requests never reached the report. The fixed-sequence result records them as benchmark_outcome.failed, so the report now reads that count. 中文: Klaud 最终报告从一个没有任何结果生产者写入的 errors 键读取固定序列点的请求错误数。因此每个通过的点都显示请求错误不可用,真实的失败请求从未出现在报告中。固定序列结果把该数量记录在 benchmark_outcome.failed 中,现在报告改为读取这个计数。 --- inferencex-e2e/infx/klaud/reporting.py | 2 +- .../infx/tests/klaud/test_klaud_github.py | 38 +++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index c15081a93a..32338d5103 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -322,7 +322,7 @@ def values(row: dict) -> Values: output_tps_gpu=metrics.get("output_tput_per_gpu"), ttft_ms=metrics["mean_ttft"] * 1000 if metrics.get("mean_ttft") is not None else None, tpot_ms=metrics["mean_tpot"] * 1000 if metrics.get("mean_tpot") is not None else None, - request_errors=metrics.get("errors"), + request_errors=row.get("benchmark_outcome", {}).get("failed"), ) diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 1430b826ab..7b97c342c4 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -1731,7 +1731,6 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( "output_tput_per_gpu": 8.0, "mean_ttft": 0.1, "mean_tpot": 0.02, - "errors": 0, } info = { "runs": [{"github_run_id": "42", "run_attempt": 1}], @@ -1962,7 +1961,7 @@ def publish(monkeypatch, head): "decode_ep": 1, "decode_dp_attention": False, "decode_num_workers": 0, "run_url": "https://github.com/example/project/actions/runs/42", "tput_per_gpu": 5.0 * conc, "output_tput_per_gpu": 4.0 * conc, "mean_ttft": 0.1, - "mean_tpot": 0.02, "errors": 0, + "mean_tpot": 0.02, } for conc in (2, 6) ] @@ -2133,7 +2132,7 @@ def serve_dsr1_baseline(monkeypatch, current, markdown): "recipe_fingerprint": f"{isl:064x}", "run_url": "https://github.com/example/project/actions/runs/42", "tput_per_gpu": 10.0, "output_tput_per_gpu": 8.0, "mean_ttft": 0.1, - "mean_tpot": 0.02, "errors": 0, + "mean_tpot": 0.02, } for isl in (1024, 8192) ] @@ -2932,6 +2931,39 @@ def test_agentx_deltas_compare_only_the_same_replay_corpus(old, new, cell, note) assert notes == ([f"**Note:** All rows: Δ N/A: {note}."] if note else []) +@pytest.mark.parametrize(("completed", "errors", "note"), [ + pytest.param(100, 0, None, id="no-failed-requests"), + pytest.param(97, 3, "**Note:** All rows: 3 request errors.", id="failed-requests"), +]) # fmt: skip +def test_a_fixed_sequence_result_reports_its_failed_requests(completed, errors, note): + from infx.results.fixed_sequence import build_result + + raw = { + "max_concurrency": 4, "model_id": "m", "num_prompts": 100, "completed": completed, + "total_token_throughput": 800.0, "output_throughput": 400.0, + "mean_ttft_ms": 100.0, "mean_tpot_ms": 10.0, + "benchmark_outcome": {"status": "passed", "requested": 100, "completed": completed, + "failed": 100 - completed, "max_failure_rate": 0.05}, + } # fmt: skip + env = { + "RUNNER_TYPE": "h100", "FRAMEWORK": "sglang", "PRECISION": "fp8", "SPEC_DECODING": "none", + "ISL": "1024", "OSL": "1024", "DISAGG": "false", "MODEL_PREFIX": "dsr1", "IMAGE": "i:1", + "TP": "8", "EP_SIZE": "1", "DP_ATTENTION": "false", + } # fmt: skip + values = reporting.values(build_result(raw, env)) + point = reporting.Point( + key="f" * 64, label="1024/1024 c4 TP8 EP1 ffffff", conc=4, scenario="fixed-seq-len", + values=values, result="passed", + ) # fmt: skip + + text = reporting.point_table([point]) + + assert values.request_errors == errors + assert values.output_tps_gpu == 50.0 + notes = [line.rstrip() for line in text.splitlines() if "request error" in line] + assert notes == ([note] if note else []) + + def test_baseline_notes_name_concurrency_rows_like_their_table(): points = [ agentx_point("cc", output_tps_gpu=10.0, request_errors=2), From f07f9b3e28cd106522f2f8de173bef3644a09de5 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:34:35 -0500 Subject: [PATCH 165/203] fix(launch): stop waiting on NFS locks after the last cleanup attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cleanup_outputs slept another 10 seconds after its fifth failed rmtree even though no sixth attempt follows, and announced that wait as a retry. The srt driver runs it at the end of every job, so a checkout whose outputs stay busy held the runner for nothing. It now stops after the last attempt and says so. 中文:cleanup_outputs 在第五次 rmtree 失败后仍会再等待 10 秒,并把这次等待 报告为重试,但之后并不会有第六次尝试。srt 驱动在每个作业结束时都会调用它, 因此 outputs 一直被占用时,运行器会白白多占用一段时间。现在最后一次尝试 失败后即停止并给出说明。 --- inferencex-e2e/infx/launch/drivers/srt/collect.py | 3 +++ .../infx/tests/launch/test_launch_artifacts.py | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index 2cd09331f7..108fb88b3d 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -258,6 +258,9 @@ def cleanup_outputs(root: Path, *, sleep: Callable[[float], None] = time.sleep) except FileNotFoundError: break except OSError: + if attempt == attempts: + print(f"outputs still busy after {attempts} attempts; leaving it", flush=True) + break print(f"Retry {attempt}/{attempts}: Waiting for NFS locks to release...", flush=True) sleep(10) for directory, dirnames, filenames in os.walk(root, topdown=False): diff --git a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py index 5396239af0..6c85895c50 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py @@ -77,7 +77,8 @@ def test_bundle_skips_empty_logs_and_archives_contents(tmp_path): assert "./worker.log" in tar.getnames() -def test_cleanup_outputs_retries_then_removes_nfs_files(tmp_path, monkeypatch): +@pytest.mark.parametrize(("busy", "removed"), [(2, True), (5, False)], ids=["frees", "stays-busy"]) +def test_cleanup_outputs_retries_then_removes_nfs_files(tmp_path, monkeypatch, busy, removed): (tmp_path / "outputs" / "job").mkdir(parents=True) (tmp_path / "keep" / ".nfs0001").parent.mkdir() (tmp_path / "keep" / ".nfs0001").write_text("") @@ -86,15 +87,15 @@ def test_cleanup_outputs_retries_then_removes_nfs_files(tmp_path, monkeypatch): def flaky_rmtree(path): calls.append(path) - if len(calls) < 3: + if len(calls) <= busy: raise OSError("Device or resource busy") real_rmtree(path) monkeypatch.setattr(collect.shutil, "rmtree", flaky_rmtree) sleeps = [] cleanup_outputs(tmp_path, sleep=sleeps.append) - assert len(calls) == 3 and sleeps == [10, 10] - assert not (tmp_path / "outputs").exists() + assert len(calls) == min(busy + 1, 5) and sleeps == [10] * min(busy, 4) + assert (tmp_path / "outputs").exists() is not removed assert sorted(p.name for p in (tmp_path / "keep").iterdir()) == ["log"] From e36c8cf8c0f22e691ba80d883b8f849047bae2b3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:43:29 -0500 Subject: [PATCH 166/203] fix(klaud): match legacy baseline producers on the config keys ingestion publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit InferenceX-app ingestion rewrites framework aliases inside changelog config keys (trtllm to trt, sglang-disagg to mori-sglang, atom-disagg to mooncake-atom), but Klaud matched a candidate's raw family key against them. A legacy baseline row for the minimaxm3 trtllm or the mi355x disagg AgentX families therefore never found its producer run, and the candidate was deferred every cycle. The family key is now rewritten the same way, and the alias table mirrors the app's four entries. 中文:InferenceX-app 入库时会改写 changelog 配置键中的框架别名(trtllm 改为 trt, sglang-disagg 改为 mori-sglang,atom-disagg 改为 mooncake-atom),而 Klaud 用候选项 原始的 family 键去匹配它们。因此 minimaxm3 trtllm 以及 mi355x disagg AgentX family 的 旧版基线行永远找不到其生产运行,候选项在每个周期都被推迟。现在 family 键按相同方式 改写,别名表也与应用的四个条目保持一致。 --- inferencex-e2e/infx/klaud/reporting.py | 5 ++++- inferencex-e2e/infx/results/identity.py | 10 ++++++++++ .../infx/tests/klaud/test_klaud_github.py | 18 +++++++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index 32338d5103..95b8257214 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -1354,6 +1354,8 @@ def resolve_baseline( """ from fnmatch import fnmatchcase + from infx.results.identity import public_config_key + from .api import fetch from .validation import PRODUCER_UNAVAILABLE @@ -1400,10 +1402,11 @@ def dataset(row: dict, run_id: int, run_attempt: int | None) -> str | None: sources.append(feed.url) return datasets[run_id, run_attempt] + published_family = public_config_key(candidate.family.split(":", 1)[1]) family_runs = { int(change["workflow_run_id"]) for change in public.changelogs - if any(fnmatchcase(candidate.family.split(":", 1)[1], key) for key in change["config_keys"]) + if any(fnmatchcase(published_family, key) for key in change["config_keys"]) } for row, run_id, run_attempt, head in public.rows: if head is None: diff --git a/inferencex-e2e/infx/results/identity.py b/inferencex-e2e/infx/results/identity.py index 666dd65b07..412466707f 100644 --- a/inferencex-e2e/infx/results/identity.py +++ b/inferencex-e2e/infx/results/identity.py @@ -6,10 +6,20 @@ FRAMEWORK_ALIASES = { "atom-disagg": "mooncake-atom", "sglang-disagg": "mori-sglang", + "trtllm": "trt", + "dynamo-trtllm": "dynamo-trt", } +_LONGEST_FIRST = sorted(FRAMEWORK_ALIASES.items(), key=lambda alias: -len(alias[0])) def public_framework(framework: str) -> str: """The framework name the database and public API hold for a config's ``framework``.""" framework = framework.lower() return FRAMEWORK_ALIASES.get(framework, framework) + + +def public_config_key(key: str) -> str: + """A changelog config key as ingestion stores it: every alias replaced, longest first.""" + for alias, canonical in _LONGEST_FIRST: + key = key.replace(alias, canonical) + return key diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 7b97c342c4..167346a1d0 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -1667,8 +1667,12 @@ def test_prepare_baseline_keeps_the_baseline_the_candidate_pr_froze(tmp_path, mo published if frozen else preflight.model_copy(update={"goal": goal})) +@pytest.mark.parametrize(("family_key", "published_key", "fingerprint"), [ + ("test-family", "test-family", "fingerprint"), + ("model-fp8-h200-trtllm-mtp", "model-fp8-h200-trt-mtp", None), +], ids=["fingerprinted", "legacy-row-under-aliased-key"]) # fmt: skip def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( - monkeypatch, + monkeypatch, family_key, published_key, fingerprint ): base = "a" * 40 historical_head = "b" * 40 @@ -1677,7 +1681,7 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( entry = { "model-prefix": "Model", "runner": "h200", - "framework": "trtllm", + "framework": "trt", "precision": "fp8", "spec-decoding": "mtp", "disagg": False, @@ -1706,7 +1710,7 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( public_row = { "model": "Model", "hardware": "h200", - "framework": "trtllm", + "framework": "trt", "precision": "fp8", "spec_method": "mtp", "disagg": False, @@ -1725,7 +1729,7 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( "decode_ep": 1, "decode_dp_attention": False, "decode_num_workers": 0, - "recipe_fingerprint": "fingerprint", + "recipe_fingerprint": fingerprint, "run_url": "https://github.com/example/project/actions/runs/42", "tput_per_gpu": 10.0, "output_tput_per_gpu": 8.0, @@ -1735,7 +1739,7 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( info = { "runs": [{"github_run_id": "42", "run_attempt": 1}], "runConfigs": [{"github_run_id": "42", "head_sha": historical_head}], - "changelogs": [{"workflow_run_id": "42", "config_keys": ["test-family"]}], + "changelogs": [{"workflow_run_id": "42", "config_keys": [published_key]}], } feeds = { @@ -1759,7 +1763,7 @@ def fetch(resource, **_kwargs): producers = {historical_head: historical_matrix} candidate = OwnedCandidate( id="1" * 16 + "-" + "2" * 16, - family="configs/nvidia-master.yaml:test-family", + family=f"configs/nvidia-master.yaml:{family_key}", base=base, ) context = { @@ -1768,7 +1772,7 @@ def fetch(resource, **_kwargs): "image": public_image, "model": "Model", "hardware": "h200", - "framework": "trtllm", + "framework": "trt", "precision": "fp8", "spec_method": "mtp", "disagg": False, From 67520a1e6c4c9a569ddbf4df69baa308a5828e05 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:53:25 -0500 Subject: [PATCH 167/203] fix(launch): read a Slurm job killed by signal N as exit 128 plus N MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slurm reports ExitCode as status:signal, but job_status kept only the status, so a batch or script job killed by SIGKILL or cancelled with SIGTERM read as exit 0 and the launch exited 1. Those jobs now report 137 and 143, matching how the launcher maps its own children. 中文:Slurm 以 status:signal 的形式报告 ExitCode,而 job_status 只保留了 status, 因此被 SIGKILL 终止或被 SIGTERM 取消的 batch 或 script 作业会被读成退出码 0, 启动器最终以 1 退出。现在这些作业分别报告 137 和 143,与启动器映射自身子进程的方式一致。 --- inferencex-e2e/infx/launch/backends/slurm/cli.py | 13 ++++++++++--- inferencex-e2e/infx/tests/launch/test_slurm_cli.py | 4 +++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/infx/launch/backends/slurm/cli.py b/inferencex-e2e/infx/launch/backends/slurm/cli.py index 9e32cd7e7d..79eaf6f00b 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/cli.py +++ b/inferencex-e2e/infx/launch/backends/slurm/cli.py @@ -266,10 +266,17 @@ def observe(job: Job) -> JobStatus: def job_status(state: str, exit_code: str) -> JobStatus: - """Only ``COMPLETED`` with ``0:0`` succeeds; any other state after the job ran failed.""" - code, _, _ = exit_code.partition(":") + """Only ``COMPLETED`` with ``0:0`` succeeds; any other state after the job ran failed. + + Slurm's ``ExitCode`` is ``status:signal``, and a death by signal N reads as 128 + N. + """ + code, _, signal = exit_code.partition(":") raw = f"{state}|{exit_code}" - exit_status = int(code) if code.isdigit() else None + # Linux signals stop at 64. OUT_OF_MEMORY reports the marker 0:125, not a signal. + if signal.isdigit() and 0 < int(signal) <= 64: + exit_status: int | None = 128 + int(signal) + else: + exit_status = int(code) if code.isdigit() else None if state == "COMPLETED" and exit_code == "0:0": return JobStatus(JobState.SUCCEEDED, raw, 0) if not state: diff --git a/inferencex-e2e/infx/tests/launch/test_slurm_cli.py b/inferencex-e2e/infx/tests/launch/test_slurm_cli.py index 4cf2e25b8d..943875f3ab 100644 --- a/inferencex-e2e/infx/tests/launch/test_slurm_cli.py +++ b/inferencex-e2e/infx/tests/launch/test_slurm_cli.py @@ -80,7 +80,9 @@ def test_salloc_without_a_grant_fails(fake_bin, body): @pytest.mark.parametrize("record,state,exit_code,nodes", [ ("COMPLETED|0:0|gpu[01-02]", JobState.SUCCEEDED, 0, "gpu[01-02]"), ("FAILED|1:0|gpu03", JobState.FAILED, 1, "gpu03"), - ("COMPLETED|0:15", JobState.FAILED, 0, None), + ("COMPLETED|0:15", JobState.FAILED, 143, None), + ("FAILED|0:9|gpu03", JobState.FAILED, 137, "gpu03"), + ("OUT_OF_MEMORY|0:125|gpu03", JobState.FAILED, 0, "gpu03"), ("CANCELLED by 1001|0:0|None assigned", JobState.CANCELLED, 0, None), ]) def test_final_status_maps_allocation_accounting(fake_bin, tmp_path, record, state, exit_code, nodes): From 7333afdd4431a3982a1b9ba0932f809d6ea23715 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:07 -0500 Subject: [PATCH 168/203] fix(recipes): match qwen3.5 h100 agentic DRAM to runner memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The h100-dsxe runner memory change in 82fc21bd6 makes the matrix compute 1607 GB of CPU DRAM, so the 1731 GB recipes no longer bound and every DRAM point lost its fingerprint. Set TOTAL_CPU_DRAM_GB to 1607 and shrink the per-rank HiCache pools to 1607 / TP8 / 2 = 100 GB so they fit the smaller host budget. 中文:82fc21bd6 修改了 h100-dsxe 运行器内存,矩阵现计算出 1607 GB CPU DRAM,原 1731 GB 的配方无法绑定,所有 DRAM 点失去指纹。将 TOTAL_CPU_DRAM_GB 设为 1607,并将每个 rank 的 HiCache 池缩小为 1607 / TP8 / 2 = 100 GB 以适配更小的主机预算。 --- .../qwen3.5/sglang/h100-fp8-mtp/agentic.yaml | 6 +++--- .../srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml index 033f7562c3..9a480d8bcb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml @@ -47,18 +47,18 @@ override_gpu_kv: env: KV_OFFLOADING: none # HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one -# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +# Mamba host pool per rank: 1607 GB / TP8 / 2 pools = 100 GB each. override_hicache: roles: agg: args: page-size: 64 enable-hierarchical-cache: true - hicache-size: 108 + hicache-size: 100 hicache-io-backend: kernel hicache-mem-layout: page_first hicache-write-policy: write_through_selective benchmark: env: KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml index 0cffc53fec..1314bed3b5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml @@ -44,18 +44,18 @@ override_gpu_kv: env: KV_OFFLOADING: none # HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one -# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +# Mamba host pool per rank: 1607 GB / TP8 / 2 pools = 100 GB each. override_hicache: roles: agg: args: page-size: 64 enable-hierarchical-cache: true - hicache-size: 108 + hicache-size: 100 hicache-io-backend: kernel hicache-mem-layout: page_first hicache-write-policy: write_through_selective benchmark: env: KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' From bc7fb407e833f94ca1c0d2d89da65bd632fbb20e Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:07 -0500 Subject: [PATCH 169/203] fix(recipes): match minimaxm3 h100 agentic DRAM to runner memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The h100-dsxe runner memory change in 82fc21bd6 makes the matrix compute 1607 GB of CPU DRAM, so the conc 6 and conc 8 Mooncake points no longer bound. Set TOTAL_CPU_DRAM_GB to 1607 and recompute the per-rank global segment as (1496 GiB - 414) / 8 - 26 - 4 = 105 GB. 中文:82fc21bd6 修改了 h100-dsxe 运行器内存,矩阵现计算出 1607 GB CPU DRAM,conc 6 与 conc 8 的 Mooncake 点无法绑定。将 TOTAL_CPU_DRAM_GB 设为 1607,并将每个 rank 的 global segment 重新计算为 (1496 GiB - 414) / 8 - 26 - 4 = 105 GB。 --- .../minimaxm3/vllm/h100-fp8-mtp/agentic.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml index 95cd487ae7..89fc5d35ec 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -111,8 +111,8 @@ override_tp8_c5: KV_OFFLOADING: none # DRAM points offload KV to an embedded Mooncake store. Per rank: the host budget -# (1731 GB = 1612 GiB) less the 414 GiB checkpoint page cache, over TP8, less the -# 26 GiB CPU weight offload and the 4 GiB local buffer = 119 GB. +# (1607 GB = 1496 GiB) less the 414 GiB checkpoint page cache, over TP8, less the +# 26 GiB CPU weight offload and the 4 GiB local buffer = 105 GB. override_tp8_c6_dram: # The worker's Mooncake client and the mooncake-master both install the master config's release. setup_script: vllm-mooncake.sh @@ -131,7 +131,7 @@ override_tp8_c6_dram: store_config: mode: embedded metadata_server: P2PHANDSHAKE - global_segment_size: 119GB + global_segment_size: 105GB local_buffer_size: 4GB protocol: rdma device_name: '' @@ -151,7 +151,7 @@ override_tp8_c6_dram: env: CONC: '6' KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' override_tp8_c8_dram: # The worker's Mooncake client and the mooncake-master both install the master config's release. @@ -171,7 +171,7 @@ override_tp8_c8_dram: store_config: mode: embedded metadata_server: P2PHANDSHAKE - global_segment_size: 119GB + global_segment_size: 105GB local_buffer_size: 4GB protocol: rdma device_name: '' @@ -191,4 +191,4 @@ override_tp8_c8_dram: env: CONC: '8' KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' From 95e1bb40ff963f40b114d6c1484a54b24849f95f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:07 -0500 Subject: [PATCH 170/203] fix(recipes): match qwen3.8next gb300 agentic DRAM to runner memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gb300-nv runner memory change in 82fc21bd6 makes the matrix compute 241 GB of CPU DRAM per GPU, so the 225 GB recipe no longer bound at any concurrency. Set TOTAL_CPU_DRAM_GB to 241. The 224 GB HiCache budget still fits and is left unchanged. 中文:82fc21bd6 修改了 gb300-nv 运行器内存,矩阵现计算出每 GPU 241 GB CPU DRAM,原 225 GB 的配方在所有并发下都无法绑定。将 TOTAL_CPU_DRAM_GB 设为 241。224 GB 的 HiCache 预算仍然适用,保持不变。 --- .../qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml index 3c21ee1306..119a4e953e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml @@ -98,7 +98,7 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # Native single-node binding verifies these against the generated point. - TOTAL_CPU_DRAM_GB: '225' + TOTAL_CPU_DRAM_GB: '241' KV_OFFLOADING: dram # Only client concurrency changes; serving/admission/graph/cache settings stay fixed. From c078b4dd7802d80d258adbb62392b3a61563048d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:47:49 -0500 Subject: [PATCH 171/203] fix(ci-priority): give draft-model speculative decoding its bonus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec-decoding adjustment listed eagle and eagle3, but matrix validation only allows mtp, draft_model or none, so the 217 draft_model jobs never got the speculative-decoding bonus. Add draft_model at the same 0.1 as the other methods. The eagle keys stay because the run-sweep PR classifier enum still emits them. 中文:推测解码调整项列出了 eagle 和 eagle3,但矩阵校验只允许 mtp、draft_model 或 none,因此 217 个 draft_model 任务从未获得推测解码加分。现为 draft_model 添加与其他方法相同的 0.1 加分。保留 eagle 键,因为 run-sweep 的 PR 分类器枚举仍会输出它们。 --- inferencex-e2e/configs/ci-priority.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/configs/ci-priority.yaml b/inferencex-e2e/configs/ci-priority.yaml index 435bf2ebd4..83692e0dcb 100644 --- a/inferencex-e2e/configs/ci-priority.yaml +++ b/inferencex-e2e/configs/ci-priority.yaml @@ -19,6 +19,7 @@ adjustments: fp4: 0.1 spec-decoding: mtp: 0.1 + draft_model: 0.1 eagle: 0.1 eagle3: 0.1 framework-prefix: From 284a55d7eb852af175bc68d8e168be6da8f647fd Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:36 -0500 Subject: [PATCH 172/203] fix(recipes): drop binder-owned fields from llm-d AgentX recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dsv4 GB200 llm-d agg and disagg recipes set model, identity values and launcher client env that the binder writes, so every dsv4-fp4-gb200-llmd point failed fingerprinting and launch. They now declare empty identity.model/identity.container like their siblings and leave model and the client env to the binder. The bound image, identity, precision and client env match the old values, and model.path becomes the master hf: reference. 中文:dsv4 GB200 llm-d 的 agg 与 disagg 配方设置了由绑定器写入的 model、identity 字段以及启动器客户端环境变量,导致所有 dsv4-fp4-gb200-llmd 点位在指纹计算和启动时失败。现在它们与同类配方一样声明空的 identity.model/identity.container,并将 model 和客户端环境交给绑定器。绑定后的镜像、identity、精度和客户端环境与原值一致,model.path 改为主配置的 hf: 引用。 --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 13 ++----------- .../dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 13 ++----------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index e87b83bb98..27d8fb28ff 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -8,13 +8,9 @@ schema: 2 base: - model: - path: deepseek-v4-pro-0813 - container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 - precision: fp4 identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} - container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + model: {} + container: {} dynamo: install: false slurm: {time_limit: "8:00:00"} @@ -134,12 +130,7 @@ base: command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic PORT: "8000" - IS_MULTINODE: "true" - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index fd6c847285..a5f7b5dcee 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -10,13 +10,9 @@ schema: 2 base: - model: - path: deepseek-v4-pro-0813 - container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 - precision: fp4 identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} - container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + model: {} + container: {} dynamo: install: false slurm: {time_limit: "8:00:00"} @@ -248,12 +244,7 @@ base: command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic PORT: "8000" - IS_MULTINODE: "true" - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" From 6f9a6c235d18b6fb5f491d150445eeedc03ab8ad Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:47:58 -0500 Subject: [PATCH 173/203] fix(configs): drop dsr1 MI355X FP4 MTP disagg rows duplicated in the 8k1k-mtp entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dsr1-fp4-mi355x-sglang-disagg-mtp repeated the mtp3 1P1D TP8 and mtp1 2P1D DEP8 search-space rows of dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp, so full sweeps ran those seven points twice under the same result filenames. PR #1584 split this key into the 1k1k and 8k1k entries, and PR #1571 later restored the pre-split copy without a changelog entry. Remove the two repeated rows from the restored entry and keep its three distinct topologies. 中文:dsr1-fp4-mi355x-sglang-disagg-mtp 重复了 dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp 中 mtp3 1P1D TP8 和 mtp1 2P1D DEP8 两个搜索空间行,导致完整扫描以相同结果文件名重复运行这七个点。PR #1584 已将该键拆分为 1k1k 与 8k1k 条目,PR #1571 随后在没有变更日志条目的情况下恢复了拆分前的副本。从恢复的条目中删除这两行重复配置,并保留其三个独有拓扑。 --- inferencex-e2e/configs/amd-master.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 37346b42fe..d3ebba5b28 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1001,15 +1001,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - isl: 8192 osl: 1024 search-space: - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8] - srt-recipe: disagg-variants.yaml:override_mtp3_1p1d_tp8 - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [2, 4, 8, 16, 32] srt-recipe: disagg-variants.yaml:override_mtp3_1p2d_tp8_narrow @@ -1037,15 +1028,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: ep: 8 dp-attn: true decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - - spec-decoding: "mtp" - conc-list: [1024, 2048, 4096] - srt-recipe: disagg-variants.yaml:override_mtp1_2p1d_dep8 - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} minimaxm3-fp8-mi300x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.32.0@sha256:14f6a575e1fb0a580b0f633c09e06f648709d60405b85f6a7ff3f80d49f39905 From 74495740dac618dc6c37c22882915795cb3de94e Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:48:40 -0500 Subject: [PATCH 174/203] fix(configs): drop Qwen3.5 FP8 GB300 AgentX midcurve points duplicated from pareto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The midcurve entries from PR #3840 re-listed five points that the pareto entries from PR #3231 already run, with the same recipe overrides and therefore the same result filenames. Remove the c1000 and c200 PP4/DEP4 points from qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve and delete qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve, whose three points were all copies. The DEP16 MTP6 points stay in the midcurve entry and the pareto entries keep every repeated point and their eval selection. 中文:PR #3840 新增的 midcurve 条目重复列出了 PR #3231 的 pareto 条目已在运行的五个点,二者使用相同的配方覆盖项,因此结果文件名也相同。从 qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve 中删除 c1000 与 c200 的 PP4/DEP4 点,并删除三个点全部为副本的 qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve。DEP16 MTP6 点保留在 midcurve 条目中,pareto 条目保留所有重复点及其评测选择。 --- inferencex-e2e/configs/nvidia-master.yaml | 94 ----------------------- 1 file changed, 94 deletions(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 111c4a13c9..ae11cdc922 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9759,44 +9759,6 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: agentic-coding: - dram-utilization: 0.9 search-space: - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 1000 - srt-recipe: disagg-variants.yaml:override_5p3d_pp4_dep4_c1000 - prefill: - num-worker: 5 - tp: 1 - pp: 4 - ep: 1 - dp-attn: false - decode: - num-worker: 3 - tp: 4 - pp: 1 - ep: 4 - dp-attn: true - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 200 - srt-recipe: disagg-variants.yaml:override_1p1d_pp4_dep4_c200 - prefill: - num-worker: 1 - tp: 1 - pp: 4 - ep: 1 - dp-attn: false - decode: - num-worker: 1 - tp: 4 - pp: 1 - ep: 4 - dp-attn: true - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -9854,62 +9816,6 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: pp: 1 ep: 16 dp-attn: true -qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve: - image: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:gb300-nv - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: false - srt-recipe-dir: qwen3.5/sglang/gb300-fp8/agentx - scenarios: - agentic-coding: - - dram-utilization: 0.9 - search-space: - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 44 - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c44_b1 - worker: - num-worker: 1 - tp: 4 - pp: 1 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 1 - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c1_b1 - worker: - num-worker: 1 - tp: 4 - pp: 1 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 1 - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8_c1_b1 - worker: - num-worker: 1 - tp: 8 - pp: 1 - ep: 1 - dp-attn: false # GLM-5.2 NVFP4 AgentX on B300 behind Dynamo (KV router) + SGLang: one # aggregated TP4 engine on 4 of the node's 8 GPUs per point, so tok/s/GPU is From a31ce395ef2dad888be0b90d56ddc5a7f9c40cb7 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:47:26 -0500 Subject: [PATCH 175/203] docs(configs): match CONFIGS.md master example to the validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The master-config example omitted the required multinode field, showed kv-p2p-transfer on a single-node search-space entry, and used an agentic trace-source key that validation rejects as unknown. It now shows multinode, disagg and dram-utilization, and says model-prefix and precision feed launcher policy rows rather than benchmark script selection. 中文:主配置示例缺少必填的 multinode 字段,在单节点 search-space 条目中使用了 kv-p2p-transfer,并使用了会被校验拒绝的 agentic trace-source 字段。现示例改为包含 multinode、disagg 和 dram-utilization,并说明 model-prefix 与 precision 用于匹配启动器策略行,而非选择基准脚本。 --- inferencex-e2e/configs/CONFIGS.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index da1be8d4d5..9a1aaca4cd 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -12,9 +12,11 @@ entry-name: runner: string precision: string framework: string + multinode: bool + disagg: bool # optional, multinode only # Optional defaults for every search-space entry in this config. router: { name: string, version: string } - kv-p2p-transfer: string + kv-p2p-transfer: string # multinode only # Recipe directory under benchmarks/_node/srt-slurm-recipes/. srt-recipe-dir: string scenarios: @@ -25,16 +27,15 @@ entry-name: - { tp: int, conc-start: int, conc-end: int, srt-recipe: string } # Optionally, specify pipeline/expert/data-attention/context parallelism. - { tp: int, pp: int, ep: int, dp-attn: bool, dcp-size: int, pcp-size: int, conc-start: int, conc-end: int } - # Optionally, declare router metadata and the P2P KV transfer engine. + # Optionally, declare router metadata. - tp: int router: { name: string, version: string } - kv-p2p-transfer: string conc-start: int conc-end: int - ... - ... agentic-coding: # optional - - trace-source: string + - dram-utilization: float # required by kv-offloading: dram search-space: - tp: int kv-offloading: dram @@ -82,13 +83,14 @@ The below list describes what each field is: - `image`: The image used to serve the benchmark, e.g., `vllm/vllm-openai:v0.10.2` - `model`: The model to serve, e.g., `deepseek-ai/DeepSeek-R1-0528` -- `model-prefix`: The canonical InferenceX model prefix reference, i.e., `dsr1` for DeepSeek-R1 or `qwen3.5` for Qwen3.5. Consult `docs/MODELS.md` for supported model/scenario combinations. This value is used to decipher which script in `benchmarks/` should be used in order to launch the benchmark. +- `model-prefix`: The canonical InferenceX model prefix reference, i.e., `dsr1` for DeepSeek-R1 or `qwen3.5` for Qwen3.5. Consult `docs/MODELS.md` for supported model/scenario combinations. The launcher's workload policy rows in [`infx/launch/policy.py`](../infx/launch/policy.py) match on this value. - `runner`: This is the runner label on which to run the benchmark. This must be a valid key under `labels` in `runners.yaml`. Agentic configs must use an exact `cluster:` runner label, not a broad SKU or capacity label, so every search-space point runs on the same hardware fleet. -- `precision`: The precision to run the benchmark. Again, this is used to find which script to run in `benchmarks/`. +- `precision`: The precision to run the benchmark. Launcher workload policy rows may also match on it. - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. +- `multinode`: Required. `true` selects the multinode schema with `prefill`/`decode` or `worker` blocks, and `false` the single-node schema with `tp` entries. - `disagg`: Enables disaggregated serving and may only be `true` when `multinode` is also `true`. - `hardware`: Optional metadata within each `prefill` and `decode` worker block @@ -115,9 +117,9 @@ The below list describes what each field is: - (Optional) `pcp-size`: Prefill context-parallel size. Default is 1. A topology consumes `tp * pp * pcp-size` GPUs per worker. DCP does not add GPUs. - For single-node entries, set `pp`, `dcp-size`, and `pcp-size` directly in the search-space entry. - For multinode entries, set them independently inside each `prefill` and `decode` worker block. A worker pool allocates `num-worker * tp * pp * pcp-size` GPUs. - - `agentic-coding`: Agentic trace replay benchmarks using real conversation traces. Each entry must have: - - `trace-source`: Identifier for the trace dataset to use. - - `search-space`: Same structure as `fixed-seq-len` search-space entries. + - `agentic-coding`: Agentic trace replay benchmarks using real conversation traces. Each entry has: + - (Optional) `dram-utilization`: The fraction of each node's available CPU DRAM that DRAM KV offloading may use, greater than 0 and at most 1. It is required when any search-space entry sets `kv-offloading: dram`. + - `search-space`: Same structure as `fixed-seq-len` search-space entries, plus `kv-offloading` (`none` or `dram`, required for single-node entries) and the optional `kv-offload-backend`. `router` and `kv-p2p-transfer` may be omitted independently. Router metadata requires non-empty `name` and `version` strings. Its `version` must be the From 3cf53bca8b78484d5ab10462552e4b9dbd27caac Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:47:46 -0500 Subject: [PATCH 176/203] docs(recipes): list the atom engine and drop the fixed-seq-len workload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recipe tree has atom engine directories (for example dsv4/atom and the MI355X ATOM recipes) that the engine list omitted. No fixed-seq-len workload directory exists any more, so the guide no longer tells authors to keep one. 中文:配方目录中已有 atom 引擎目录(例如 dsv4/atom 与 MI355X ATOM 配方),但引擎列表未列出。现已不存在 fixed-seq-len 工作负载目录,因此指南不再要求保留该目录。 --- .../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index e17a07545b..2cf40a553a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -16,8 +16,8 @@ glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml ``` -- Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, and `tilert`; frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. -- Workloads are `1k1k`, `8k1k`, or `agentx`. Existing bundles spanning several fixed sequence lengths use `fixed-seq-len`; keep their override selectors intact. +- Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, `atom`, and `tilert`. Frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. +- Workloads are `1k1k`, `8k1k`, or `agentx`. - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. - A master-config entry with two or more rows of one recipe design keeps them in one `[-]-variants.yaml`; the axis only tells bundles in one directory apart. `base` holds everything the points share, with `schema: 2` beside it, and each row gets one plain `override__c` block (or a similarly point-identifying name, without dates or hardware) holding only its deltas. An override's `null` deletes the base key, so a null that a point needs, such as TensorRT-LLM's `cuda_graph_config:`, stays in `base`. Rows select it with `srt-recipe: .yaml:override_`, relative to `srt-recipe-dir`. Do not add `zip_override_*` blocks or share a bundle across entries, and keep one override per row: recipe references participate in eval grouping. An entry that reruns another entry's points selects that entry's overrides instead of copying them. Standalone files remain for single-row entries, adapted upstream recipes whose `# Source:` base materially differs, and points too different to share a base, such as a TP low-latency recipe beside wide-EP ones. From bb3008f199aba1b670758f15cec28c566ef01270 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:48:55 -0500 Subject: [PATCH 177/203] docs(models): state which A/B baselines active configs still carry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pending A/B retirement list named glm5.2 non-MTP and minimaxm3 non-EAGLE3 baselines, but every active glm5.2 and minimaxm3 search-space point declares spec decoding. The policy now names the models whose active configs still contain non-spec-decode points and notes that glm5.2 and minimaxm3 have none. 中文:待退役的 A/B 基线列表列出了 glm5.2 非 MTP 与 minimaxm3 非 EAGLE3 基线,但当前所有 glm5.2 与 minimaxm3 的 search-space 点均声明了投机解码。现政策改为列出当前配置中仍含非投机解码点的模型,并注明 glm5.2 与 minimaxm3 已无此类配置。 --- inferencex-e2e/docs/MODELS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index c293cbc359..1cb32dda0d 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -10,7 +10,7 @@ Following the principles of fairness and equitableness, the service-level object InferenceX-e2e runs on a fixed GPU pool with a small team, so scenarios, precisions, and recipe variants that no longer earn their cluster hours are retired. The support matrix below is the current record of what is active and retired; git history and `perf-changelog.yaml` hold past settings. -- **Pending A/B baseline retirement (not yet enacted):** standalone non-spec-decode baselines kept only for comparison with speculative decoding are slated for retirement: `dsv4`, `qwen3.5`, and `glm5.2` non-MTP; `minimaxm3` non-EAGLE3; `kimik3` non-DSpark. Retire one only once replacement coverage exists for the affected model, hardware, and engine. A spec-decode counterpart existing is not by itself a reason to remove a config. +- **Pending A/B baseline retirement (not yet enacted):** standalone non-spec-decode baselines kept only for comparison with speculative decoding are slated for retirement. This covers `dsv4` and `qwen3.5` non-MTP and `kimik3` non-DSpark, whose active configs still contain non-spec-decode points. `glm5.2` and `minimaxm3` have no active non-spec-decode configs. Retire one only once replacement coverage exists for the affected model, hardware, and engine. A spec-decode counterpart existing is not by itself a reason to remove a config. - **No separate A/B tracks for new models:** [`infx/golden_al_distribution/`](../infx/golden_al_distribution) pins speculative AgentX runs to one measured acceptance-length curve, so spec-decode results are comparable on their own. New models, including Kimi-K3, do not need a non-spec-decode baseline from day 0. - **Publish the best Pareto points, with or without speculative decoding.** Recipes may disable MTP, EAGLE/EAGLE3, DSpark, or another draft method when that gives a better operating point. Valid non-spec-decode results stay eligible under the [North-star Pareto policy](#north-star-pareto-policy), so a frontier may mix both. From c003ce39622c4dd854bdc1038c915868a2cf6921 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:50:30 -0500 Subject: [PATCH 178/203] docs(runner-setup): point at the scripts under inferencex-e2e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The setup and start commands ran utils/runner_setup from the clone root, but the scripts live under inferencex-e2e/utils/runner_setup. The guide also cited unpadded gb200-nv_0 runner names that configs/runners.yaml no longer lists. 中文:设置与启动命令从克隆根目录运行 utils/runner_setup,但脚本实际位于 inferencex-e2e/utils/runner_setup。指南还引用了未补零的 gb200-nv_0 运行器名称,而 configs/runners.yaml 中已不再包含此类名称。 --- inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md b/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md index 1a1f7d9205..73409e40c5 100644 --- a/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md +++ b/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md @@ -92,7 +92,7 @@ Required permissions (all of these endpoints require **admin access to the repos 4. Configure the runners: ```bash - ./InferenceX/utils/runner_setup/setup.sh \ + ./InferenceX/inferencex-e2e/utils/runner_setup/setup.sh \ \ \ \ @@ -104,7 +104,7 @@ Required permissions (all of these endpoints require **admin access to the repos For example, configure 18 runners (`b300-dsxe_00` … `b300-dsxe_17`) on the B300 DSXE cluster: ```bash - ./InferenceX/utils/runner_setup/setup.sh \ + ./InferenceX/inferencex-e2e/utils/runner_setup/setup.sh \ AOPHAHI... \ https://github.com/actions/runner/releases/download/v2.335.1/actions-runner-linux-x64-2.335.1.tar.gz \ 0 17 \ @@ -119,7 +119,7 @@ Required permissions (all of these endpoints require **admin access to the repos 5. Start the runners: ```bash - ./InferenceX/utils/runner_setup/start_runners.sh 0 17 ~/gharunners + ./InferenceX/inferencex-e2e/utils/runner_setup/start_runners.sh 0 17 ~/gharunners ``` This (re)creates a tmux session (default name: `github-actions`) with one tiled pane @@ -151,9 +151,7 @@ key off that name: runner names listed under each `labels` entry in [`configs/runners.yaml`](../../configs/runners.yaml). New runners do **not** receive sweep jobs until they are added there, and the entries must match the - registered names exactly, including zero-padding. Some older fleets predate the - padded convention, such as `gb200-nv_0`. Because `setup.sh` always zero-pads, new - entries should use the padded form. + registered names exactly, including zero-padding. `setup.sh` always zero-pads. ## Labels / `ADDITIONAL_RUNNER_TAGS` From 024182e938e0f502c78290f0610ec94cda10b4e3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:49:26 -0500 Subject: [PATCH 179/203] fix(agentx): drop the unread trace idle gap cap from Kimi-K3 recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine Kimi-K3 AgentX recipes set AIPERF_TRACE_IDLE_GAP_CAP_SECONDS, which nothing has read since infx.bench replaced benchmark_lib.sh. infx.bench.agentic never passes --trace-idle-gap-cap-seconds and agentx-harness 1.0.12 has no such environment setting, so the agentx scenario's 300 s default applies. That equals the recipes' value, so the replay is unchanged and the setting only misled readers. 中文:九个 Kimi-K3 AgentX 配方设置了 AIPERF_TRACE_IDLE_GAP_CAP_SECONDS,但自从 infx.bench 取代 benchmark_lib.sh 后已无任何代码读取它。infx.bench.agentic 从不传递 --trace-idle-gap-cap-seconds,agentx-harness 1.0.12 也没有对应的环境变量设置,因此实际生效的是 agentx 场景默认的 300 秒。该默认值与配方中的值相同,回放行为不变,这个设置只会误导读者。 --- .../kimik3/vllm/b200-fp4/agentx/agg-variants.yaml | 1 - .../gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml | 1 - .../vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml | 1 - .../kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml | 1 - .../kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml | 1 - .../kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml | 1 - .../kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml | 4 ---- .../kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml | 4 ---- .../vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml | 4 ---- 9 files changed, 18 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml index d00b45710c..b41e608430 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml @@ -109,7 +109,6 @@ base: benchmark: env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml index d300a0d8d8..fd9a30d641 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -141,7 +141,6 @@ benchmark: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml index 4f48b45148..e2e2d65f20 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -137,7 +137,6 @@ benchmark: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml index cfa1500d5e..925dd6aae7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml @@ -111,7 +111,6 @@ base: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml index 92137c0e88..d35b8a2478 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -106,7 +106,6 @@ base: container-remap-root: "" benchmark: env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml index 37760ff43f..2a6d5f8f83 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -170,7 +170,6 @@ base: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml index 73d3456bed..d34c126929 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml @@ -77,7 +77,3 @@ srun_options: telemetry: collector_join_timeout_seconds: 12 - -benchmark: - env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml index c4c2b380db..ae474b2072 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -78,7 +78,3 @@ srun_options: telemetry: collector_join_timeout_seconds: 12 - -benchmark: - env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml index 017ca3c7a8..cf5c57b9bd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -79,7 +79,3 @@ srun_options: telemetry: collector_join_timeout_seconds: 12 - -benchmark: - env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" From 3eb50e9861dd809fc9c274f9c7ed4ef9b72548b7 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:49:55 -0500 Subject: [PATCH 180/203] fix(agentx): drop the unread live-assistant switch from GLM-5.2 and MiniMax-M3 TRT recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GLM-5.2 GB300 and MiniMax-M3 GB200 TRT-LLM AgentX recipes set AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES to 0. infx.bench never reads it and agentx-harness 1.0.12 defines no such dataset setting. Its Weka trace loader always replays the recorded assistant turns and discards live responses, which is what 0 selected, so the replay is unchanged. 中文:GLM-5.2 GB300 与 MiniMax-M3 GB200 的 TRT-LLM AgentX 配方把 AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES 设为 0。infx.bench 从不读取它,agentx-harness 1.0.12 也没有定义这一数据集设置。其 Weka 轨迹加载器始终回放记录的助手回复并丢弃实时回复,这正是 0 所选择的行为,因此回放不变。 --- .../glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml | 1 - .../minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index dc1fda9cd1..c152524ace 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -188,7 +188,6 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" SERVED_MODEL_NAME: GLM-5.2-NVFP4 AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml index 4457b20714..4b6ba59dd7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -109,7 +109,6 @@ base: benchmark: env: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' From 5600108a2f8652359b560bce3ef48a2be4654ef1 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:50:16 -0500 Subject: [PATCH 181/203] fix(agentx): drop the unread warmup override from the DSv4.1-Flash ATOM recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MI355X ATOM AgentX recipe sets AIPERF_WARMUP_REQUESTS_PER_LANE to 5, but infx.bench only passes --warmup-requests-per-lane in fast mode and agentx-harness 1.0.12 reads no such environment setting. Every run of this srt-slurm recipe has therefore used the agentx scenario's 10 warmup requests per lane. Removing the line keeps that behavior and stops the recipe from claiming 5. 中文:MI355X ATOM AgentX 配方将 AIPERF_WARMUP_REQUESTS_PER_LANE 设为 5,但 infx.bench 只在快速模式下传递 --warmup-requests-per-lane,agentx-harness 1.0.12 也不读取这一环境变量。因此该 srt-slurm 配方的每次运行实际都使用 agentx 场景默认的每通道 10 个预热请求。删除这一行保持该行为不变,并且配方不再声称使用 5。 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 1b6d9bb4d7..881ce3cb68 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -45,8 +45,6 @@ base: AITER_LOG_LEVEL: WARNING benchmark: env: - # The upstream one-hour profile warms five requests per lane. - AIPERF_WARMUP_REQUESTS_PER_LANE: '5' # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' From c733445a2a175e139b723b8db7d985dca317ade3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:51:35 -0500 Subject: [PATCH 182/203] fix(bench): remove the unread INFERENCEX_RUNTIME_ENV_VARS forwarding list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runtime_settings.sh still exported INFERENCEX_RUNTIME_ENV_VARS as the list of settings to forward across container boundaries, but its last reader went away with the standalone llm-d launcher. Jobs inherit every workflow export through sbatch and srun --export=ALL, and the srt post-eval container receives only names matching WORKLOAD_ENV in infx/launch/drivers/srt/submit.py. A name added only to the list was silently not forwarded, so drop the list and point at the real contract. 中文:runtime_settings.sh 仍导出 INFERENCEX_RUNTIME_ENV_VARS,作为跨容器边界转发的设置列表,但它的最后一个读取方已随独立的 llm-d 启动器一起移除。作业通过 sbatch 和 srun --export=ALL 继承工作流的全部导出变量,而 srt 的 post-eval 容器只接收与 infx/launch/drivers/srt/submit.py 中 WORKLOAD_ENV 匹配的名称。只加入该列表的名称会被静默忽略而不转发,因此删除该列表,并注明真正的转发约定。 --- inferencex-e2e/benchmarks/runtime_settings.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/runtime_settings.sh b/inferencex-e2e/benchmarks/runtime_settings.sh index 9b8f0790e3..c674e646ae 100644 --- a/inferencex-e2e/benchmarks/runtime_settings.sh +++ b/inferencex-e2e/benchmarks/runtime_settings.sh @@ -2,6 +2,7 @@ # Workflow-owned common settings, loaded before recipe-specific overrides. # Benchmark receivers validate these inputs; they do not choose missing values. +# Jobs inherit these exports. Post-eval gets only WORKLOAD_ENV names (infx/launch/drivers/srt/submit.py). export OPENAI_API_KEY='EMPTY' export EVAL_ENDPOINT_READY_TIMEOUT_SECONDS='1800' export EVAL_MODEL_STABILIZATION_SECONDS='30' @@ -12,6 +13,3 @@ export ENABLE_AGENTX_POWER='1' export VLLM_ENGINE_READY_TIMEOUT_S='3600' export SGLANG_TORCH_PROFILER_DIR='/workspace' export VLLM_TORCH_PROFILER_DIR='/workspace' - -# Explicitly forward these settings across container boundaries. -export INFERENCEX_RUNTIME_ENV_VARS="OPENAI_API_KEY EVAL_ENDPOINT_READY_TIMEOUT_SECONDS EVAL_MODEL_STABILIZATION_SECONDS AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS AIPERF_EXPERIMENTAL_FAST ENABLE_AGENTX_POWER REQUIRE_POWER VLLM_ENGINE_READY_TIMEOUT_S SGLANG_TORCH_PROFILER_DIR VLLM_TORCH_PROFILER_DIR" From f8193338723c22c94a3e4cbe3d4f3daf860913cc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:58:11 -0500 Subject: [PATCH 183/203] chore(recipes): delete 1k1k recipes no master config references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1k1k scenario was retired, and no master config, recipe, test, workflow or infx code references the 35 multi-node 1k1k recipes for dsr1, minimaxm3 and qwen3.5. Removing them leaves full-sweep output and every bound recipe generated for those models byte-identical. The recipe guide no longer lists 1k1k as a workload. 中文:1k1k 场景已退役,dsr1、minimaxm3 与 qwen3.5 的 35 个多节点 1k1k 配方未被任何主配置、配方、测试、工作流或 infx 代码引用。删除后,full-sweep 输出以及这些模型生成的所有绑定配方均逐字节一致。配方指南不再将 1k1k 列为工作负载。 --- .../multi_node/srt-slurm-recipes/RECIPES.md | 2 +- ...dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml | 128 ------------- ...4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml | 128 ------------- ...p4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml | 123 ------------- ...dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml | 123 ------------- .../1k1k/disagg-1p1d-dep2-tep8-stp.yaml | 85 --------- .../1k1k/disagg-1p1d-dep2-tp4-stp.yaml | 86 --------- .../1k1k/disagg-1p2d-dep2-dep4-stp.yaml | 87 --------- .../1k1k/disagg-2p1d-dep2-dep8-stp.yaml | 87 --------- .../1k1k/disagg-2p1d-dep2-tep8-stp.yaml | 85 --------- .../1k1k/disagg-2p2d-dep2-tep8-stp.yaml | 85 --------- .../1k1k/disagg-3p2d-dep2-tep8-stp.yaml | 85 --------- .../1k1k/disagg-1p1d-dep2-tep8-stp.yaml | 83 --------- .../1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml | 85 --------- .../1k1k/disagg-1p2d-dep2-dep4-stp.yaml | 85 --------- .../1k1k/disagg-2p1d-dep2-dep8-stp.yaml | 85 --------- .../1k1k/disagg-2p1d-dep2-tep8-stp.yaml | 83 --------- .../1k1k/disagg-2p2d-dep2-tep8-stp.yaml | 83 --------- .../1k1k/disagg-3p2d-dep2-tep8-stp.yaml | 83 --------- .../1k1k/disagg-1p1d-dep4-dep8-stp.yaml | 108 ----------- .../1k1k/disagg-1p1d-dep4-tep8-stp.yaml | 106 ----------- .../1k1k/disagg-1p2d-dep4-dep8-stp.yaml | 108 ----------- .../1k1k/disagg-1p2d-dep4-tep8-stp.yaml | 106 ----------- .../1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml | 107 ----------- .../1k1k/disagg-1p1d-dep2-dep4-stp.yaml | 108 ----------- .../1k1k/disagg-1p2d-dep2-tep8-stp.yaml | 106 ----------- .../1k1k/disagg-2p2d-dep2-tep8-stp.yaml | 106 ----------- .../1k1k/disagg-2p3d-dep2-dep4-stp.yaml | 108 ----------- .../1k1k/disagg-2p4d-dep2-dep4-stp.yaml | 108 ----------- .../1k1k/disagg-4p2d-dep2-dep8-stp.yaml | 108 ----------- .../1k1k/disagg-1p1d-dep4-dep16-stp.yaml | 169 ------------------ .../1k1k/disagg-1p1d-tp4-tp4-stp.yaml | 137 -------------- .../1k1k/disagg-2p1d-dep4-dep16-stp.yaml | 169 ------------------ .../1k1k/disagg-1p1d-dep4-dep16-stp.yaml | 168 ----------------- .../1k1k/disagg-1p1d-tp4-tp4-stp.yaml | 137 -------------- .../1k1k/disagg-2p1d-dep4-dep16-stp.yaml | 168 ----------------- 36 files changed, 1 insertion(+), 3817 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 2cf40a553a..cdfde32bd4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -17,7 +17,7 @@ qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml ``` - Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, `atom`, and `tilert`. Frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. -- Workloads are `1k1k`, `8k1k`, or `agentx`. +- Workloads are `8k1k` or `agentx`. - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. - A master-config entry with two or more rows of one recipe design keeps them in one `[-]-variants.yaml`; the axis only tells bundles in one directory apart. `base` holds everything the points share, with `schema: 2` beside it, and each row gets one plain `override__c` block (or a similarly point-identifying name, without dates or hardware) holding only its deltas. An override's `null` deletes the base key, so a null that a point needs, such as TensorRT-LLM's `cuda_graph_config:`, stays in `base`. Rows select it with `srt-recipe: .yaml:override_`, relative to `srt-recipe-dir`. Do not add `zip_override_*` blocks or share a bundle across entries, and keep one override per row: recipe references participate in eval grouping. An entry that reruns another entry's points selects that entry's overrides instead of copying them. Standalone files remain for single-row entries, adapted upstream recipes whose `# Source:` base materially differs, and points too different to share a base, such as a TP low-latency recipe beside wide-EP ones. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml deleted file mode 100644 index 3aa145b921..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ - -schema: 2 -name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - install: true - - source: - rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" -resources: - gpu_type: b200 - gpus_per_node: 8 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 1024 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - nodes: 1 - workers: 1 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.75 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: 512x1024 - use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml deleted file mode 100644 index b5f701dc8d..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ - -schema: 2 -name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-2d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - install: true - - source: - rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" -resources: - gpu_type: b200 - gpus_per_node: 8 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - nodes: 2 - workers: 2 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - cuda-graph-max-bs: 512 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: '512' - use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml deleted file mode 100644 index c7997e3eb0..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml +++ /dev/null @@ -1,123 +0,0 @@ - -schema: 2 -name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - install: true - - source: - rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" -resources: - gpu_type: b200 - gpus_per_node: 8 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - nodes: 5 - workers: 5 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - cuda-graph-max-bs: 512 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: 16x512 - use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml deleted file mode 100644 index 07d65033b3..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml +++ /dev/null @@ -1,123 +0,0 @@ - -schema: 2 -name: b200-fp4-mtp-low-latency-dep4-1p-tep8-6d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - install: true - - source: - rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" -resources: - gpu_type: b200 - gpus_per_node: 8 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - nodes: 6 - workers: 6 - env: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - args: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - cuda-graph-max-bs: 512 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: 32x64x256x512 - use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml deleted file mode 100644 index d596da9d23..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml deleted file mode 100644 index ea386d18c8..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml +++ /dev/null @@ -1,86 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tp4-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - allow_prefill_decode_colocation: true - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml deleted file mode 100644 index 11748f090e..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-1p2d-fp4-dep2-dep4-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 2 - gpus: 4 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml deleted file mode 100644 index 77ced205d1..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml +++ /dev/null @@ -1,87 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-dep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "512x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml deleted file mode 100644 index e19cb3c317..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml deleted file mode 100644 index da521ba2e4..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 2 - workers: 2 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml deleted file mode 100644 index b05f3d0c94..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-3p2d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 3 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 2 - workers: 2 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml deleted file mode 100644 index b1bbc5f8c5..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,83 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml deleted file mode 100644 index ec4d9fd6e2..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - allow_prefill_decode_colocation: true - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 4 - enable-expert-parallel: false - moe-backend: marlin - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml deleted file mode 100644 index 03aa32c375..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-dep4-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 2 - gpus: 4 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml deleted file mode 100644 index 75325490a5..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml +++ /dev/null @@ -1,85 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-dep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "512x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml deleted file mode 100644 index f2be29e938..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,83 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml deleted file mode 100644 index 67f04f0885..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,83 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 2 - workers: 2 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml deleted file mode 100644 index a7964b126f..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,83 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 3 - gpus: 2 - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 2 - workers: 2 - gpus: 8 - - env: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - args: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml deleted file mode 100644 index da862ce192..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml deleted file mode 100644 index da88369694..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml +++ /dev/null @@ -1,106 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 2 - workers: 1 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "128x256" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml deleted file mode 100644 index 4543d2546d..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 2 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml deleted file mode 100644 index 9f95d023c5..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml +++ /dev/null @@ -1,106 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 2 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml deleted file mode 100644 index d8f0916520..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml +++ /dev/null @@ -1,107 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp4-marlin-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 4 - gpus: 4 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - moe-backend: marlin - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml deleted file mode 100644 index 1276ea6467..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep4-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 1 - workers: 1 - gpus: 4 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml deleted file mode 100644 index fe58b26912..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,106 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 2 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 2 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x256" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml deleted file mode 100644 index c7741f7671..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml +++ /dev/null @@ -1,106 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 2 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml deleted file mode 100644 index 1179174bfd..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p3d-dep2-dep4-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 3 - workers: 3 - gpus: 4 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml deleted file mode 100644 index d0166478c4..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-dep4-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 1 - workers: 2 - gpus: 2 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 4 - gpus: 4 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml deleted file mode 100644 index 025a10e564..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml +++ /dev/null @@ -1,108 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb300-4p2d-dep2-dep8-fp8-1k1k" - - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - source: - pypi: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -frontend: - type: dynamo - enable_multiple_frontends: false - -engine: - type: vllm - connector: - -roles: - prefill: - nodes: 2 - workers: 4 - gpus: 2 - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - nodes: 4 - workers: 2 - gpus: 8 - - env: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - args: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml deleted file mode 100644 index 7fe8236123..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml +++ /dev/null @@ -1,169 +0,0 @@ -schema: 2 -name: "qwen3.5-1p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -dynamo: - install: true - - source: - rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - nodes: 4 - workers: 1 - - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 4096 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "512x1024x2048" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml deleted file mode 100644 index 5a80ed2d86..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml +++ /dev/null @@ -1,137 +0,0 @@ -schema: 2 -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - install: true - - source: - rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - nodes: 1 - workers: 1 - - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 10 - num_warmup_mult: 1 - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml deleted file mode 100644 index 0911998150..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml +++ /dev/null @@ -1,169 +0,0 @@ -schema: 2 -name: "qwen3.5-2p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -dynamo: - install: true - - source: - rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -engine: sglang -roles: - prefill: - nodes: 2 - workers: 2 - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - nodes: 4 - workers: 1 - - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 8192 - mem-fraction-static: 0.75 - max-mamba-cache-size: 4096 - max-running-requests: 4096 - cuda-graph-max-bs: 256 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "4096" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml deleted file mode 100644 index ab59f2818e..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml +++ /dev/null @@ -1,168 +0,0 @@ -schema: 2 -name: "qwen3.5-1p1d-dep4-dep16" - - -dynamo: - install: true - - source: - rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - nodes: 4 - workers: 1 - - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 4096 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "512x1024x2048" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml deleted file mode 100644 index 9afce77209..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml +++ /dev/null @@ -1,137 +0,0 @@ -schema: 2 -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - install: true - - source: - rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - nodes: 1 - workers: 1 - - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 10 - num_warmup_mult: 1 - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml deleted file mode 100644 index 06ca1cf9fd..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml +++ /dev/null @@ -1,168 +0,0 @@ -schema: 2 -name: "qwen3.5-2p1d-dep4-dep16" - - -dynamo: - install: true - - source: - rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 -engine: sglang -roles: - prefill: - nodes: 2 - workers: 2 - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - nodes: 4 - workers: 1 - - env: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - args: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 8192 - mem-fraction-static: 0.75 - max-mamba-cache-size: 4096 - max-running-requests: 4096 - cuda-graph-max-bs: 256 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "4096" - -telemetry: - enabled: true - collect_interval_ms: 1000 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" From d8dc468086d5dbd6b6338c77af7094c5a8cf5cf4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:10:29 -0500 Subject: [PATCH 184/203] revert: restore the unreferenced 1k1k recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recipe deletions do not belong in this PR. This restores the 35 recipe files and the RECIPES.md workload list that eae0cc91a removed. 中文:删除配方不属于本 PR 的范围。此提交恢复 eae0cc91a 删除的 35 个配方文件以及 RECIPES.md 中的工作负载列表。 --- .../multi_node/srt-slurm-recipes/RECIPES.md | 2 +- ...dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml | 128 +++++++++++++ ...4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml | 128 +++++++++++++ ...p4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml | 123 +++++++++++++ ...dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml | 123 +++++++++++++ .../1k1k/disagg-1p1d-dep2-tep8-stp.yaml | 85 +++++++++ .../1k1k/disagg-1p1d-dep2-tp4-stp.yaml | 86 +++++++++ .../1k1k/disagg-1p2d-dep2-dep4-stp.yaml | 87 +++++++++ .../1k1k/disagg-2p1d-dep2-dep8-stp.yaml | 87 +++++++++ .../1k1k/disagg-2p1d-dep2-tep8-stp.yaml | 85 +++++++++ .../1k1k/disagg-2p2d-dep2-tep8-stp.yaml | 85 +++++++++ .../1k1k/disagg-3p2d-dep2-tep8-stp.yaml | 85 +++++++++ .../1k1k/disagg-1p1d-dep2-tep8-stp.yaml | 83 +++++++++ .../1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml | 85 +++++++++ .../1k1k/disagg-1p2d-dep2-dep4-stp.yaml | 85 +++++++++ .../1k1k/disagg-2p1d-dep2-dep8-stp.yaml | 85 +++++++++ .../1k1k/disagg-2p1d-dep2-tep8-stp.yaml | 83 +++++++++ .../1k1k/disagg-2p2d-dep2-tep8-stp.yaml | 83 +++++++++ .../1k1k/disagg-3p2d-dep2-tep8-stp.yaml | 83 +++++++++ .../1k1k/disagg-1p1d-dep4-dep8-stp.yaml | 108 +++++++++++ .../1k1k/disagg-1p1d-dep4-tep8-stp.yaml | 106 +++++++++++ .../1k1k/disagg-1p2d-dep4-dep8-stp.yaml | 108 +++++++++++ .../1k1k/disagg-1p2d-dep4-tep8-stp.yaml | 106 +++++++++++ .../1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml | 107 +++++++++++ .../1k1k/disagg-1p1d-dep2-dep4-stp.yaml | 108 +++++++++++ .../1k1k/disagg-1p2d-dep2-tep8-stp.yaml | 106 +++++++++++ .../1k1k/disagg-2p2d-dep2-tep8-stp.yaml | 106 +++++++++++ .../1k1k/disagg-2p3d-dep2-dep4-stp.yaml | 108 +++++++++++ .../1k1k/disagg-2p4d-dep2-dep4-stp.yaml | 108 +++++++++++ .../1k1k/disagg-4p2d-dep2-dep8-stp.yaml | 108 +++++++++++ .../1k1k/disagg-1p1d-dep4-dep16-stp.yaml | 169 ++++++++++++++++++ .../1k1k/disagg-1p1d-tp4-tp4-stp.yaml | 137 ++++++++++++++ .../1k1k/disagg-2p1d-dep4-dep16-stp.yaml | 169 ++++++++++++++++++ .../1k1k/disagg-1p1d-dep4-dep16-stp.yaml | 168 +++++++++++++++++ .../1k1k/disagg-1p1d-tp4-tp4-stp.yaml | 137 ++++++++++++++ .../1k1k/disagg-2p1d-dep4-dep16-stp.yaml | 168 +++++++++++++++++ 36 files changed, 3817 insertions(+), 1 deletion(-) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index cdfde32bd4..2cf40a553a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -17,7 +17,7 @@ qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml ``` - Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, `atom`, and `tilert`. Frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. -- Workloads are `8k1k` or `agentx`. +- Workloads are `1k1k`, `8k1k`, or `agentx`. - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. - A master-config entry with two or more rows of one recipe design keeps them in one `[-]-variants.yaml`; the axis only tells bundles in one directory apart. `base` holds everything the points share, with `schema: 2` beside it, and each row gets one plain `override__c` block (or a similarly point-identifying name, without dates or hardware) holding only its deltas. An override's `null` deletes the base key, so a null that a point needs, such as TensorRT-LLM's `cuda_graph_config:`, stays in `base`. Rows select it with `srt-recipe: .yaml:override_`, relative to `srt-recipe-dir`. Do not add `zip_override_*` blocks or share a bundle across entries, and keep one override per row: recipe references participate in eval grouping. An entry that reruns another entry's points selects that entry's overrides instead of copying them. Standalone files remain for single-row entries, adapted upstream recipes whose `# Source:` base materially differs, and points too different to share a base, such as a TP low-latency recipe beside wide-EP ones. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml new file mode 100644 index 0000000000..3aa145b921 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml @@ -0,0 +1,128 @@ + +schema: 2 +name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 1024 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.75 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 512x1024 + use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml new file mode 100644 index 0000000000..b5f701dc8d --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml @@ -0,0 +1,128 @@ + +schema: 2 +name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-2d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 2 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: '512' + use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml new file mode 100644 index 0000000000..c7997e3eb0 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml @@ -0,0 +1,123 @@ + +schema: 2 +name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 5 + workers: 5 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 16x512 + use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml new file mode 100644 index 0000000000..07d65033b3 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml @@ -0,0 +1,123 @@ + +schema: 2 +name: b200-fp4-mtp-low-latency-dep4-1p-tep8-6d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 32x64x256x512 + use_chat_template: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..d596da9d23 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64x128x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml new file mode 100644 index 0000000000..ea386d18c8 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml @@ -0,0 +1,86 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tp4-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x4x8x16" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..11748f090e --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p2d-fp4-dep2-dep4-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..77ced205d1 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-dep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "512x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..e19cb3c317 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..da521ba2e4 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..b05f3d0c94 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-3p2d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..b1bbc5f8c5 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64x128x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml new file mode 100644 index 0000000000..ec4d9fd6e2 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 4 + enable-expert-parallel: false + moe-backend: marlin + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x4x8x16" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..03aa32c375 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-dep4-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..75325490a5 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-dep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "512x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..f2be29e938 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..67f04f0885 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..a7964b126f --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..da862ce192 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml new file mode 100644 index 0000000000..da88369694 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "128x256" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..4543d2546d --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml new file mode 100644 index 0000000000..9f95d023c5 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml new file mode 100644 index 0000000000..d8f0916520 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml @@ -0,0 +1,107 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp4-marlin-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 4 + moe-backend: marlin + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..1276ea6467 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep4-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..fe58b26912 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64x128x256" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..c7741f7671 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..1179174bfd --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p3d-dep2-dep4-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 3 + workers: 3 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..d0166478c4 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-dep4-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..025a10e564 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-4p2d-dep2-dep8-fp8-1k1k" + + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024x4096" + req_rate: "inf" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..7fe8236123 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "qwen3.5-1p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 4096 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "512x1024x2048" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..5a80ed2d86 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 1 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..0911998150 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "qwen3.5-2p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 8192 + mem-fraction-static: 0.75 + max-mamba-cache-size: 4096 + max-running-requests: 4096 + cuda-graph-max-bs: 256 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "4096" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..ab59f2818e --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,168 @@ +schema: 2 +name: "qwen3.5-1p1d-dep4-dep16" + + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 4096 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "512x1024x2048" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..9afce77209 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 1 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..06ca1cf9fd --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml @@ -0,0 +1,168 @@ +schema: 2 +name: "qwen3.5-2p1d-dep4-dep16" + + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 8192 + mem-fraction-static: 0.75 + max-mamba-cache-size: 4096 + max-running-requests: 4096 + cuda-graph-max-bs: 256 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "4096" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + command: "dcgm-exporter --collect-interval=100 --address :{port} -f /configs/dcgm-counters-noprof.csv" From 90fbf7d790b6f869d3c7d75df76495ce3f63df5d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:14:37 -0500 Subject: [PATCH 185/203] fix(ci): offer draft_model instead of eagle in PR priority classifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run-sweep classifier schema and prompt offered eagle/eagle3, but generated jobs only carry spec-decoding mtp, draft_model or none, so the draft_model bonus in ci-priority.yaml could never be selected. Replace eagle/eagle3 with draft_model and drop the dead eagle/eagle3 policy keys. 中文:PR 优先级分类器的 schema 与提示词提供 eagle/eagle3,但作业的 spec-decoding 仅为 mtp、draft_model 或 none,导致 draft_model 加分永远无法命中。现将其替换为 draft_model,并删除 ci-priority.yaml 中无用的 eagle/eagle3 键。 --- .github/workflows/run-sweep.yml | 4 ++-- inferencex-e2e/configs/ci-priority.yaml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 96ae8973a3..f8caac9179 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -245,7 +245,7 @@ jobs: --effort low --max-turns 8 --allowedTools "Read,Glob,Grep,Bash(git diff:*)" - --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","musespark","dsv41flash","kimik3","glm5.2","dsv4","minimaxm3","qwen3.5","qwen3.8next","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' + --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","draft_model","sglang","vllm","dynamo-sglang","dynamo-vllm","musespark","dsv41flash","kimik3","glm5.2","dsv4","minimaxm3","qwen3.5","qwen3.8next","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' prompt: | Inspect this pull request's diff against its base branch. @@ -255,7 +255,7 @@ jobs: - agentic: agentic-coding or AgentX workload - eval-only: evaluation without throughput measurement - fp4: FP4 precision - - mtp, eagle, eagle3: speculative decoding method + - mtp, draft_model: speculative decoding method - sglang, vllm, dynamo-vllm: runtime framework - model criterion: matching configured model family - checklist-complete: PR checklist is satisfied diff --git a/inferencex-e2e/configs/ci-priority.yaml b/inferencex-e2e/configs/ci-priority.yaml index 83692e0dcb..d986316bd3 100644 --- a/inferencex-e2e/configs/ci-priority.yaml +++ b/inferencex-e2e/configs/ci-priority.yaml @@ -20,8 +20,6 @@ adjustments: spec-decoding: mtp: 0.1 draft_model: 0.1 - eagle: 0.1 - eagle3: 0.1 framework-prefix: sglang: 1.5 vllm: 1.5 From 1d6ef0bb2fddc34fe1ffcce2dc65510297d8fee0 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:14:37 -0500 Subject: [PATCH 186/203] docs(ci): fix full-sweep examples and all-evals/conc wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README full-sweep examples used runner types b200 and b200-trt, which fail with "Invalid runner type"; they now use cluster:b200-nscale and cluster:h200-dgxc. The --all-evals and --min-conc/--max-conc descriptions in the README and e2e-tests.yml inputs now match generate.py --help. 中文:README 中 full-sweep 示例使用了无效的 runner 类型 b200 与 b200-trt,现改为 cluster:b200-nscale 与 cluster:h200-dgxc。README 与 e2e-tests.yml 中 --all-evals 及并发上下限的描述已与 generate.py --help 保持一致。 --- .github/workflows/README.md | 10 +++++----- .github/workflows/e2e-tests.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0ef456826a..cfe8867fe3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -50,9 +50,9 @@ usage: python -m infx.matrix.generate full-sweep If neither `--single-node` nor `--multi-node` is specified, both types are generated. -By default, throughput runs for every generated config and eval-only jobs run for the selected 8k1k subset and the AgentX GSM8K subset. `--no-evals` disables eval jobs, `--evals-only` emits only that selected subset, and adding `--all-evals` expands it to every fixed-sequence config. `--all-evals` alone is an equivalent eval-only shorthand, but it cannot be combined with `--no-evals`. +By default, throughput runs for every generated config and eval-only jobs run for the selected 8k1k subset and the AgentX GSM8K subset. `--no-evals` disables eval jobs, `--evals-only` emits only that selected subset, and adding `--all-evals` expands it to every eligible 8k1k fixed-sequence and agentic config. `--all-evals` always emits only those eval-only jobs, with or without `--evals-only`, but it cannot be combined with `--no-evals`. -`--step-size` must be greater than 1 and applies to concurrency ranges. Explicit `conc-list` values are emitted directly and are filtered by `--min-conc` / `--max-conc` when provided. When both bounds are set, `--min-conc` must not exceed `--max-conc`. +`--step-size` must be greater than 1 and applies to concurrency ranges. `--min-conc` / `--max-conc` drop concurrencies outside the bounds, including explicit `conc-list` values. A single-node fixed-sequence `conc-start`/`conc-end` range that crosses a bound is clamped to it, and a fixed-sequence config with no concurrency left under `--max-conc` runs at the cap. When both bounds are set, `--min-conc` must not exceed `--max-conc`. `--trim-conc` (the `trim-conc` input of `e2e-tests.yml`) keeps only the minimum concurrency of every generated single- and multi-node deployment shape, after eval selection, for a lowest-concurrency smoke run; in changelog-ref mode only throughput rows are trimmed, and evals keep their selected concurrency. No PR label enables trimming. @@ -65,7 +65,7 @@ full-sweep --config-files configs/nvidia-master.yaml **Test all single-node dsr1 configurations on B200 with 8k1k sequence lengths:** ``` -full-sweep --single-node --model-prefix dsr1 --runner-type b200 --seq-lens 8k1k --config-files configs/nvidia-master.yaml +full-sweep --single-node --model-prefix dsr1 --runner-type cluster:b200-nscale --seq-lens 8k1k --config-files configs/nvidia-master.yaml ``` **Test all single-node fp8 precision configs for 8k1k workloads:** @@ -75,12 +75,12 @@ full-sweep --single-node --precision fp8 --seq-lens 8k1k --config-files configs/ **Test all single-node TRT configs on H200 runners:** ``` -full-sweep --single-node --framework trt --runner-type h200 b200-trt --config-files configs/nvidia-master.yaml +full-sweep --single-node --framework trt --runner-type h200 cluster:h200-dgxc --config-files configs/nvidia-master.yaml ``` **Test specific single-node model on specific hardware with specific sequence lengths:** ``` -full-sweep --single-node --model-prefix dsr1 --runner-type b200 --precision fp4 --framework sglang --seq-lens 8k1k --config-files configs/nvidia-master.yaml +full-sweep --single-node --model-prefix dsr1 --runner-type cluster:b200-nscale --precision fp4 --framework sglang --seq-lens 8k1k --config-files configs/nvidia-master.yaml ``` **Limit concurrency and parallelism for faster testing:** diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index d45c1b9962..891e8e74e9 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -77,7 +77,7 @@ on: # zizmor: ignore[concurrency-limits] type: boolean default: false all-evals: - description: "Expand eval selection to every fixed-sequence config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" + description: "Expand eval selection to every eligible 8k1k fixed-sequence and agentic config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" required: false type: boolean default: false @@ -165,7 +165,7 @@ on: # zizmor: ignore[concurrency-limits] type: boolean default: false all-evals: - description: "Expand eval selection to every fixed-sequence config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" + description: "Expand eval selection to every eligible 8k1k fixed-sequence and agentic config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" required: false type: boolean default: false From d6685981c3d491db077f9a6c07635ed8d03d8c27 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:15:37 -0500 Subject: [PATCH 187/203] ci: drop workflow env vars that no reader consumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KEEP_LOGS, MODEL_DOWNLOAD_LOCK_TIMEOUT, DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE and SCENARIO_SUBDIR were read only by the bash launchers and legacy single-node scripts. Those readers were removed in #3460, #3576, #3618 and #3553, and nothing in infx, benchmarks, recipes or the srt-slurm submodule reads them under any name. Remove the settings from the benchmark, multi-node, profile and speedbench workflows. 中文:删除 benchmark、multi-node、profile 与 speedbench 工作流中已无任何读取方的环境变量。这些变量原本只由已删除的 bash launcher 和旧单节点脚本读取(#3460、#3576、#3618、#3553),infx、benchmarks、配方和 srt-slurm 子模块中均无同名或改名的读取方。 --- .github/workflows/benchmark-multinode-tmpl.yml | 4 ---- .github/workflows/benchmark-tmpl.yml | 4 ---- .github/workflows/profile.yml | 4 ---- .github/workflows/speedbench-al.yml | 4 ---- 4 files changed, 16 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 6fbadb13c9..8f5d86f7b6 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -114,11 +114,8 @@ env: INFERENCEX_E2E_ROOT: ${{ github.workspace }} PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' SALLOC_TIME_LIMIT: '480' - KEEP_LOGS: '0' IS_MULTINODE: 'true' RANDOM_RANGE_RATIO: 0.8 # Day-zero models resolved via hf: ids download from the Hub inside the @@ -155,7 +152,6 @@ env: POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} - SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} CONC: ${{ inputs.conc }} DURATION: ${{ inputs.duration }} diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index abe72ae7f2..ef25ec69ed 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -95,11 +95,8 @@ env: INFERENCEX_E2E_ROOT: ${{ github.workspace }} PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' SALLOC_TIME_LIMIT: '480' - KEEP_LOGS: '0' IS_MULTINODE: 'false' REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }} RANDOM_RANGE_RATIO: 0.8 @@ -132,7 +129,6 @@ env: EVAL_SUITE: ${{ inputs.eval-suite }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} - SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} KV_OFFLOADING: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offloading) || '' }} KV_OFFLOAD_BACKEND: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offload-backend.name) || '' }} diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index ce9feb53bb..929975765f 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -36,16 +36,12 @@ env: INFERENCEX_E2E_ROOT: ${{ github.workspace }} PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' SALLOC_TIME_LIMIT: '480' - KEEP_LOGS: '0' EVAL_ONLY: 'false' RUN_EVAL: 'false' IS_AGENTIC: '0' IS_MULTINODE: 'false' - SCENARIO_SUBDIR: fixed_seq_len/ HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} HF_HUB_CACHE: '/mnt/hf_hub_cache/' RANDOM_RANGE_RATIO: '0.8' diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index fa61b3e23a..b84462ff29 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -86,15 +86,11 @@ env: DRAFT_SAMPLE_METHOD: greedy PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' - KEEP_LOGS: '0' IS_MULTINODE: 'false' EVAL_ONLY: 'false' RUN_EVAL: 'false' IS_AGENTIC: '0' - SCENARIO_SUBDIR: fixed_seq_len/ HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} HF_HUB_CACHE: '/mnt/hf_hub_cache/' MODEL: ${{ inputs.model }} From e7e8f49449c9e3c94e0087f9f34bdf86626dea8d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:15:05 -0500 Subject: [PATCH 188/203] fix(speedbench-al): let dispatch select variant collectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The framework input could only resolve _fp4_b300_.sh, so the Kimi K3 probabilistic-draft collector was unreachable. A new collector-variant input appends a suffix to that name. A step after checkout resolves the script at the measured ref and fails with the list of available collectors when it does not exist. Empty variant keeps every existing dispatch on the same script. 中文:framework 输入只能解析出 _fp4_b300_.sh,因此 Kimi K3 概率草稿收集脚本无法被选中。新增 collector-variant 输入,为脚本名追加后缀。检出后的新步骤在被测 ref 上解析脚本,若不存在则列出可用脚本并失败。变体为空时,现有调度仍选择同一脚本。 --- .github/workflows/speedbench-al.yml | 26 +++++++++++++++++-- .../infx/golden_al_distribution/README.md | 2 +- .../infx/golden_al_distribution/README_zh.md | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index b84462ff29..c04da088f5 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -26,11 +26,16 @@ on: # zizmor: ignore[concurrency-limits] type: string default: 'dsv4' framework: - description: "Serving framework; selects the _fp4_b300_.sh collector" + description: "Serving framework; selects the _fp4_b300_[_].sh collector" required: false type: choice options: [vllm, sglang] default: 'vllm' + collector-variant: + description: "Collector variant suffix; non-empty selects _fp4_b300__.sh (e.g. probabilistic_sample_method_block_rejection_sample_method)" + required: false + type: string + default: '' image: # vLLM collectors need vLLM >= 0.30: thinking-on cells rely on --chat-template-kwargs (vllm-project/vllm#44244). description: "Container image (vLLM >= 0.30 for vLLM collectors)" @@ -103,7 +108,6 @@ env: EP_SIZE: '1' DP_ATTENTION: 'false' SPEC_DECODING: mtp - BENCH_SCRIPT_OVERRIDE: benchmarks/single_node/speedbench/${{ inputs.model-prefix }}_fp4_b300_${{ inputs.framework }}.sh SALLOC_TIME_LIMIT: ${{ inputs.salloc-time }} # Matrix-collector tunables (propagated into the container via srun --export=ALL). MTP_LIST: ${{ inputs.mtp-list }} @@ -240,6 +244,24 @@ jobs: rm -f speedbench-reference-al.yaml || true rm -rf speed_bench_data || true + - name: Resolve AL collector + env: + COLLECTOR_VARIANT: ${{ inputs.collector-variant }} + run: | + if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi + collector_dir=benchmarks/single_node/speedbench + collector="${MODEL_PREFIX}_fp4_b300_${FRAMEWORK}" + if [[ -n "$COLLECTOR_VARIANT" ]]; then + collector+="_${COLLECTOR_VARIANT}" + fi + # The name must stay a plain file in the collector directory. + if [[ "$collector" == */* || "$collector" == *$'\n'* || ! -f "$collector_dir/$collector.sh" ]]; then + echo "::error::No SPEED-Bench collector $collector_dir/$collector.sh at this ref. Available collectors are listed below." >&2 + ls "$collector_dir" >&2 + exit 1 + fi + echo "BENCH_SCRIPT_OVERRIDE=$collector_dir/$collector.sh" >> "$GITHUB_ENV" + - name: Checkout launch tooling uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/inferencex-e2e/infx/golden_al_distribution/README.md b/inferencex-e2e/infx/golden_al_distribution/README.md index ed38aede28..bd39e5175c 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README.md +++ b/inferencex-e2e/infx/golden_al_distribution/README.md @@ -86,7 +86,7 @@ This policy follows the same broad principle as MLPerf Inference: prescribe the The push-button [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.yml) workflow, introduced in [InferenceX#1650](https://github.com/SemiAnalysisAI/InferenceX/pull/1650) and extended to additional MTP and EAGLE3 models in [InferenceX#1706](https://github.com/SemiAnalysisAI/InferenceX/pull/1706), performs the following process. It superseded the early manually assembled reference in [InferenceX#1592](https://github.com/SemiAnalysisAI/InferenceX/pull/1592), making the exact commands, logs, outputs, and generated YAML auditable from one run. 1. A maintainer dispatches the workflow with a model, model prefix, serving framework (`vllm` or `sglang`), image, draft lengths (normally 1–8), thinking modes, `category=coding`, and `output-len=4096`. -2. The workflow launches the model on a B300 runner and selects the matching `_fp4_b300_.sh` collector under [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench). +2. The workflow launches the model on a B300 runner and selects the matching `_fp4_b300_.sh` collector under [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench). A non-empty `collector-variant` input selects `_fp4_b300__.sh` instead. For example, `model-prefix=kimik3` with `collector-variant=probabilistic_sample_method_block_rejection_sample_method` runs the Kimi K3 probabilistic-draft collector. The run fails before launch if the selected script does not exist. 3. For every `(thinking mode, draft length)` cell, the collector starts a clean vLLM or SGLang server with real speculative decoding and the model's production sampling/chat-template settings. 4. A vLLM collector snapshots vLLM's cumulative accepted-token and verification-draft counters, runs every prompt in the SPEED-Bench Qualitative `coding` category through `vllm bench serve`, and snapshots the counters again. The SGLang collector runs the same prompts through `sglang.bench_serving` and records the `accept_length` it reports. 5. A vLLM collector computes the mean acceptance length as: diff --git a/inferencex-e2e/infx/golden_al_distribution/README_zh.md b/inferencex-e2e/infx/golden_al_distribution/README_zh.md index cf11c2d808..aa8967153c 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README_zh.md +++ b/inferencex-e2e/infx/golden_al_distribution/README_zh.md @@ -79,7 +79,7 @@ python -m atom.entrypoints.openai_server \ 一键触发的 [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.yml) 工作流最初由 [InferenceX#1650](https://github.com/SemiAnalysisAI/InferenceX/pull/1650) 引入,随后在 [InferenceX#1706](https://github.com/SemiAnalysisAI/InferenceX/pull/1706) 中扩展到更多 MTP 和 EAGLE3 模型。它取代了 [InferenceX#1592](https://github.com/SemiAnalysisAI/InferenceX/pull/1592) 中早期手工整理的参考值,使精确命令、日志、输出和生成的 YAML 都可以从同一次运行中审计。其流程如下: 1. 维护者触发工作流,指定模型、模型前缀、服务框架(`vllm` 或 `sglang`)、镜像、草稿长度(通常为 1–8)、思考模式、`category=coding` 和 `output-len=4096`。 -2. 工作流在 B300 runner 上启动模型,并选择 [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench) 下对应的 `_fp4_b300_.sh` 收集脚本。 +2. 工作流在 B300 runner 上启动模型,并选择 [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench) 下对应的 `_fp4_b300_.sh` 收集脚本。若 `collector-variant` 输入非空,则改为选择 `_fp4_b300__.sh`。例如 `model-prefix=kimik3` 加 `collector-variant=probabilistic_sample_method_block_rejection_sample_method` 会运行 Kimi K3 概率草稿收集脚本。所选脚本不存在时,工作流会在启动前失败。 3. 对每个“思考模式 × 草稿长度”组合,收集脚本使用真实投机解码以及该模型的生产采样和聊天模板设置,启动一个干净的 vLLM 或 SGLang 服务。 4. vLLM 收集脚本读取 vLLM 累计的已接受 token 和验证草稿计数器,通过 `vllm bench serve` 运行 SPEED-Bench Qualitative `coding` 类别中的全部提示词,然后再次读取计数器。SGLang 收集脚本通过 `sglang.bench_serving` 运行相同的提示词,并记录其报告的 `accept_length`。 5. vLLM 收集脚本按以下公式计算平均接受长度: From e7c52c90c027950b7dea73ee14c7f9f418e90c41 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:17:39 -0500 Subject: [PATCH 189/203] fix(speedbench-al): open the PR against the real golden curve file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The open-pr step copied the matrix to benchmarks/speedbench-reference-al.yaml, a file nothing reads. It now writes infx/golden_al_distribution/.yaml with the source run URL as the first line, as the README requires. The file is loaded through python -m infx.golden_al_distribution show before any git work, so an N/A cell or malformed matrix fails the step. The setup job rejects open-pr without a valid golden-curve name before the GPU run starts. 中文:open-pr 步骤原先把矩阵复制到无人读取的 benchmarks/speedbench-reference-al.yaml。现在它按 README 要求写入 infx/golden_al_distribution/.yaml,首行为源 run URL。在任何 git 操作前,先通过 python -m infx.golden_al_distribution show 加载文件,因此 N/A 格子或格式错误的矩阵会让步骤失败。setup 作业会在 GPU 运行前拒绝缺少有效 golden-curve 名称的 open-pr。 --- .github/workflows/speedbench-al.yml | 54 ++++++++++++++----- .../infx/golden_al_distribution/README.md | 2 +- .../infx/golden_al_distribution/README_zh.md | 2 +- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index c04da088f5..eb7b4c6088 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -2,9 +2,9 @@ name: SpeedBench AL Collection # Push-button (workflow_dispatch) collection of a SPEED-Bench acceptance-length # (AL) matrix: thinking_on/off x MTP levels, for the given model (defaults to -# DeepSeek-V4-Pro). Produces the golden reference consumed by the -# synthetic-acceptance framework and (optionally) opens a PR updating -# benchmarks/speedbench-reference-al.yaml. +# DeepSeek-V4-Pro). Produces the golden curve consumed by the +# synthetic-acceptance framework and (optionally) opens a PR writing it to +# infx/golden_al_distribution/.yaml. # Independent collection requests use priority-scheduled runner slots. on: # zizmor: ignore[concurrency-limits] @@ -73,10 +73,15 @@ on: # zizmor: ignore[concurrency-limits] type: string default: '480' open-pr: - description: "Open a PR updating benchmarks/speedbench-reference-al.yaml (default off: artifact-only, paste values in manually)" + description: "Open a PR writing the matrix to infx/golden_al_distribution/.yaml (default off: artifact-only)" required: false type: boolean default: false + golden-curve: + description: "Golden curve file stem that open-pr writes, e.g. dsv4_mtp or kimik3_dspark (required with open-pr)" + required: false + type: string + default: '' ref: description: "Git ref (branch/sha) to checkout" required: false @@ -127,6 +132,17 @@ jobs: priority: ${{ steps.score.outputs.priority }} queue-token: ${{ steps.score.outputs.queue-token }} steps: + # Fail before the GPU run if open-pr has no curve to write. Same name rule as + # infx.golden_al_distribution.curves.CURVE_NAME, which keeps the path in that directory. + - name: Validate golden curve target + if: ${{ inputs.open-pr }} + env: + GOLDEN_CURVE: ${{ inputs.golden-curve }} + run: | + if [[ ! "$GOLDEN_CURVE" =~ ^[a-z0-9_.-]+$ ]]; then + echo "::error::open-pr needs golden-curve, a curve file stem such as dsv4_mtp (got '$GOLDEN_CURVE')." >&2 + exit 1 + fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: { clean: true, persist-credentials: false } - name: Checkout the measured revision's launcher @@ -322,33 +338,43 @@ jobs: path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench-reference-al.yaml if-no-files-found: warn - - name: Open PR updating reference yaml + - name: Open PR updating golden AL curve if: ${{ inputs.open-pr && success() }} env: # Repository write credential for authorized reference-data collection. GH_TOKEN: ${{ secrets.SPEEDBENCH_PAT }} # zizmor: ignore[secrets-outside-env] + GOLDEN_CURVE: ${{ inputs.golden-curve }} run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi set -eo pipefail - # NOTE: the reference yaml is keyed by model at the top level. This - # overwrites it with the current model's matrix; when more than one - # model is collected, replace this cp with a per-model-key YAML merge. - cp speedbench-reference-al.yaml benchmarks/speedbench-reference-al.yaml + source benchmarks/check_env.sh + check_env_vars GOLDEN_CURVE INFERENCEX_LAUNCH_PYTHON BENCH_SCRIPT_OVERRIDE + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + curve_file="infx/golden_al_distribution/${GOLDEN_CURVE}.yaml" + # The README requires the source run URL as the first line of a committed curve. + # The whole file is replaced, so cells this run did not collect are dropped. + { + echo "# Source GitHub Actions run: ${run_url}" + cat speedbench-reference-al.yaml + } > "$curve_file" + # Load every cell through the golden curve CLI. A failed cell (N/A) or a malformed + # matrix stops here instead of reaching a PR. + PYTHONPATH="$PWD" "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.golden_al_distribution show "$GOLDEN_CURVE" BRANCH="speedbench-al/${MODEL_PREFIX}-auto-${GITHUB_RUN_ID}" git config user.name "github-actions" git config user.email "github-actions@github.com" git checkout -b "$BRANCH" - git add benchmarks/speedbench-reference-al.yaml + git add "$curve_file" if git diff --cached --quiet; then - echo "No change in reference yaml; skipping PR." + echo "No change in ${curve_file}; skipping PR." exit 0 fi - git commit -m "Update SpeedBench AL reference matrix for ${MODEL} (auto, run ${GITHUB_RUN_ID})" + git commit -m "Update golden AL curve ${GOLDEN_CURVE} for ${MODEL} (auto, run ${GITHUB_RUN_ID})" git -c credential.helper= -c 'credential.helper=!gh auth git-credential' push -u origin "$BRANCH" gh pr create \ - --title "Update SpeedBench AL reference matrix for ${MODEL_PREFIX} (auto)" \ - --body "Auto-generated by the SpeedBench AL Collection workflow (run ${GITHUB_RUN_ID}). Model: \`${MODEL}\`, category: \`${CATEGORY}\`, MTP: \`${MTP_LIST}\`, thinking: \`${THINKING_MODES}\`, output_len: \`${SPEEDBENCH_OUTPUT_LEN}\`. Please review the measured values before merging." \ + --title "Update golden AL curve ${GOLDEN_CURVE} (auto)" \ + --body "Auto-generated by the SpeedBench AL Collection workflow (${run_url}). Curve: \`${curve_file}\`, collector: \`${BENCH_SCRIPT_OVERRIDE}\`, model: \`${MODEL}\`, category: \`${CATEGORY}\`, MTP: \`${MTP_LIST}\`, thinking: \`${THINKING_MODES}\`, output_len: \`${SPEEDBENCH_OUTPUT_LEN}\`. Review the measured values against the checklist in infx/golden_al_distribution/README.md before merging. A new curve also needs its row in that README's curve table." \ --base main \ --head "$BRANCH" diff --git a/inferencex-e2e/infx/golden_al_distribution/README.md b/inferencex-e2e/infx/golden_al_distribution/README.md index bd39e5175c..dfdc080cf5 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README.md +++ b/inferencex-e2e/infx/golden_al_distribution/README.md @@ -98,7 +98,7 @@ The push-button [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.y The `1` is the target model's guaranteed verification token. Values are rounded to two decimal places. 6. The collector emits a YAML matrix. The workflow publishes it in the GitHub Actions step summary and uploads it as a `speedbench-reference-al-` artifact. 7. Server logs and detailed per-request results are retained so reviewers can confirm sensible output, correct thinking mode, and the absence of silent server or chat-template failures. -8. After review, the matrix is committed here with its exact sampling metadata and source Actions run URL. +8. After review, the matrix is committed here with its exact sampling metadata and source Actions run URL. With `open-pr=true` and `golden-curve=` (for example `dsv4_mtp`), the workflow writes `.yaml` here with the source run URL on its first line. It loads the file through `python -m infx.golden_al_distribution show` and then opens the PR. The whole file is replaced, so cells the run did not collect are dropped. The collection measures real head quality only to establish the golden curve. AgentX then uses that committed curve as the synthetic acceptance target for every comparable submission. diff --git a/inferencex-e2e/infx/golden_al_distribution/README_zh.md b/inferencex-e2e/infx/golden_al_distribution/README_zh.md index aa8967153c..e25bdd372c 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README_zh.md +++ b/inferencex-e2e/infx/golden_al_distribution/README_zh.md @@ -91,7 +91,7 @@ python -m atom.entrypoints.openai_server \ 其中 `1` 是目标模型保证生成的验证 token。结果四舍五入到小数点后两位。 6. 收集脚本生成 YAML 矩阵。工作流将其发布到 GitHub Actions step summary,并上传为 `speedbench-reference-al-` artifact。 7. 工作流保留服务日志和逐请求详细结果,以便审阅者确认输出合理、思考模式正确,并且没有静默的服务或聊天模板故障。 -8. 审阅完成后,将矩阵连同准确的采样元数据和源 Actions run URL 一并提交到本目录。 +8. 审阅完成后,将矩阵连同准确的采样元数据和源 Actions run URL 一并提交到本目录。设置 `open-pr=true` 和 `golden-curve=<曲线文件名>`(例如 `dsv4_mtp`)时,工作流会写入本目录下的 `<曲线文件名>.yaml`,首行为源 run URL,再通过 `python -m infx.golden_al_distribution show` 加载校验,然后开启 PR。它会替换整个文件,因此本次未收集的格子会被删除。 收集过程仅通过测量真实草稿头质量来建立黄金曲线。之后,AgentX 对所有可比较提交使用该已提交曲线作为合成接受目标。 From b9f2ad14bd4069381d1383990a02b378aa28abaa Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:18:23 -0500 Subject: [PATCH 190/203] docs(speedbench): point the DSv4 collector at its golden curve file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment named benchmarks/speedbench-reference-al.yaml, which no longer exists. The measured curve lives in infx/golden_al_distribution/dsv4_mtp.yaml. 中文:注释中引用的 benchmarks/speedbench-reference-al.yaml 已不存在。实测曲线位于 infx/golden_al_distribution/dsv4_mtp.yaml。 --- .../benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh b/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh index d01cbc2ad5..926a1735fb 100755 --- a/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh +++ b/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh @@ -25,7 +25,7 @@ SERVE_MODEL="${MODEL_PATH}" MODEL_KEY="$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')" CONCURRENCY="1" TEMPERATURE="1.0" -# MUST match the golden config: benchmarks/speedbench-reference-al.yaml was measured +# MUST match the golden config: infx/golden_al_distribution/dsv4_mtp.yaml was measured # with reasoning_effort=high. SPEEDBENCH_DIR="/workspace/speed_bench_data" From d0c9dbce902882f5c01a34a5fbc12abaaf62efd8 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:22:06 -0500 Subject: [PATCH 191/203] fix(ci): launch profile runs with the inputs single-node srt needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Profile runs failed at launch because the job never set SRT_RECIPE and its checkout skipped the srt-slurm submodule. The job now passes SRT_RECIPE, RECIPE_FINGERPRINT and an explicit REQUIRE_POWER, checks out submodules, and names results through infx.results.result_filename like benchmark-tmpl. Nothing in infx has read PROFILE or MOE_DEBUG_LOG since the infx.bench port, so the PROFILE env, the moe-debug input, the trace and MoE log uploads, the trace-storage push and the Perfetto link are gone. 中文:Profile 工作流从未设置 SRT_RECIPE,检出时也没有拉取 srt-slurm 子模块,因此每次运行都在启动阶段失败。现在作业会传入 SRT_RECIPE、RECIPE_FINGERPRINT 和显式的 REQUIRE_POWER,检出子模块,并像 benchmark-tmpl 一样通过 infx.results.result_filename 生成结果文件名。自 infx.bench 移植以来,infx 中没有任何代码读取 PROFILE 或 MOE_DEBUG_LOG,所以删除了 PROFILE 环境变量、moe-debug 输入、trace 与 MoE 日志上传、trace-storage 推送和 Perfetto 链接。 --- .github/workflows/profile.yml | 167 ++---------------- .../tests/workflows/test_launch_layout.py | 7 +- 2 files changed, 17 insertions(+), 157 deletions(-) diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 929975765f..449413ecc1 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -1,5 +1,7 @@ name: Profile +# Runs one single-node matrix point through infx.launch. The infx launch path +# has no profiler capture, so this workflow produces no trace. # Independent profiling requests use priority-scheduled runner slots. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: @@ -18,11 +20,6 @@ on: # zizmor: ignore[concurrency-limits] required: false type: string default: '64' - moe-debug: - description: "Enable MoE debug patch and log (MOE_DEBUG_LOG)" - required: false - type: boolean - default: false ref: description: "Ref (branch/sha) to checkout" required: false @@ -45,7 +42,7 @@ env: HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} HF_HUB_CACHE: '/mnt/hf_hub_cache/' RANDOM_RANGE_RATIO: '0.8' - PERFETTO_RELAY_URL: https://semianalysisai.github.io/InferenceX-trace-storage + REQUIRE_POWER: '0' PYTHONDONTWRITEBYTECODE: '1' PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache @@ -104,12 +101,10 @@ jobs: python -P -m infx.workflows.ci_priority \ --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ --event-name "${GITHUB_EVENT_NAME}" \ - --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}" \ - --criteria-json "${PRIORITY_CRITERIA}") + --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}") echo "raw=$CONFIG_JSON" >> "$GITHUB_OUTPUT" env: - PRIORITY_CRITERIA: ${{ inputs.moe-debug && '["patchwork"]' || '' }} INPUTS_CONFIG_FILE: ${{ inputs.config-file }} INPUTS_CONFIG_KEY: ${{ inputs.config-key }} INPUTS_CONC: ${{ inputs.conc }} @@ -206,6 +201,8 @@ jobs: ) }} env: EXP_NAME: ${{ matrix.config.exp-name }} + SRT_RECIPE: ${{ matrix.config.srt-recipe }} + RECIPE_FINGERPRINT: ${{ matrix.config.recipe-fingerprint }} MODEL: ${{ matrix.config.model }} MODEL_PREFIX: ${{ matrix.config.model-prefix }} ISL: ${{ matrix.config.isl }} @@ -223,8 +220,6 @@ jobs: CONC: ${{ matrix.config.conc }} SPEC_DECODING: ${{ matrix.config.spec-decoding }} DISAGG: ${{ matrix.config.disagg }} - MOE_DEBUG: '0' - MOE_DEBUG_LOG: ${{ (inputs.moe-debug) && '/workspace/moe_debug.tp0.log' || '' }} steps: - name: Slurm cleanup run: | @@ -249,6 +244,7 @@ jobs: fetch-depth: 0 ref: ${{ inputs.ref || github.sha }} clean: false + submodules: true persist-credentials: false - name: Wipe workspace after failed checkout @@ -309,59 +305,31 @@ jobs: run: | "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.launch cleanup - - name: Launch + Profile (single-node sglang/vllm) - id: run + - name: Launch job script env: RUNNER_NAME: ${{ runner.name }} - PROFILE: '1' - SGLANG_TORCH_PROFILER_DIR: /workspace/ - VLLM_TORCH_PROFILER_DIR: /workspace/ - VLLM_RPC_TIMEOUT: '1800000' + JOB_CHECK_RUN_ID: ${{ job.check_run_id }} + RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_a${{ github.run_attempt }}_${{ runner.name }} shell: bash run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi export GITHUB_WORKSPACE="$PWD" echo "INFERENCEX_E2E_ROOT=$PWD" >> "$GITHUB_ENV" set -eo pipefail + RESULT_FILENAME=$(python3 -m infx.results.result_filename) + export RESULT_FILENAME + echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE)) echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV" - ep_val="${EP_SIZE:-1}" - res_name="${EXP_NAME}_${PRECISION}_${FRAMEWORK}_tp${TP}-pp${PP_SIZE}-dcp${DCP_SIZE}-pcp${PCP_SIZE}_ep${ep_val}_dpa_${DP_ATTENTION}_conc${CONC}_${RUNNER_NAME}" - export RESULT_FILENAME="${res_name}" - echo "RESULT_FILENAME=${res_name}" >> "$GITHUB_ENV" source benchmarks/runtime_settings.sh "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run - if [ ! -f "${res_name}.json" ]; then - echo "Run failed: Benchmark result ${res_name}.json not found." >&2 + if [ ! -f "${RESULT_FILENAME}.json" ]; then + echo "Run failed: Benchmark result ${RESULT_FILENAME}.json not found." >&2 exit 1 fi - trace_path="profile_${res_name}.trace.json.gz" - if [ -f "$trace_path" ]; then - echo "trace=$PWD/$trace_path" >> "$GITHUB_OUTPUT" - if [ "${FRAMEWORK}" = "sglang" ]; then - # Try to locate corresponding TP-0 traces produced by SGLang profiler - merged_latest=$(ls -t profiles/merged-*.trace.json.gz 2>/dev/null | head -n1 || true) - if [ -n "${merged_latest}" ] && [ -f "${merged_latest}" ]; then - ts_name="${merged_latest##*/}" - ts_name="${ts_name#merged-}" - ts_name="${ts_name%.trace.json.gz}" - tp0_decode="profiles/${ts_name}-TP-0-DECODE.trace.json.gz" - tp0_extend="profiles/${ts_name}-TP-0-EXTEND.trace.json.gz" - if [ -f "${tp0_decode}" ]; then - echo "tp0_decode=$PWD/${tp0_decode}" >> "$GITHUB_OUTPUT" - fi - if [ -f "${tp0_extend}" ]; then - echo "tp0_extend=$PWD/${tp0_extend}" >> "$GITHUB_OUTPUT" - fi - fi - fi - else - echo "Profile trace not found: $trace_path" >&2 - fi - - name: Process result (json -> agg) env: RUNNER_TYPE: ${{ matrix.config.runner }} @@ -371,108 +339,3 @@ jobs: source "$PYTHONPATH/benchmarks/check_env.sh" check_env_vars INFERENCEX_RESULTS_PYTHON "$INFERENCEX_RESULTS_PYTHON" -P -m infx.results.fixed_sequence - - - name: Upload profile as artifact - if: ${{ steps.run.outputs.trace != '' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: profile_${{ env.RESULT_FILENAME }} - path: ${{ steps.run.outputs.trace }} - if-no-files-found: ignore - - - name: Upload TP-0-DECODE trace as artifact - if: ${{ steps.run.outputs.tp0_decode != '' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: profile_${{ env.RESULT_FILENAME }}_TP0_DECODE - path: ${{ steps.run.outputs.tp0_decode }} - if-no-files-found: ignore - - - name: Upload TP-0-EXTEND trace as artifact - if: ${{ steps.run.outputs.tp0_extend != '' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: profile_${{ env.RESULT_FILENAME }}_TP0_EXTEND - path: ${{ steps.run.outputs.tp0_extend }} - if-no-files-found: ignore - - - name: Upload MoE debug log as artifact - if: ${{ env.MOE_DEBUG == '1' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: moe_debug_${{ env.RESULT_FILENAME }} - path: ${{ env.INFERENCEX_E2E_ROOT }}/moe_debug.tp0.log - if-no-files-found: ignore - - - name: Checkout storage repo - id: checkout-storage - if: ${{ steps.run.outputs.trace != '' }} - continue-on-error: true - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: &storage-inputs - repository: SemiAnalysisAI/InferenceX-trace-storage - path: storage - ref: master - # Storage-repository deploy key for authorized profiling dispatches. - ssh-key: ${{ secrets.PROFILER_STORAGE_DEPLOY_KEY }} # zizmor: ignore[secrets-outside-env] - fetch-depth: 0 - # This separate storage checkout pushes trace commits using its scoped deploy key. - persist-credentials: true # zizmor: ignore[artipacked] - - - name: Wipe storage repo after failed checkout - if: ${{ steps.checkout-storage.outcome == 'failure' }} - run: rm -rf "$GITHUB_WORKSPACE/storage" - - - name: Retry storage repo checkout - if: ${{ steps.checkout-storage.outcome == 'failure' }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: *storage-inputs # zizmor: ignore[artipacked] - - - name: Push profile to storage repo - if: ${{ steps.run.outputs.trace != '' }} - id: push - env: - MATRIX_CONFIG_RUNNER: ${{ matrix.config.runner }} - MATRIX_CONFIG_EXP_NAME: ${{ matrix.config['exp-name'] }} - MATRIX_EP: ${{ matrix.config.ep || 1 }} - TRACE_LOCAL: ${{ steps.run.outputs.trace }} - shell: bash - run: | - set -eo pipefail - - dest_dir="storage/profiles/${GITHUB_SHA}/${MATRIX_CONFIG_RUNNER}/${FRAMEWORK}/${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}" - mkdir -p "$dest_dir" - cp "$TRACE_LOCAL" "$dest_dir/trace.json.gz" - - pushd storage >/dev/null - git config user.name "github-actions" - git config user.email "github-actions@github.com" - git add -A - git commit -m "Add profile: ${GITHUB_SHA} ${MATRIX_CONFIG_EXP_NAME} tp${TP} pp${PP_SIZE} dcp${DCP_SIZE} pcp${PCP_SIZE} ep${MATRIX_EP} conc${CONC}" || echo "Nothing to commit" - git push - STORAGE_SHA="$(git rev-parse HEAD)" - popd >/dev/null - - export RAW_URL="https://raw.githubusercontent.com/SemiAnalysisAI/InferenceX-trace-storage/${STORAGE_SHA}/profiles/${GITHUB_SHA}/${MATRIX_CONFIG_RUNNER}/${FRAMEWORK}/${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}/trace.json.gz" - export TITLE="${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}" - - enc_src="$(python3 -c 'import os,urllib.parse; print(urllib.parse.quote(os.environ["RAW_URL"], safe=""))')" - enc_title="$(python3 -c 'import os,urllib.parse; print(urllib.parse.quote(os.environ["TITLE"], safe=""))')" - - relay="${PERFETTO_RELAY_URL%/}" - RELAY_URL="${relay}/?src=${enc_src}&title=${enc_title}" - - echo "raw_url=$RAW_URL" >> "$GITHUB_OUTPUT" - echo "relay_url=$RELAY_URL" >> "$GITHUB_OUTPUT" - - - name: Print Perfetto link (relay) - if: ${{ steps.push.outputs.relay_url != '' }} - env: - RELAY_URL: ${{ steps.push.outputs.relay_url }} - RAW_URL: ${{ steps.push.outputs.raw_url }} - shell: bash - run: | - set -eo pipefail - echo "RAW trace URL: $RAW_URL" - echo "Perfetto Relay URL: $RELAY_URL" - printf "\n**Perfetto (Relay):** %s\n" "$RELAY_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/inferencex-e2e/infx/tests/workflows/test_launch_layout.py b/inferencex-e2e/infx/tests/workflows/test_launch_layout.py index 1e3ab04092..47f16969b2 100644 --- a/inferencex-e2e/infx/tests/workflows/test_launch_layout.py +++ b/inferencex-e2e/infx/tests/workflows/test_launch_layout.py @@ -27,7 +27,6 @@ result = os.environ["RESULT_FILENAME"] Path(result + ".json").write_text("{}\\n") Path(result + "_conc1.json").write_text('{"num_requests_successful": 1}\\n') -Path("profile_" + result + ".trace.json.gz").write_bytes(b"fixture trace") Path("speedbench-reference-al.yaml").write_text("fixture: 1\\n") """ @@ -72,7 +71,7 @@ def workflow_step(workflow: str, step_name: str) -> dict: [ ("benchmark-tmpl.yml", "Launch job script"), ("benchmark-multinode-tmpl.yml", "Launch multi-node job script"), - ("profile.yml", "Launch + Profile (single-node sglang/vllm)"), + ("profile.yml", "Launch job script"), ("speedbench-al.yml", "Collect AL matrix"), ], ) @@ -133,6 +132,4 @@ def test_launch_runs_the_python_entrypoint_from_the_project_root(tmp_path, workf assert "fixture: 1" in summary.read_text() else: assert (project / (exported["RESULT_FILENAME"] + ".json")).read_text() == "{}\n" - if workflow == "profile.yml": - output = dict(line.split("=", 1) for line in github_output.read_text().splitlines()) - assert Path(output["trace"]).read_bytes() == b"fixture trace" + From 7a8a279a637e14f97a26741ab6c02e698b2832d2 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:30:18 -0500 Subject: [PATCH 192/203] docs(infx): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give every function in the top-level infx modules and infx/datasets a docstring describing its inputs and result, and fold the technical comments into those docstrings. 中文:为 infx 顶层模块和 infx/datasets 中的每个函数添加说明输入与结果的文档字符串,并将技术性注释并入这些文档字符串。 --- inferencex-e2e/infx/cli.py | 1 + inferencex-e2e/infx/config.py | 1 + .../infx/datasets/build_weka_hf_dataset.py | 14 +++++++- .../datasets/plot_subagent_distributions.py | 9 ++++- .../infx/datasets/plot_weka_distributions.py | 9 ++++- inferencex-e2e/infx/datasets/proxy_to_weka.py | 9 +++++ .../infx/datasets/sample_proxy_traces.py | 19 +++++++--- inferencex-e2e/infx/fastcopy.py | 1 + inferencex-e2e/infx/github.py | 10 +++++- inferencex-e2e/infx/yamlio.py | 36 +++++++++---------- 10 files changed, 83 insertions(+), 26 deletions(-) diff --git a/inferencex-e2e/infx/cli.py b/inferencex-e2e/infx/cli.py index 6ad09e8691..79d2fc1ef2 100644 --- a/inferencex-e2e/infx/cli.py +++ b/inferencex-e2e/infx/cli.py @@ -12,6 +12,7 @@ def main(argv: list[str] | None = None) -> int: + """Run the ``infx`` command line on ``argv`` and return the exit status.""" parser = argparse.ArgumentParser(prog="infx") commands = parser.add_subparsers(dest="command", required=True) generate = commands.add_parser( diff --git a/inferencex-e2e/infx/config.py b/inferencex-e2e/infx/config.py index fca8964433..2a0f0df46f 100644 --- a/inferencex-e2e/infx/config.py +++ b/inferencex-e2e/infx/config.py @@ -9,6 +9,7 @@ def repository_root() -> Path: + """Return the project root from ``INFERENCEX_REPOSITORY_ROOT``, the source tree, or the working directory.""" if root := os.environ.get("INFERENCEX_REPOSITORY_ROOT"): return project_root(Path(root)) source_root = Path(__file__).resolve().parent.parent diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index e0619274cd..193048c1b9 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -49,6 +49,7 @@ def parse_args() -> argparse.Namespace: + """Parse the command line and return the options, rejecting ``--dwbug-min-peak`` alone.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -153,6 +154,7 @@ def parse_args() -> argparse.Namespace: def _run(cmd: list, **kw: Any) -> None: + """Echo ``cmd`` and run it, raising ``CalledProcessError`` on failure.""" print(f"$ {' '.join(shlex.quote(str(c)) for c in cmd)}", flush=True) subprocess.run(cmd, check=True, **kw) @@ -165,8 +167,11 @@ def _recreate(stage_dir: Path) -> None: def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: + """Dump sampled proxy sessions into ``work_dir/proxy`` and return that directory. + + With ``--skip-sample`` a complete earlier dump is reused. A missing manifest.json marks a crashed dump. + """ proxy_dir = work_dir / "proxy" - # The sampler writes manifest.json after every session file, so its absence marks a crashed dump. complete = (proxy_dir / "manifest.json").is_file() if args.skip_sample and complete and any(proxy_dir.glob("*.jsonl")): n = sum(1 for _ in proxy_dir.glob("*.jsonl")) @@ -203,6 +208,7 @@ def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: def stage_convert(args: argparse.Namespace, proxy_dir: Path, work_dir: Path) -> Path: + """Convert the proxy JSONLs in ``proxy_dir`` to weka JSONs under ``work_dir`` and return that directory.""" per_trace = work_dir / "per_trace" traces = {p.stem for p in per_trace.glob("*.json")} if args.skip_convert and traces and traces == {p.stem for p in proxy_dir.glob("*.jsonl")}: @@ -278,6 +284,7 @@ def _compute_stats(per_trace_dir: Path) -> dict: def _format_stats(stats: dict) -> str: + """Return ``stats`` as one aligned ``key: value`` line per entry.""" lines = [] for k, v in stats.items(): lines.append(f"{k}: {v:>15,}") @@ -306,6 +313,7 @@ def _build_readme( pretty_date: str | None = None, isl_cap: int | None = None, ) -> str: + """Return the dataset card README for ``repo_id`` from its stats, filters, and sampler command.""" now = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC") pretty_label = ( f"CC Traces — Weka, With Subagents, " @@ -410,6 +418,7 @@ def _build_readme( def _filters_block(args: argparse.Namespace, *, cap_256k: bool = False) -> str: + """Return the Markdown bullet list of the sampling filters in ``args``.""" bits = [] if ( args.min_trace_version is not None @@ -511,6 +520,7 @@ def _build_payload( def _is_oversize(req: dict, cap: int = CAP_TOKENS) -> bool: + """Whether the request's input plus output tokens exceed ``cap``.""" return (req.get("in") or 0) + (req.get("out") or 0) > cap @@ -646,6 +656,7 @@ def stage_isl_filter(per_trace_dir: Path, work_dir: Path, cap: int) -> Path: def stage_upload( args: argparse.Namespace, payload_dir: Path, repo_id: str, commit_msg: str ) -> None: + """Create the dataset repo ``repo_id`` and upload ``payload_dir`` to it, unless ``--skip-upload``.""" if args.skip_upload: print(f"[upload] --skip-upload: payload ready at {payload_dir}") return @@ -697,6 +708,7 @@ def _reconstruct_sampler_cmd(args: argparse.Namespace) -> list: def main() -> int: + """Sample, convert, build, and upload the base dataset and the optional 256k one, returning 0.""" args = parse_args() work_dir = args.work_dir work_dir.mkdir(parents=True, exist_ok=True) diff --git a/inferencex-e2e/infx/datasets/plot_subagent_distributions.py b/inferencex-e2e/infx/datasets/plot_subagent_distributions.py index 33394f7e14..e57d17e49b 100644 --- a/inferencex-e2e/infx/datasets/plot_subagent_distributions.py +++ b/inferencex-e2e/infx/datasets/plot_subagent_distributions.py @@ -90,6 +90,7 @@ def parse_args() -> argparse.Namespace: + """Parse the command line and return the input, output, and binning options.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -119,6 +120,7 @@ def parse_args() -> argparse.Namespace: def collect_metrics(in_dir: Path) -> dict[str, list[float]]: + """Read every weka trace in ``in_dir`` and return the sub-agent fan-out samples by metric name.""" groups_per_trace: list[int] = [] inners_per_group: list[int] = [] duration_sec: list[float] = [] @@ -178,6 +180,10 @@ def _draw_histogram( value_fmt: str, linear_clip_pct: float, ) -> None: + """Draw a histogram of ``values`` with percentile markers on ``ax``, or an empty panel if none. + + A log axis centres a single repeated value, since ``geomspace(v, v)`` can round to non-monotonic edges. + """ if not values: ax.set_title(f"{title}\n(no values)") ax.axis("off") @@ -193,7 +199,6 @@ def _draw_histogram( return hi = float(positive.max()) lo = max(float(positive.min()), 1e-3) if hi > 1e-3 else float(positive.min()) - # geomspace(v, v) can round to non-monotonic edges; centre a single value instead. edges = ( np.geomspace(lo, hi, bins + 1) if hi > lo else np.geomspace(lo / 2, hi * 2, bins + 1) ) @@ -242,6 +247,7 @@ def plot_combined( use_log: bool, linear_clip_pct: float, ) -> None: + """Draw every metric of ``metrics`` into one 3x2 figure and save it to ``out_path``.""" fig, axes = plt.subplots(3, 2, figsize=(20, 16)) for spec, ax in zip(PLOT_SPECS, axes.flat, strict=False): log_x = use_log and spec["log_in_log_fig"] @@ -267,6 +273,7 @@ def plot_combined( def main() -> int: + """Collect the metrics from ``--in-dir`` and write the log and linear figures, returning 0.""" args = parse_args() args.out_dir.mkdir(parents=True, exist_ok=True) metrics = collect_metrics(args.in_dir) diff --git a/inferencex-e2e/infx/datasets/plot_weka_distributions.py b/inferencex-e2e/infx/datasets/plot_weka_distributions.py index 29f58f17fa..d97a015c33 100644 --- a/inferencex-e2e/infx/datasets/plot_weka_distributions.py +++ b/inferencex-e2e/infx/datasets/plot_weka_distributions.py @@ -91,6 +91,7 @@ def parse_args() -> argparse.Namespace: + """Parse the command line and return the input, output, and binning options.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -126,6 +127,7 @@ def parse_args() -> argparse.Namespace: def collect_metrics(in_dir: Path) -> dict[str, list[float]]: + """Read every weka trace in ``in_dir`` and return the per-request and per-session samples by metric name.""" think_times: list[float] = [] isls: list[int] = [] osls: list[int] = [] @@ -201,6 +203,10 @@ def _draw_histogram( value_fmt: str, linear_clip_pct: float, ) -> None: + """Draw a histogram of ``values`` with percentile markers on ``ax``, or an empty panel if none. + + A log axis centres a single repeated value, since ``geomspace(v, v)`` can round to non-monotonic edges. + """ if not values: ax.set_title(f"{title}\n(no values)") ax.axis("off") @@ -216,7 +222,6 @@ def _draw_histogram( return hi = float(positive.max()) lo = max(float(positive.min()), 1e-3) if hi > 1e-3 else float(positive.min()) - # geomspace(v, v) can round to non-monotonic edges; centre a single value instead. edges = ( np.geomspace(lo, hi, bins + 1) if hi > lo else np.geomspace(lo / 2, hi * 2, bins + 1) ) @@ -271,6 +276,7 @@ def plot_combined( use_log: bool, linear_clip_pct: float, ) -> None: + """Draw every metric of ``metrics`` into one 3x2 figure and save it to ``out_path``.""" fig, axes = plt.subplots(3, 2, figsize=(20, 16)) for spec, ax in zip(PLOT_SPECS, axes.flat, strict=False): log_x = use_log and spec["log_in_log_fig"] @@ -296,6 +302,7 @@ def plot_combined( def main() -> int: + """Collect the metrics from ``--in-dir`` and write the log and linear figures, returning 0.""" args = parse_args() args.out_dir.mkdir(parents=True, exist_ok=True) diff --git a/inferencex-e2e/infx/datasets/proxy_to_weka.py b/inferencex-e2e/infx/datasets/proxy_to_weka.py index d5fff5a814..70f1e3492c 100644 --- a/inferencex-e2e/infx/datasets/proxy_to_weka.py +++ b/inferencex-e2e/infx/datasets/proxy_to_weka.py @@ -38,6 +38,7 @@ def _dump_trace_inline_hash_ids(trace: dict, path: Path) -> None: placeholders: list[list[Any]] = [] def _substitute(obj: Any) -> Any: + """Return a copy of ``obj`` with every ``hash_ids`` list swapped for a numbered placeholder.""" if isinstance(obj, dict): out: dict[str, Any] = {} for k, v in obj.items(): @@ -63,6 +64,7 @@ def _substitute(obj: Any) -> Any: def parse_args() -> argparse.Namespace: + """Parse the command line and return the input and output directories.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -88,10 +90,12 @@ def parse_args() -> argparse.Namespace: def slugify(label: str) -> str: + """Return ``label`` lowercased with non-alphanumeric runs as underscores, or ``subagent`` if empty.""" return _SLUG_RE.sub("_", label.lower()).strip("_") or "subagent" def load_session_rows(path: Path) -> list[dict]: + """Read the JSONL rows at ``path`` and return them sorted by timestamp with exact duplicates dropped.""" rows: list[dict] = [] with path.open() as f: for line in f: @@ -125,6 +129,7 @@ def load_session_rows(path: Path) -> list[dict]: def remap_hash(h: str, m: dict[str, int]) -> int: + """Return the dense integer id of hash ``h``, assigning the next one in ``m`` when it is new.""" if h not in m: m[h] = len(m) return m[h] @@ -218,6 +223,7 @@ def compute_think_times(rows: list[dict]) -> list[float | None]: def _parse_cli_version(s: str | None) -> tuple[int, int, int] | None: + """Parse a ``major.minor.patch`` CLI version string, returning None when it is missing or malformed.""" if not s: return None parts = s.split(".") @@ -268,6 +274,7 @@ def build_subagent_entry( items: list[tuple[dict, float | None]], hash_map: dict[str, int], ) -> dict: + """Return the weka sub-agent entry for one ``label`` instance built from its rows and think times.""" inner = [build_normal_request(row, hash_map, tt) for row, tt in items] first_row = items[0][0] last_row = items[-1][0] @@ -296,6 +303,7 @@ def build_subagent_entry( def session_to_weka(session_id: str, rows: list[dict]) -> dict: + """Convert one session's proxy ``rows`` into a weka trace dict for ``session_id``.""" if not rows: return { "id": session_id, @@ -386,6 +394,7 @@ def session_to_weka(session_id: str, rows: list[dict]) -> dict: def main() -> int: + """Convert every session JSONL in ``--in-dir`` into a weka trace in ``--out-dir``, returning 0.""" args = parse_args() in_files = sorted(p for p in args.in_dir.glob("*.jsonl")) diff --git a/inferencex-e2e/infx/datasets/sample_proxy_traces.py b/inferencex-e2e/infx/datasets/sample_proxy_traces.py index 9e3e22d988..20d9ee2f6d 100644 --- a/inferencex-e2e/infx/datasets/sample_proxy_traces.py +++ b/inferencex-e2e/infx/datasets/sample_proxy_traces.py @@ -47,10 +47,11 @@ def _redact_url(url: str) -> str: - """Show host + db, drop credentials.""" + """Show host + db, drop credentials. + + A libpq ``key=value`` conninfo has no URL netloc, so it is shown as a placeholder.""" try: p = urlsplit(url) - # libpq key=value conninfo has no URL netloc and would land whole in .path. if not p.netloc: return "" return f"{p.hostname}{p.path}" @@ -113,6 +114,7 @@ def _redact_url(url: str) -> str: def parse_args() -> argparse.Namespace: + """Parse and validate the command line, exiting on bad flag combinations, and return the options.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -353,6 +355,7 @@ def parse_args() -> argparse.Namespace: def get_db_url(args: argparse.Namespace) -> str: + """Return the connection string from ``--db-url`` or the environment, exiting when neither is set.""" url = args.db_url or os.environ.get(DEFAULT_ENV) if not url: sys.exit(f"ERROR: connection string missing. Set ${DEFAULT_ENV} or pass --db-url.") @@ -469,9 +472,9 @@ def _sort_and_limit(rows: list[dict], args: argparse.Namespace) -> list[dict]: md5(session_id || seed) is the same expression Postgres uses for the 'random' mode, so deterministic-random output is identical to the - one-pass version. + one-pass version. Rows are first sorted by session_id because phase 1 + has no ORDER BY, which makes ties reproducible. """ - # Phase 1 has no ORDER BY, so pre-sort by session_id to make ties reproducible. rows = sorted(rows, key=lambda r: r["session_id"]) if args.sampling == "top": rows = sorted(rows, key=lambda r: r["req_count"], reverse=True) @@ -494,7 +497,10 @@ def _sort_and_limit(rows: list[dict], args: argparse.Namespace) -> list[dict]: def find_sessions( conn: psycopg.Connection, args: argparse.Namespace, model_like: str ) -> list[dict]: + """Return the sampled summary rows of sessions passing every filter in ``args``.""" + def _encode_cli_int(s: str | None) -> int | None: + """Encode ``X.Y.Z`` as one comparable int, None when empty, exiting when malformed.""" if not s: return None parts = s.split(".") @@ -634,6 +640,7 @@ def _encode_cli_int(s: str | None) -> int | None: def find_one_session( conn: psycopg.Connection, session_id: str, privacy_mode: str, model_like: str ) -> list[dict]: + """Return the summary row of session ``session_id``, or an empty list when no rows match.""" logger.info("looking up session %s (--session-id mode)", session_id) t = time.time() with conn.cursor(row_factory=dict_row) as cur: @@ -703,6 +710,7 @@ def find_one_session( def _parse_cli_tuple(s: str | None) -> tuple[int, int, int] | None: + """Parse a ``major.minor.patch`` CLI version, returning None when it is missing or malformed.""" if not s: return None parts = s.split(".") @@ -715,6 +723,7 @@ def _parse_cli_tuple(s: str | None) -> tuple[int, int, int] | None: def _hash_lcp(a: list, b: list) -> int: + """Return the length of the common prefix of hash lists ``a`` and ``b``.""" n = 0 for x, y in zip(a, b, strict=False): if x == y: @@ -786,6 +795,7 @@ def fetch_session_rows( def write_session_rows(rows: list[dict], out_path: Path) -> int: + """Write ``rows`` to ``out_path`` as JSONL and return how many were written.""" with out_path.open("w") as f: for row in rows: f.write(json.dumps(row, default=str) + "\n") @@ -793,6 +803,7 @@ def write_session_rows(rows: list[dict], out_path: Path) -> int: def main() -> int: + """Find the requested sessions, dump each to a JSONL file plus manifest.json, and return 0.""" args = parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, diff --git a/inferencex-e2e/infx/fastcopy.py b/inferencex-e2e/infx/fastcopy.py index 19a9e528b7..194e5e53f6 100644 --- a/inferencex-e2e/infx/fastcopy.py +++ b/inferencex-e2e/infx/fastcopy.py @@ -12,6 +12,7 @@ def deepcopy(value: Any) -> Any: def _copy(value: Any, memo: dict[int, Any]) -> Any: + """Return a deep copy of ``value``, reusing copies already recorded in ``memo`` by object id.""" cls = type(value) if cls is str or cls is int or cls is float or cls is bool or value is None: return value diff --git a/inferencex-e2e/infx/github.py b/inferencex-e2e/infx/github.py index 7aade6e5f7..fe8f16cc1c 100644 --- a/inferencex-e2e/infx/github.py +++ b/inferencex-e2e/infx/github.py @@ -19,7 +19,7 @@ class APIError(RuntimeError): """Authenticated gh failure, with an HTTP status when gh reports one.""" def __init__(self, path: str, stderr: str) -> None: - # gh can print scope or SSO hints after its "(HTTP nnn)" line. + """Wrap a failed gh call on ``path``, parsing the HTTP status from ``stderr`` even when hints follow it.""" match = re.search(r"\(HTTP ([1-5][0-9]{2})\)\s*$", stderr, re.MULTILINE) self.status = int(match[1]) if match else None super().__init__(f"GitHub API {path} failed: {stderr.strip()}") @@ -35,6 +35,10 @@ def api( data: dict[str, Any] | None = None, paginate: bool = False, ) -> Any: + """Call the GitHub REST endpoint ``path`` under ``repo`` through gh and return the decoded JSON. + + With ``token``, failures raise ``APIError`` and an empty DELETE reply returns None. + """ if token is not None and not token.strip(): raise RuntimeError("GitHub API requires a non-empty token") endpoint, _, query = path.partition("?") @@ -85,6 +89,10 @@ def paginate( item_key: str = "", params: dict[str, str] | None = None, ) -> list[dict[str, Any]]: + """Fetch every page of a listing and return its rows, read from ``item_key`` when pages are objects. + + Raises ``ListingError`` when the listing is missing, malformed, or shorter than ``total_count``. + """ pages = api( repo, path, token, {**(params or {}), "per_page": "100", "page": "1"}, paginate=True ) diff --git a/inferencex-e2e/infx/yamlio.py b/inferencex-e2e/infx/yamlio.py index 93ec47480a..10d8750f06 100644 --- a/inferencex-e2e/infx/yamlio.py +++ b/inferencex-e2e/infx/yamlio.py @@ -35,13 +35,7 @@ "__firstlineno__", "__static_attributes__", } -# libyaml accepts documents PyYAML rejects when they hold directives, tabs, byte order marks, a -# comment glued to a block scalar header, or characters PyYAML's reader refuses. Line breaks -# other than "\n" go to PyYAML too, because the line-start patterns here only see "\n". _DIVERGENT = re.compile(r"(?m)^%|[\t\ufeff\r\x85\u2028\u2029]|[|>][-+0-9]*#") -# Deeper documents are left to PyYAML: its recursive composer raises RecursionError near 500 -# levels, where libyaml's recursive C composer returns a value or overflows the C stack. -# Indentless sequences add at most one level per counted collection token. _MAX_TOKEN_DEPTH = 50 _OPENING_TOKENS = ( yaml.BlockMappingStartToken, @@ -56,18 +50,16 @@ yaml.FlowMappingEndToken: -1, yaml.FlowSequenceEndToken: -1, } -# The largest srt recipe is 110 KiB; master configs and the changelog are parsed once a run. _CACHED_TEXT_CHARACTERS = 128 * 1024 _PYYAML = object() -# Each entry is (twin, watched classes, their attributes, the loader's tables) as of the build. _TWINS: dict[type, tuple[type, tuple[type, ...], tuple[dict[str, Any], ...], tuple[Any, ...]]] = {} -# libyaml skips the token after an empty explicit key in a flow collection; PyYAML rejects it. _AFTER_EMPTY_FLOW_KEY = (yaml.ValueToken, yaml.FlowEntryToken, yaml.FlowSequenceEndToken) def _class_state(classes: tuple[type, ...]) -> tuple[dict[str, Any], ...]: + """Return a snapshot of each class's own attributes in ``classes``, lookup tables excluded.""" return tuple( {key: value for key, value in dict(vars(cls)).items() if key not in _TABLES} for cls in classes @@ -87,6 +79,7 @@ def _class_state(classes: tuple[type, ...]) -> tuple[dict[str, Any], ...]: def _table_state(loader: type) -> tuple[dict[Any, Any], ...]: + """Return copies of the constructor and resolver tables of ``loader``.""" return tuple( {key: list(value) if isinstance(value, list) else value for key, value in table.items()} for table in (dict(getattr(loader, name)) for name in _TABLES) @@ -145,8 +138,10 @@ def _may_nest_deeply(text: str) -> bool: def _needs_pyyaml(text: str, fast: type) -> bool: """Whether the document has a tag, which libyaml and PyYAML scan differently, nesting too - deep for libyaml's composer, or a plain scalar inside a flow collection holding ``?``, - which ends it in PyYAML.""" + deep for libyaml's composer, a plain scalar inside a flow collection holding ``?``, which + ends it in PyYAML, or a token after an empty explicit flow key, which libyaml skips. + + Past 50 levels PyYAML nears its RecursionError while libyaml's C composer may overflow.""" if "?" not in text and "!" not in text and not _may_nest_deeply(text): return False depth = flow = 0 @@ -171,7 +166,10 @@ def _needs_pyyaml(text: str, fast: type) -> bool: def _libyaml_load(text: str, fast: type) -> Any: - """libyaml's result for ``text``, or ``_PYYAML`` where PyYAML has to decide.""" + """libyaml's result for ``text``, or ``_PYYAML`` where PyYAML has to decide. + + Directives, tabs, byte order marks, a comment glued to a block scalar header, characters + PyYAML's reader refuses, and line breaks other than newline all go to PyYAML.""" if _DIVERGENT.search(text) or yaml.reader.Reader.NON_PRINTABLE.search(text): return _PYYAML try: @@ -184,11 +182,15 @@ def _libyaml_load(text: str, fast: type) -> Any: @functools.lru_cache(maxsize=512) def _cached_libyaml_load(text: str, fast: type, int_digits: int) -> Any: # noqa: ARG001 - """Planner runs bind the same recipes and shared blocks once per matrix row.""" + """Return the libyaml result for ``text``, cached per loader and integer digit limit.""" return _libyaml_load(text, fast) def _fast_value(text: str, fast: type) -> Any: + """Return a fresh copy of the libyaml result for ``text``, or ``_PYYAML`` for PyYAML to decide. + + Only texts up to 128 KiB are cached, which covers the largest srt recipe at 110 KiB. Alias + chains too deep to copy go to PyYAML.""" if len(text) > _CACHED_TEXT_CHARACTERS: return _libyaml_load(text, fast) value = _cached_libyaml_load(text, fast, sys.get_int_max_str_digits()) @@ -197,7 +199,6 @@ def _fast_value(text: str, fast: type) -> Any: try: return fastcopy.deepcopy(value) except RecursionError: - # Chains of aliases can nest values far deeper than the document's own syntax. return _PYYAML @@ -212,7 +213,7 @@ def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: once per process and returned as fresh deep copies. ``INFX_PURE_YAML=1`` turns libyaml off. PyYAML internals patched before this module is imported, or mutated in place (such as ``Scanner.ESCAPE_REPLACEMENTS``), are not detected. Each loader class is kept for the life - of the process. + of the process. A stream whose ``tell()`` fails, as after iteration or closing, goes to PyYAML. """ fast = _fast_loader(loader) if fast is not None and type(stream) is str: @@ -221,7 +222,6 @@ def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: return value elif fast is not None and type(stream) in (io.TextIOWrapper, io.StringIO): try: - # Iterating a file disables tell(), and a closed one fails differently from read(). start = stream.tell() if stream.seekable() else None except (OSError, ValueError): start = None @@ -284,8 +284,8 @@ class UniqueKeySafeLoader(yaml.SafeLoader): def _reject_repeated_keys(loader: UniqueKeySafeLoader, node: yaml.MappingNode) -> None: """Check ``node``'s own keys, and those of the mappings it merges, before PyYAML's - ``flatten_mapping`` rewrites any of them in place.""" - # Holding the nodes keeps their ids from being reused by a later document's nodes. + ``flatten_mapping`` rewrites any of them in place. Checked nodes stay referenced on the + loader so a later document cannot reuse their ids.""" checked = loader.__dict__.setdefault("_infx_checked_mappings", {}) if id(node) in checked: return From 54c54d8b0012932b6bf710abe904c078d0976565 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:31:47 -0500 Subject: [PATCH 193/203] docs(matrix): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give every function and method in infx/matrix a short docstring that names its inputs, result, and errors. Delete the # comments, folding the fork server protocol, the Ctrl-C session note, the lazy fingerprint import, the fingerprint exclusion in trim_conc, and the run-sweep group limit into the docstrings that own them. Code is unchanged apart from docstrings. 中文:为 infx/matrix 中的每个函数和方法添加简短的文档字符串,说明其输入、 结果和异常。删除所有 # 注释,并将 fork 服务器协议、Ctrl-C 会话说明、 延迟导入指纹模块、trim_conc 中忽略指纹的原因以及 run-sweep 分组限制 并入相应函数的文档字符串。除文档字符串外代码未改动。 --- inferencex-e2e/infx/matrix/changed.py | 17 +++- inferencex-e2e/infx/matrix/fingerprint.py | 10 ++ inferencex-e2e/infx/matrix/generate.py | 116 +++++++++++++++++++++- inferencex-e2e/infx/matrix/plan.py | 29 +++++- inferencex-e2e/infx/matrix/revision.py | 45 +++++++-- inferencex-e2e/infx/matrix/validation.py | 66 ++++++++++++ 6 files changed, 269 insertions(+), 14 deletions(-) diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index 47db409ca6..f18b6fe7b5 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -22,6 +22,10 @@ def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: + """Expand generated rows into one point per concurrency, keyed by (fingerprint, conc). + + Return the points and the messages of rows whose recipe could not be fingerprinted. + """ points, errors = {}, [] for row in rows: try: @@ -37,6 +41,7 @@ def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: def head_matrix(config_files: list[str], runner_config: str) -> Matrix: + """Generate the working tree's matrix for each config key, as points and fingerprint errors.""" master = load_config_files(config_files) runners = load_runner_file(runner_config) root = config_root(config_files) @@ -49,6 +54,7 @@ def head_matrix(config_files: list[str], runner_config: str) -> Matrix: def base_matrix(ref: str, config_files: list[str]) -> Matrix: + """Generate the matrix of each config key at git ``ref``, as points and fingerprint errors.""" with snapshot(ref) as revision: configs = Path(revision.runner_config).parent files = [configs / Path(path).name for path in config_files] @@ -70,14 +76,17 @@ def base_matrix(ref: str, config_files: list[str]) -> Matrix: def _sorted(points: Iterable[dict]) -> list[dict]: + """Return the points ordered by experiment name, then concurrency.""" return sorted(points, key=lambda point: (point["exp-name"], point["conc"])) def _keys(base: Matrix, head: Matrix) -> list[str]: + """Return head's config keys followed by keys that exist only in base.""" return [*head, *(key for key in base if key not in head)] def compare(base: Matrix, head: Matrix) -> list[dict]: + """List the config keys whose points differ, with counts and the added and removed points.""" changed = [] for key in _keys(base, head): before, after = base.get(key, ({}, []))[0], head.get(key, ({}, []))[0] @@ -97,6 +106,7 @@ def compare(base: Matrix, head: Matrix) -> list[dict]: def _label(point: dict) -> str: + """Describe a point in one line by experiment, shape, conc, and short fingerprint.""" if "prefill" in point: shape = str(point.get("srt-recipe", "")).rpartition("/")[2] else: @@ -107,6 +117,7 @@ def _label(point: dict) -> str: def main(argv: list[str] | None = None) -> None: + """Report config keys whose points or recipes changed since ``--base``, as text or JSON.""" parser = argparse.ArgumentParser( description="List config keys whose benchmark points or recipes differ from a base." ) @@ -115,11 +126,13 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--runner-config", default=RUNNER_CONFIG) parser.add_argument("--json", action="store_true", help="print the report as JSON") args = parser.parse_args(argv) - # The head matrix is built in a daemon thread while the base revision's generators run, so - # an interrupt during base generation exits at once and head errors keep their precedence. head_outcome: dict[str, Any] = {} def build_head() -> None: + """Build the head matrix into ``head_outcome``, storing any error instead of raising. + + Runs in a daemon thread during base generation so an interrupt exits at once. + """ try: head_outcome["matrix"] = head_matrix(args.config_files, args.runner_config) except BaseException as error: # noqa: BLE001 diff --git a/inferencex-e2e/infx/matrix/fingerprint.py b/inferencex-e2e/infx/matrix/fingerprint.py index 41a60577da..432ddc3c13 100644 --- a/inferencex-e2e/infx/matrix/fingerprint.py +++ b/inferencex-e2e/infx/matrix/fingerprint.py @@ -24,19 +24,23 @@ def _digest(value: Any) -> str: + """Return the SHA-256 hex digest of ``value`` serialized as canonical JSON.""" canonical = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() def _row_identity(entry: Mapping[str, Any]) -> dict[str, Any]: + """Return the row without the keys that do not identify a benchmark point.""" return {key: value for key, value in entry.items() if key not in ROW_EXCLUDED} def row_fingerprint(entry: Mapping[str, Any]) -> str: + """Return the digest of a matrix row, ignoring conc, experiment name, and fingerprint.""" return _digest(_row_identity(entry)) def _without(block: dict[str, Any], path: tuple[str, ...]) -> dict[str, Any]: + """Return a copy of ``block`` without the nested key at ``path``, or ``block`` if absent.""" key, *rest = path if key not in block: return block @@ -48,6 +52,7 @@ def _without(block: dict[str, Any], path: tuple[str, ...]) -> dict[str, Any]: def _concrete_recipes(entry: Mapping[str, Any], root: Path) -> list[dict[str, Any]]: + """Render the row's bound srt recipe, minus the fields that vary per point or run.""" power_port = EXPORTER_PORT if entry.get("power") else None _, recipe = bound_variant( entry, point_environment(entry), root, expand=expand_variants, power_port=power_port @@ -58,6 +63,10 @@ def _concrete_recipes(entry: Mapping[str, Any], root: Path) -> list[dict[str, An def recipe_fingerprint(entry: Mapping[str, Any], root: Path) -> str: + """Return the digest of a row and its rendered srt recipe, or the row digest if it has none. + + Raise ValueError naming the recipe and point when the recipe cannot be rendered. + """ if not entry.get("srt-recipe"): return row_fingerprint(entry) try: @@ -69,6 +78,7 @@ def recipe_fingerprint(entry: Mapping[str, Any], root: Path) -> str: def main() -> None: + """Read matrix rows as JSON from stdin and print their recipe fingerprints as a JSON list.""" root = repository_root() print(json.dumps([recipe_fingerprint(row, root) for row in json.load(sys.stdin)])) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 0ae80db1d6..a47e8922fd 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -43,10 +43,12 @@ def seq_len_to_str(isl: int, osl: int) -> str: + """Return the short name of an ISL/OSL pair, such as 1k1k, or ``{isl}_{osl}``.""" return seq_len_itos.get((isl, osl), f"{isl}_{osl}") def freeze_config_value(value: Any) -> Any: + """Return ``value`` with dicts and lists turned into sorted tuples so it can be hashed.""" if isinstance(value, dict): return tuple(sorted((key, freeze_config_value(item)) for key, item in value.items())) if isinstance(value, list): @@ -55,10 +57,14 @@ def freeze_config_value(value: Any) -> Any: def trim_conc(entries: list[dict]) -> list[dict]: + """Keep only the lowest-concurrency row of each otherwise identical group of rows. + + Multinode rows shrink to their lowest conc, and the kept row inherits the group's eval flags. + The recipe fingerprint is ignored since single-node bundles bind one variant per conc. + """ ignored_fields = { "conc", "exp-name", - # Single-node bundles bind one variant per CONC, so the fingerprint varies with it. Fields.RECIPE_FINGERPRINT.value, "run-eval", "eval-only", @@ -71,6 +77,7 @@ def trim_conc(entries: list[dict]) -> list[dict]: out: list[dict] = [] def minimum_concurrency(entry: dict) -> int: + """Return the lowest concurrency of a row whose conc is an int or a list.""" conc = entry["conc"] return min(conc) if isinstance(conc, list) else conc @@ -123,6 +130,7 @@ def minimum_concurrency(entry: dict) -> int: def smoke_entries(entries: list[dict]) -> list[dict]: + """Return trimmed benchmark rows with evals off, followed by trimmed eval-only rows.""" benchmarks = trim_conc( [{**row, "run-eval": False} for row in entries if not row.get(Fields.EVAL_ONLY.value)] ) @@ -140,10 +148,12 @@ def smoke_entries(entries: list[dict]) -> list[dict]: def runner_labels(runner_data: dict) -> dict: + """Return the runner config's map of runner labels to node names.""" return runner_data["labels"] def runner_hardware(runner_data: dict) -> dict: + """Return GPU and CPU DRAM facts keyed by cluster label, for clusters that declare them.""" node_fields = (Fields.GPUS_PER_NODE.value, Fields.AVAILABLE_CPU_DRAM_MIB.value) return { f"{CLUSTER_LABEL_PREFIX}{cluster_id}": { @@ -154,10 +164,12 @@ def runner_hardware(runner_data: dict) -> dict: def runner_nodes_for_label(runner: str, runner_data: dict) -> list[str]: + """Return the node names of runner label ``runner``, or an empty list.""" return runner_labels(runner_data).get(runner, []) def runner_hardware_int(runner: str, runner_data: dict, field: str) -> int: + """Return the integer hardware ``field`` of ``runner``, or raise ValueError if it is unset.""" hardware = runner_hardware(runner_data).get(runner, {}) value = hardware.get(field) if value is None: @@ -170,18 +182,25 @@ def runner_hardware_int(runner: str, runner_data: dict, field: str) -> int: def runner_available_cpu_dram_mib(runner: str, runner_data: dict) -> int: + """Return the CPU DRAM in MiB available to ``runner``.""" return runner_hardware_int(runner, runner_data, Fields.AVAILABLE_CPU_DRAM_MIB.value) def runner_gpus_per_node(runner: str, runner_data: dict) -> int: + """Return the GPUs per node of ``runner``.""" return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) def _hardware_family(label: str) -> str: + """Return the hardware family of a label, such as b200 for cluster:b200-dgxc.""" return label.removeprefix("cluster:").split("-", 1)[0] def scheduling_gpus_per_node(label: str, runner_data: dict) -> int: + """Return the GPUs per node of ``label``, or of its hardware family when that is unambiguous. + + Raise ValueError when no cluster or more than one GPU count matches. + """ hardware = runner_hardware(runner_data) exact = hardware.get(label) if exact is not None: @@ -201,6 +220,10 @@ def scheduling_gpus_per_node(label: str, runner_data: dict) -> int: def _worker_node_override(worker: dict, setting_name: str) -> int | None: + """Return the positive ``setting_name=N`` value in a worker's additional settings, or None. + + Raise ValueError when the settings disagree or are not positive. + """ pattern = re.compile(rf"^{re.escape(setting_name)}=(\d+)$") values = [] for setting in worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or []: @@ -215,6 +238,7 @@ def _worker_node_override(worker: dict, setting_name: str) -> int | None: def recipe_auxiliary_node_count(recipe: dict) -> int: + """Return the nodes a recipe needs beyond its workers, for service pools and dedicated roles.""" services = [service for service in recipe.get("services") or [] if service.get("enabled", True)] pool_nodes = sum(service.get("nodes") or 0 for service in services) infra_dedicated = (recipe.get("infra") or {}).get("etcd_nats_dedicated_node", False) @@ -242,6 +266,10 @@ def recipe_auxiliary_node_count(recipe: dict) -> int: def recipe_node_count(root: Path, srt_recipe: str) -> int | None: + """Return the total node count of a schema 2 srt recipe, or None for an unknown variant. + + Raise ValueError when the recipe is not schema 2 or its roles lack node counts. + """ path, _, selector = srt_recipe.partition(":") recipe_path = root / path recipe = yamlio.safe_load(recipe_path.read_text()) @@ -271,6 +299,7 @@ def recipe_node_count(root: Path, srt_recipe: str) -> int | None: def _label_fabrics(label: str, runner_data: dict) -> dict[str, Fabric]: + """Return the srt-slurm fabric of each cluster that runs ``label``, keyed by cluster ID.""" labels = runner_labels(runner_data) runners = set(labels.get(label, [label])) fabrics = {} @@ -284,6 +313,7 @@ def _label_fabrics(label: str, runner_data: dict) -> dict[str, Fabric]: def check_fabric_references(root: Path, srt_recipe: str, label: str, runner_data: dict) -> None: + """Raise ValueError if a recipe variant references a fabric field ``label``'s clusters lack.""" path, _, selector = srt_recipe.partition(":") variants = expand_variants(yamlio.safe_load((root / path).read_text()), selector or None) fabrics = { @@ -305,6 +335,7 @@ def worker_node_count( runner: str, runner_data: dict, ) -> int: + """Return the nodes a worker role needs, from a ``_NODES`` setting or its GPU count.""" override = _worker_node_override(worker, f"{role.upper()}_NODES") if override is not None: return override @@ -329,6 +360,7 @@ def multinode_node_count( srt_recipe: str | None, root: Path | None, ) -> int: + """Return a multinode entry's node count from its srt recipe, else from its workers.""" if srt_recipe is not None and root is None: raise ValueError(f"Reading {srt_recipe} needs the master config's project root") recipe_count = recipe_node_count(root, srt_recipe) if srt_recipe is not None else None @@ -345,6 +377,10 @@ def add_multinode_node_count( num_nodes: int | None, root: Path | None, ) -> dict: + """Set the entry's node count from ``num_nodes`` or its workers, and return the entry. + + Raise ValueError when ``num_nodes`` is given for a disaggregated entry. + """ if not entry[Fields.DISAGG.value] and num_nodes is not None: entry[Fields.NODE_COUNT.value] = num_nodes elif num_nodes is not None: @@ -362,6 +398,7 @@ def add_multinode_node_count( def effective_gpu_count(benchmark: dict) -> int: + """Return the GPUs one benchmark worker uses, TP times PP times PCP.""" return ( benchmark[Fields.TP.value] * benchmark.get(Fields.PP.value, 1) @@ -370,6 +407,7 @@ def effective_gpu_count(benchmark: dict) -> int: def with_worker_parallelism_defaults(worker: dict) -> dict: + """Return a copy of the worker with PP, DCP, and PCP defaulting to 1.""" return { **worker, Fields.PP.value: worker.get(Fields.PP.value, 1), @@ -379,6 +417,7 @@ def with_worker_parallelism_defaults(worker: dict) -> dict: def multinode_worker_pair(benchmark: dict, disagg: bool) -> tuple[dict, dict]: + """Return the prefill and decode workers, splitting an aggregated worker into a pair.""" if disagg: return ( with_worker_parallelism_defaults(benchmark[Fields.PREFILL.value]), @@ -403,6 +442,10 @@ def multinode_worker_pair(benchmark: dict, disagg: bool) -> tuple[dict, dict]: def worker_gpus_per_node(worker: dict, gpus_per_node: int, role: str = "worker") -> int: + """Return how many of a worker's GPUs land on each node. + + Raise ValueError when its replica size and ``gpus_per_node`` do not divide one another. + """ gpus_per_replica = ( worker[Fields.TP.value] * worker.get(Fields.PP.value, 1) @@ -432,6 +475,10 @@ def worker_gpus_per_node(worker: dict, gpus_per_node: int, role: str = "worker") def agentic_dram_offload_gb( agentic_config: dict, benchmark: dict, runner: str, runner_data: dict ) -> tuple[int, int]: + """Return the CPU DRAM offload in GB for the prefill and decode sides of an agentic point. + + Both are 0 without DRAM offload. Raise ValueError when the workers do not fit the nodes. + """ kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none") if kv_offloading != "dram": return 0, 0 @@ -444,6 +491,7 @@ def agentic_dram_offload_gb( gpus_per_node = runner_gpus_per_node(runner, runner_data) def share_gb(gpu_count: int) -> int: + """Return the GB of offload DRAM that ``gpu_count`` GPUs get as their share of a node.""" proportional_bytes = ( Decimal(available_mib) * BYTES_PER_MIB * utilization * gpu_count / gpus_per_node ) @@ -474,6 +522,7 @@ def agentic_kv_offload_suffix( kv_offloading: str, kv_offload_backend: dict | None, ) -> str: + """Return the experiment-name suffix for a KV offload mode and its backend.""" if kv_offloading == "none": return "kvnone" return f"kv{kv_offloading}-{kv_offload_backend['name']}" @@ -486,8 +535,10 @@ def multinode_agentic_exp_name( conc_batch: list[int], offload_suffix: str, ) -> str: + """Return a multinode agentic experiment name from the model, worker shapes, and concs.""" def _worker_tag(worker: dict, role_prefix: str) -> str: + """Return a short tag of a worker's count, TP, EP, DP attention, and DWDP size.""" ep = worker.get(Fields.EP.value, 1) dpa = worker.get(Fields.DP_ATTN.value, False) tag = f"{role_prefix}{worker[Fields.NUM_WORKER.value]}x{worker[Fields.TP.value]}" @@ -508,6 +559,7 @@ def _worker_tag(worker: dict, role_prefix: str) -> str: def component_metadata(benchmark: dict, config: dict) -> dict: + """Return the router and KV transfer settings of a benchmark, falling back to its config.""" metadata = {} for field in (Fields.ROUTER, Fields.KV_P2P_TRANSFER): value = benchmark.get(field.value, config.get(field.value)) @@ -517,6 +569,7 @@ def component_metadata(benchmark: dict, config: dict) -> dict: def srt_recipe_fields(config: dict, benchmark: dict) -> dict: + """Return the benchmark's srt recipe fields resolved to paths under the config's recipe dir.""" return { field.value: srt_recipe_path( config.get(Fields.MULTINODE.value, False), @@ -529,6 +582,7 @@ def srt_recipe_fields(config: dict, benchmark: dict) -> dict: def _multinode_parallelism_key(entry: dict) -> tuple: + """Return a hashable identity of a multinode row, ignoring conc, eval, and naming fields.""" ignored_fields = { Fields.CONC.value, Fields.RUN_EVAL.value, @@ -549,12 +603,14 @@ def _multinode_parallelism_key(entry: dict) -> tuple: def automatic_agentic_vendor_eval(entry: dict) -> tuple[str, str] | None: + """Return the vendor eval framework and suite run automatically for an agentic row, or None.""" if entry.get(Fields.SCENARIO_TYPE.value) != "agentic-coding": return None return AUTOMATIC_AGENTIC_VENDOR_EVALS.get(entry.get(Fields.MODEL_PREFIX.value)) def app_eval_identity(entry: dict, conc: int) -> tuple: + """Return the identity of the app eval a row runs at ``conc``, to detect duplicate evals.""" if Fields.PREFILL.value in entry: topology = tuple( tuple( @@ -588,6 +644,7 @@ def app_eval_identity(entry: dict, conc: int) -> tuple: def _eval_concs(entry: dict) -> list[int]: + """Return the concurrencies a row would evaluate at.""" conc = entry[Fields.CONC.value] conc_values = conc if isinstance(conc, list) else [conc] if Fields.PREFILL.value in entry and not entry.get(Fields.EVAL_ALL_CONCS.value): @@ -596,6 +653,10 @@ def _eval_concs(entry: dict) -> list[int]: def drop_app_colliding_evals(rows: list[dict]) -> list[dict]: + """Remove eval concurrencies already claimed by another row and return the rows. + + Rows without KV offload claim first. A row left with none stops running evals. + """ claimed: set[tuple] = set() candidates = sorted( (row for row in rows if row.get(Fields.RUN_EVAL.value)), @@ -622,6 +683,11 @@ def drop_app_colliding_evals(rows: list[dict]) -> list[dict]: def mark_eval_entries(matrix_values: list[dict]) -> list[dict]: + """Choose which rows run evals and return the rows, plus extra gsm8k eval-only rows. + + Pick the top and median 8k1k concs per single-node group, the top eligible conc per multinode + shape, one gsm8k point per agentic deployment, and every automatic vendor eval. + """ from collections import defaultdict target_isl, target_osl = seq_len_stoi["8k1k"] @@ -629,6 +695,7 @@ def mark_eval_entries(matrix_values: list[dict]) -> list[dict]: mn_eval_conc = {} def _eligible_eval_concs(entry: dict) -> list[int]: + """Return a row's concurrencies of at least MIN_EVAL_CONC, sorted.""" conc = entry[Fields.CONC.value] conc_values = conc if isinstance(conc, list) else [conc] return sorted(c for c in conc_values if c >= MIN_EVAL_CONC) @@ -773,6 +840,7 @@ def _eligible_eval_concs(entry: dict) -> list[int]: def _gsm8k_eval_only_row(entry: dict, eval_conc: int | None = None) -> dict: + """Return a gsm8k eval-only copy of ``entry``, at ``eval_conc`` when given.""" row = { **entry, Fields.RUN_EVAL.value: True, @@ -787,6 +855,10 @@ def _gsm8k_eval_only_row(entry: dict, eval_conc: int | None = None) -> dict: def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: + """Mark every eligible row to run evals and return the rows, batching multinode concs. + + Agentic vendor evals become separate eval-only rows. + """ expanded_entries: list[dict] = [] multinode_indices: dict[tuple, int] = {} multinode_agentic_indices: dict[tuple, int] = {} @@ -888,6 +960,7 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: def _concurrency_range(start: int, end: int, step: int) -> list[int]: + """Return ``start``, ``start * step``, and so on, capped so the last value is ``end``.""" values = [] while start <= end: values.append(start) @@ -906,6 +979,10 @@ def _fixed_sequence_entries( runner_data: dict, root: Path | None, ) -> list[dict]: + """Return validated fixed-sequence rows for one benchmark, per runner and concurrency. + + Multinode rows carry all concs in one row. Raise ValueError for an invalid power request. + """ is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) isl, osl = sequence[Fields.ISL.value], sequence[Fields.OSL.value] @@ -991,6 +1068,7 @@ def _agentic_entries( conc_filter: list[int] | None = None, root: Path | None = None, ) -> list[dict]: + """Return validated agentic rows for one benchmark at each runner and allowed concurrency.""" is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) model_code = config[Fields.MODEL_PREFIX.value] @@ -1127,6 +1205,7 @@ def generate_full_sweep( runner_data: dict, root: Path | None = None, ) -> list[dict]: + """Return the full-sweep rows selected by the parsed command-line ``args``.""" return expand_full_sweep( all_config_data, runner_data, @@ -1157,6 +1236,10 @@ def expand_full_sweep( options: FullSweepOptions = FullSweepOptions(), root: Path | None = None, ) -> list[dict]: + """Return rows for every master config that passes the full-sweep filters in ``options``. + + Raise ValueError for invalid options or filters that match no config. + """ if options.step_size <= 1: raise ValueError("step_size must be greater than 1") if ( @@ -1214,6 +1297,10 @@ def _fixed_sequence_concurrencies( full_sweep: FullSweepOptions | None, concurrencies: list[int] | None, ) -> list[int] | None: + """Return a fixed-sequence benchmark's concurrencies after the sweep or ``--conc`` filters. + + Return None when the filters leave nothing to run. + """ if full_sweep is None: values = ( benchmark[Fields.CONC_LIST.value] @@ -1263,6 +1350,7 @@ def _runner_values_for_filter( *, include_label: bool = True, ) -> list[str]: + """Return the runner label, or its nodes and itself, that match ``runner_node_filter``.""" if not runner_node_filter: return [runner] candidates = runner_nodes_for_label(runner, runner_data) @@ -1279,6 +1367,7 @@ def generate_test_config_sweep( runner_data: dict | None = None, root: Path | None = None, ) -> list[dict]: + """Return the rows of the config keys named on the ``test-config`` command line.""" return _expand_selected_configs( args.config_keys, all_config_data, @@ -1302,6 +1391,10 @@ def _expand_selected_configs( concurrencies: list[int] | None = None, root: Path | None = None, ) -> list[dict]: + """Return rows for the selected config keys, with the given filters applied. + + Raise ValueError when a requested concurrency is generated by no row. + """ resolved_keys = expand_config_keys(config_keys, all_config_data.keys()) rows = _expand_configs( (all_config_data[key] for key in resolved_keys), @@ -1337,6 +1430,10 @@ def _expand_configs( full_sweep: FullSweepOptions | None = None, root: Path | None = None, ) -> list[dict]: + """Return the fixed-sequence and agentic rows of each config that pass the filters. + + With ``root``, raise ValueError when a selected recipe references an unknown fabric field. + """ rows = [] for config in configs: multinode = config.get(Fields.MULTINODE.value, False) @@ -1436,6 +1533,10 @@ def _expand_configs( def expand_config_keys(config_keys: Iterable[str], available_keys: Iterable[str]) -> list[str]: + """Return the config keys, with glob patterns expanded, in order and without duplicates. + + Raise ValueError for a key or pattern that matches nothing. + """ available = list(available_keys) seen = {} for key in config_keys: @@ -1459,6 +1560,7 @@ def expand_config_keys(config_keys: Iterable[str], available_keys: Iterable[str] def filter_exp_names(entries: list[dict], exp_names: list[str]) -> list[dict]: + """Return the rows named in ``exp_names``, raising ValueError unless each matches one row.""" requested = set(exp_names) if len(requested) != len(exp_names): raise ValueError("--exp-names contains duplicate values") @@ -1479,6 +1581,7 @@ def filter_exp_names(entries: list[dict], exp_names: list[str]) -> list[dict]: def apply_node_type_defaults(args: argparse.Namespace) -> argparse.Namespace: + """Enable both node types in ``args`` when neither is set, and return ``args``.""" if ( hasattr(args, "single_node") and hasattr(args, "multi_node") @@ -1499,6 +1602,10 @@ def select_matrix_evals( mode: EvalMode = "default", trim: bool = False, ) -> list[dict]: + """Apply eval selection ``mode`` to rows, optionally trimming to minimum concurrency. + + Raise ValueError for an unknown mode or smoke combined with trimming. + """ if mode not in ("default", "none", "subset", "all", "smoke"): raise ValueError(f"Unknown eval mode: {mode!r}") if mode == "smoke" and trim: @@ -1532,6 +1639,7 @@ def generate_config_matrix( eval_mode: EvalMode = "default", root: Path | None = None, ) -> list[dict]: + """Return the JSON-ready rows of the config keys with ``eval_mode`` eval selection applied.""" rows = _expand_selected_configs( config_keys, master_config, @@ -1547,9 +1655,10 @@ def with_fingerprints(rows: list[dict], root: Path) -> list[dict]: """``rows`` with the ``recipe-fingerprint`` changelog plans give the same throughput rows. Eval selection copies it to the rows it derives. A row whose recipe cannot be bound gets - none and a warning; its job fails when it binds the recipe. + none and a warning, and its job fails when it binds the recipe. The fingerprint module is + imported here because it imports this module through srt_slurm. """ - from infx.matrix.fingerprint import recipe_fingerprint # imports this module via srt_slurm + from infx.matrix.fingerprint import recipe_fingerprint fingerprinted = [] for row in rows: @@ -1564,6 +1673,7 @@ def with_fingerprints(rows: list[dict], root: Path) -> list[dict]: def main() -> list[dict]: + """Parse the command line, print the generated matrix rows as JSON, and return them.""" parent_parser = argparse.ArgumentParser(add_help=False) parent_parser.add_argument( "--config-files", diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index 5d53aada72..35551490fb 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -31,6 +31,10 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: + """Return the lines ``filepath`` gained between two git refs, joined by newlines. + + Raise RuntimeError when git diff fails and ValueError when any non-blank line was deleted. + """ result = subprocess.run( [ "git", @@ -67,6 +71,7 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: def filter_eval_rows_by_prefill_ep(eval_rows: list[dict], min_prefill_ep: int | None) -> list[dict]: + """Keep rows with no prefill block or a prefill EP of at least ``min_prefill_ep`` if set.""" if min_prefill_ep is None: return eval_rows kept: list[dict] = [] @@ -84,6 +89,10 @@ def filter_eval_rows_by_prefill_ep(eval_rows: list[dict], min_prefill_ep: int | def get_config_keys_from_master(config_keys: list[str], master_config: dict) -> list[str]: + """Expand ``*`` wildcards against the master config and return the matching keys in order. + + Raise ValueError for a key or pattern that matches nothing. + """ resolved_keys = {} for key in config_keys: if "*" in key: @@ -103,6 +112,7 @@ def get_config_keys_from_master(config_keys: list[str], master_config: dict) -> def _matrix_curve_key(entry: dict) -> tuple: + """Return a hashable identity of a matrix row's curve, ignoring its conc and naming fields.""" return tuple( sorted( (key, freeze_config_value(value)) @@ -113,6 +123,7 @@ def _matrix_curve_key(entry: dict) -> tuple: def _matrix_visual_series_key(entry: dict) -> tuple: + """Return the fields that place a matrix row on one visual curve of the dashboard.""" is_agentic = entry.get("scenario-type") == "agentic-coding" kv_offloading = entry.get("kv-offloading", "none") offload_mode = "off" if kv_offloading in (None, "", "none") else "on" @@ -136,6 +147,7 @@ def _matrix_visual_series_key(entry: dict) -> tuple: def _matrix_concurrencies(entry: dict) -> tuple[int, ...]: + """Return a row's concurrencies as a tuple, or raise ValueError if they are not ints.""" conc = entry.get("conc") if isinstance(conc, int): return (conc,) @@ -145,6 +157,11 @@ def _matrix_concurrencies(entry: dict) -> tuple[int, ...]: def append_only_delta(base_entries: list[dict], head_entries: list[dict]) -> list[dict]: + """Return the head rows trimmed to the concurrency points the base revision lacks. + + Raise ValueError when head removes base curves or points, adds nothing, or adds points to a + curve without exactly one unchanged image. + """ base_groups: dict[tuple, set[int]] = defaultdict(set) head_groups: dict[tuple, set[int]] = defaultdict(set) for entry in base_entries: @@ -210,6 +227,7 @@ def validate_append_only_scope( head_master: dict, selected_config_scenarios: dict[str, set[str]], ) -> None: + """Raise ValueError if append-only edits touch unselected configs or scenarios.""" selected_configs = selected_config_scenarios.keys() all_keys = base_master.keys() | head_master.keys() unrelated_changes = [ @@ -255,6 +273,7 @@ def group_unseen_scenarios( scenarios: tuple[str, ...], seen: dict[str, set[str]], ) -> dict[tuple[str, ...], list[str]]: + """Group configs by the requested scenarios not yet in ``seen``, marking them as seen.""" groups: dict[tuple[str, ...], list[str]] = defaultdict(list) for config in config_keys: unseen = tuple( @@ -270,6 +289,7 @@ def group_unseen_scenarios( class MatrixGenerationError(ValueError): def __init__(self, command: list[str], cause: Exception) -> None: + """Wrap the generation error ``cause`` with the equivalent generator ``command``.""" super().__init__(str(cause)) self.command, self.cause = command, cause @@ -285,6 +305,11 @@ def build_plan( all_evals: bool = False, evals_only: bool = False, ) -> ChangelogMatrixEntry: + """Build the sweep matrix for changelog entries added between ``base_ref`` and ``head_ref``. + + Raise ValueError for invalid entry combinations, append-only violations, and fixed-sequence + points with no sweep job in run-sweep.yml, and MatrixGenerationError when generation fails. + """ if not changelog_data: raise ValueError("No valid YAML entries found in the changelog additions.") @@ -329,6 +354,7 @@ def generate_current( mode: EvalMode, scenarios: tuple[str, ...] | None, ) -> list[dict]: + """Generate the working tree's rows for ``keys``, or raise MatrixGenerationError.""" nonlocal runner_data try: if runner_data is None: @@ -435,7 +461,6 @@ def generate_current( scenario = "agentic" else: scenario = seq_len_itos.get((result["isl"], result["osl"])) - # run-sweep.yml dispatches only these groups; any other would never run. if scenario is None: raise ValueError( f"{result['exp-name']}: fixed-sequence isl={result['isl']} " @@ -458,6 +483,7 @@ def generate_current( def main() -> None: + """Print the sweep matrix, as JSON, for the changelog lines added between two refs.""" parser = argparse.ArgumentParser() parser.add_argument("--base-ref", type=str, required=True) parser.add_argument("--head-ref", type=str, required=True) @@ -499,6 +525,7 @@ def main() -> None: def _generation_flags(mode: EvalMode, scenarios: tuple[str, ...] | None) -> list[str]: + """Return the generator flags that select ``mode`` evals and the given scenario types.""" flags = ["--no-evals"] if mode == "none" else ["--evals-only"] if mode == "all": flags.append("--all-evals") diff --git a/inferencex-e2e/infx/matrix/revision.py b/inferencex-e2e/infx/matrix/revision.py index 3a0b57a71c..ee18fd1bb4 100644 --- a/inferencex-e2e/infx/matrix/revision.py +++ b/inferencex-e2e/infx/matrix/revision.py @@ -28,6 +28,7 @@ class Tool: @property def module_path(self) -> str: + """The module's file path relative to the revision root.""" return self.module.replace(".", "/") + ".py" @@ -45,12 +46,6 @@ def module_path(self) -> str: "benchmarks/multi_node/srt-slurm-recipes", ) INHERITED_ENV = ("PATH", "HOME", "LANG", "TMPDIR") -# Each request line is [argv, output path]; a fork runs the tool as ``python -m`` would, with -# stdout in the output file, and the server answers with the fork's exit status on a line of its -# own after a marker, so output from importing the tool cannot be mistaken for a reply. -# The server preloads only the tool's top-level imports, so the tool's own top-level code runs -# once per request, in the fork, as the real __main__; the fork then exits through normal -# interpreter finalization (threads joined, atexit, flushes, exit-status rules of CPython). _FORK_REPLY = "infx-fork-exit " _FORK_SERVER = """ import ast, importlib.util, json, os, runpy, sys, types @@ -89,18 +84,22 @@ class Revision: root: Path def __post_init__(self) -> None: + """Resolve ``root`` to an absolute path.""" object.__setattr__(self, "root", Path(self.root).resolve()) @property def master_configs(self) -> list[str]: + """Paths of the revision's master config files.""" return [str(self._configs / Path(path).name) for path in MASTER_CONFIGS] @property def runner_config(self) -> str: + """Path of the revision's runner config.""" return str(self._configs / "runners.yaml") @property def _configs(self) -> Path: + """The revision's config directory, the first that holds runners.yaml.""" return next( ( self.root / name @@ -111,6 +110,10 @@ def _configs(self) -> Path: ) def invocation(self, tool: Tool, args: Sequence[str]) -> tuple[list[str], dict[str, str]]: + """Return the command and environment that run ``tool`` with ``args`` in this revision. + + Prefer the module layout, else the legacy script. Raise ValueError when neither exists. + """ script = self.root / tool.legacy_script if (self.root / tool.module_path).is_file(): entrypoint, imports = ["-m", tool.module], [self.root] @@ -121,6 +124,7 @@ def invocation(self, tool: Tool, args: Sequence[str]) -> tuple[list[str], dict[s return [sys.executable, *entrypoint, *args], self._environment(imports) def _environment(self, imports: Sequence[Path]) -> dict[str, str]: + """Return a minimal environment with ``imports`` on PYTHONPATH and the revision as root.""" env = {name: os.environ[name] for name in INHERITED_ENV if name in os.environ} env["PYTHONPATH"] = os.pathsep.join(map(str, imports)) env["INFERENCEX_REPOSITORY_ROOT"] = str(self.root) @@ -129,6 +133,7 @@ def _environment(self, imports: Sequence[Path]) -> dict[str, str]: def _generator_arguments( self, config_keys: Sequence[str], flags: Sequence[str], config_files: Sequence[str] | None ) -> list[str]: + """Return the generator's ``test-config`` arguments for these keys, flags, and configs.""" return [ "test-config", "--config-keys", @@ -146,6 +151,10 @@ def generate( flags: Sequence[str], config_files: Sequence[str] | None = None, ) -> list[dict]: + """Run the revision's generator for ``config_keys`` and return its rows. + + Raise CalledProcessError when the generator fails. + """ command, env = self.invocation( GENERATOR, self._generator_arguments(config_keys, flags, config_files) ) @@ -182,6 +191,7 @@ def generate_each(self, requests: Sequence[GenerateRequest], workers: int) -> li stopping = threading.Event() def run(index: int, request: GenerateRequest) -> list[dict]: + """Return request ``index``'s rows from a fork, or its own process on failure.""" output = Path(outputs, f"{index}.json") server = servers.get() try: @@ -206,6 +216,10 @@ def run(index: int, request: GenerateRequest) -> list[dict]: pool.shutdown(wait=True, cancel_futures=True) def fingerprints(self, rows: list[dict]) -> list[str]: + """Return the revision's recipe fingerprint of each row, or row digests if it has none. + + Raise ValueError when the fingerprinter does not print one fingerprint per row. + """ if not (self.root / f"{FINGERPRINTER.replace('.', '/')}.py").is_file(): from infx.matrix.fingerprint import row_fingerprint @@ -226,11 +240,18 @@ def fingerprints(self, rows: list[dict]) -> list[str]: class _ForkServer: - """A process of ``revision``'s environment that imported ``tool`` and forks per request.""" + """A process in ``revision``'s environment that preloads ``tool``'s imports and forks. + + Each fork runs the tool as ``__main__`` with stdout in the request's output file. + """ def __init__(self, revision: Revision, tool: Tool) -> None: + """Start the server in its own session, so a terminal Ctrl-C does not reach its forks. + + Requests are JSON ``[argv, output path]`` lines. Each exit status follows a marker, so + output from importing the tool is never taken for a reply. + """ _, env = revision.invocation(tool, []) - # Its own session keeps a terminal Ctrl-C away from forks that would then fall back. self._process = subprocess.Popen( [sys.executable, "-c", _FORK_SERVER, tool.module, _FORK_REPLY], cwd=revision.root, @@ -247,13 +268,16 @@ def __init__(self, revision: Revision, tool: Tool) -> None: self._noisy = False def __enter__(self) -> Self: + """Return the server itself.""" return self def kill(self) -> None: + """Kill the server and its forks, ignoring a server that already exited.""" with suppress(ProcessLookupError): os.killpg(self._process.pid, signal.SIGKILL) def __exit__(self, error_type: type[BaseException] | None, *_: object) -> None: + """Close the server's stdin and wait for it to exit, killing it first on an error.""" if error_type is not None: self.kill() with suppress(OSError): @@ -279,6 +303,10 @@ def run(self, arguments: Sequence[str], output: Path) -> int | None: @contextmanager def snapshot(ref: str) -> Iterator[Revision]: + """Extract the matrix inputs at git ``ref`` into a temporary directory and yield its Revision. + + Raise ValueError when a blob cannot be read or the configs needed to generate are missing. + """ repository = git_repository_root() listing = subprocess.run( [ @@ -338,6 +366,7 @@ def snapshot(ref: str) -> Iterator[Revision]: def main(argv: Sequence[str] | None = None) -> None: + """Run a checkout's generator or planner with its own configs and exit with its status.""" parser = argparse.ArgumentParser( description="Run a checkout's own matrix generator or planner against its own configs." ) diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index e7c5d0011a..8d0f08d063 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -104,6 +104,7 @@ class ComponentMetadata(BaseModel): @field_validator("version") @classmethod def validate_component_version(cls, version: str) -> str: + """Return ``version``, rejecting an ``image:`` reference with ValueError.""" if version.startswith("image:"): raise ValueError( "component version must be a release, package version, or " @@ -121,6 +122,7 @@ class KVOffloadBackendMetadata(BaseModel): @field_validator("version") @classmethod def validate_component_version(cls, version: str | None) -> str | None: + """Return ``version``, rejecting an ``image:`` reference with ValueError.""" if version is not None and version.startswith("image:"): raise ValueError( "component version must be a release, package version, or " @@ -130,6 +132,7 @@ def validate_component_version(cls, version: str | None) -> str | None: def _validate_tp_context_topology(self: Any) -> Any: + """Return the model, raising ValueError unless its TP is divisible by its DCP size.""" if self.tp % self.dcp_size != 0: raise ValueError( f"'{Fields.TP.value}' ({self.tp}) must be divisible by " @@ -174,6 +177,7 @@ class SingleNodeMatrixEntry(BaseModel): @model_validator(mode="after") def validate_single_node_topology(self) -> Self: + """Return the entry, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) @@ -195,6 +199,7 @@ class WorkerConfig(BaseModel): @model_validator(mode="after") def validate_worker_topology(self) -> Self: + """Return the worker, raising ValueError for an invalid DWDP or TP and DCP topology.""" if self.dwdp_size > 1 and ( self.num_worker <= 0 or (self.tp, self.ep, self.pp, self.dcp_size, self.pcp_size) != (1, 1, 1, 1, 1) @@ -223,10 +228,12 @@ class AggregateWorkerConfig(BaseModel): @model_validator(mode="after") def validate_worker_topology(self) -> Self: + """Return the worker, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) def _validate_worker_hardware_pair(self: Any) -> Any: + """Return the model, raising ValueError for DWDP on decode or hardware set on one side only.""" if self.decode.dwdp_size > 1: raise ValueError("DWDP is supported only on prefill workers") if bool(self.prefill.hardware) != bool(self.decode.hardware): @@ -280,10 +287,12 @@ class MultiNodeMatrixEntry(BaseModel): @model_validator(mode="after") def validate_worker_hardware_pair(self) -> Self: + """Return the entry, raising ValueError unless its workers' hardware fields agree.""" return _validate_worker_hardware_pair(self) @model_validator(mode="after") def validate_disagg_transfer(self) -> Self: + """Return the entry, raising ValueError if it is disaggregated without a KV transfer.""" if self.disagg and self.kv_p2p_transfer is None: raise ValueError(f"{Fields.DISAGG.value}=true requires {Fields.KV_P2P_TRANSFER.value}") return self @@ -330,10 +339,12 @@ class SingleNodeAgenticMatrixEntry(BaseModel): @model_validator(mode="after") def validate_kv_offload_fields(self) -> Self: + """Return the entry, raising ValueError for inconsistent KV offload fields.""" return _validate_kv_offload_fields(self) @model_validator(mode="after") def validate_single_node_topology(self) -> Self: + """Return the entry, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) @@ -383,14 +394,17 @@ class MultiNodeAgenticMatrixEntry(BaseModel): @model_validator(mode="after") def validate_worker_hardware_pair(self) -> Self: + """Return the entry, raising ValueError unless its workers' hardware fields agree.""" return _validate_worker_hardware_pair(self) @model_validator(mode="after") def validate_kv_offload_fields(self) -> Self: + """Return the entry, raising ValueError for inconsistent KV offload fields.""" return _validate_kv_offload_fields(self) @model_validator(mode="after") def validate_disagg_transfer(self) -> Self: + """Return the entry, raising ValueError if it is disaggregated without a KV transfer.""" if self.disagg and self.kv_p2p_transfer is None: raise ValueError(f"{Fields.DISAGG.value}=true requires {Fields.KV_P2P_TRANSFER.value}") return self @@ -400,6 +414,7 @@ def validate_disagg_transfer(self) -> Self: def validate_agentic_matrix_entry(entry: dict) -> dict: + """Return an agentic matrix row unchanged, raising ValueError with the row if it is invalid.""" try: if Fields.PREFILL.value in entry: MultiNodeAgenticMatrixEntry(**entry) @@ -413,6 +428,7 @@ def validate_agentic_matrix_entry(entry: dict) -> dict: def validate_matrix_entry(entry: dict, is_multinode: bool) -> dict: + """Return a fixed-sequence matrix row unchanged, raising ValueError with the row if invalid.""" try: if is_multinode: MultiNodeMatrixEntry(**entry) @@ -426,6 +442,7 @@ def validate_matrix_entry(entry: dict, is_multinode: bool) -> dict: def _validate_conc_fields(self: Any) -> Any: + """Return the model, raising ValueError unless it has exactly one valid conc range or list.""" has_range = self.conc_start is not None and self.conc_end is not None has_list = self.conc_list is not None and len(self.conc_list) > 0 @@ -471,6 +488,7 @@ def _validate_conc_fields(self: Any) -> Any: def _validate_agentic_runner_is_cluster(runner: str, scenarios: Any) -> None: + """Raise ValueError if an agentic config's ``runner`` is not a cluster label.""" if scenarios.agentic_coding and not runner.startswith(CLUSTER_LABEL_PREFIX): raise ValueError( f"Agentic master configs must use a '{CLUSTER_LABEL_PREFIX}' runner " @@ -479,6 +497,7 @@ def _validate_agentic_runner_is_cluster(runner: str, scenarios: Any) -> None: def _validate_kv_offload_fields(self: Any) -> Any: + """Return the model, raising ValueError unless a KV offload backend comes with offloading.""" backend = getattr(self, "kv_offload_backend", None) if self.kv_offloading is None: if backend is not None: @@ -502,6 +521,7 @@ def _validate_kv_offload_fields(self: Any) -> Any: def _recipe_path(value: str) -> str: + """Return ``value``, raising ValueError unless it is a normalized relative POSIX path.""" path = PurePosixPath(value) if ( not path.parts @@ -515,6 +535,7 @@ def _recipe_path(value: str) -> str: def _recipe_reference(value: str) -> str: + """Return a ``path[:selector]`` recipe reference, raising ValueError for a bad part.""" path, separator, selector = value.partition(":") _recipe_path(path) if separator and not RECIPE_SELECTOR.fullmatch(selector): @@ -548,10 +569,12 @@ class SingleNodeSearchSpaceEntry(BaseModel): @model_validator(mode="after") def validate_conc_fields(self) -> Self: + """Return the entry, raising ValueError for an invalid conc range or list.""" return _validate_conc_fields(self) @model_validator(mode="after") def validate_single_node_topology(self) -> Self: + """Return the entry, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) @@ -580,10 +603,12 @@ class MultiNodeSearchSpaceEntry(BaseModel): @model_validator(mode="after") def validate_conc_fields(self) -> Self: + """Return the entry, raising ValueError for an invalid conc range or list.""" return _validate_conc_fields(self) @model_validator(mode="after") def validate_worker_hardware_pair(self) -> Self: + """Return the entry, raising ValueError unless it has one worker or a matching pair.""" has_worker = self.worker is not None has_any_disagg_worker = self.prefill is not None or self.decode is not None has_complete_disagg_workers = self.prefill is not None and self.decode is not None @@ -653,14 +678,17 @@ class AgenticCodingSearchSpaceEntry(BaseModel): @model_validator(mode="after") def validate_conc_fields(self) -> Self: + """Return the entry, raising ValueError for an invalid conc range or list.""" return _validate_conc_fields(self) @model_validator(mode="after") def validate_kv_offload_fields(self) -> Self: + """Return the entry, raising ValueError for inconsistent KV offload fields.""" return _validate_kv_offload_fields(self) @model_validator(mode="after") def validate_topology_fields(self) -> Self: + """Return the entry, raising ValueError unless it sets exactly one valid topology.""" has_single_node = self.tp is not None has_aggregate_worker = self.worker is not None has_any_multinode_field = self.prefill is not None or self.decode is not None @@ -722,6 +750,7 @@ class AgenticCodingConfig(BaseModel): @model_validator(mode="after") def validate_dram_offload_capacity(self) -> Self: + """Return the config, raising ValueError for DRAM offload it cannot size.""" for entry in self.search_space: if entry.kv_offloading != "dram": continue @@ -750,6 +779,7 @@ class SingleNodeScenarios(BaseModel): @model_validator(mode="after") def at_least_one_scenario(self) -> Self: + """Return the scenarios, raising ValueError when none is set.""" if not self.fixed_seq_len and not self.agentic_coding: raise ValueError("At least one scenario type must be specified") return self @@ -767,12 +797,14 @@ class MultiNodeScenarios(BaseModel): @model_validator(mode="after") def at_least_one_scenario(self) -> Self: + """Return the scenarios, raising ValueError when none is set.""" if not self.fixed_seq_len and not self.agentic_coding: raise ValueError("At least one scenario type must be specified") return self def _master_search_space_entries(self: BaseModel) -> list[BaseModel]: + """Return every search-space entry across a master config's scenarios.""" return [ entry for scenario_configs in ( @@ -785,6 +817,7 @@ def _master_search_space_entries(self: BaseModel) -> list[BaseModel]: def _validate_component_metadata_scope(self: BaseModel) -> BaseModel: + """Return the config, raising ValueError if router or KV transfer settings are misplaced.""" search_space_entries = _master_search_space_entries(self) for field in (Fields.ROUTER, Fields.KV_P2P_TRANSFER): @@ -826,6 +859,7 @@ def _validate_component_metadata_scope(self: BaseModel) -> BaseModel: def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: + """Return the config, raising ValueError if an entry's workers do not fit its node type.""" search_space_entries = _master_search_space_entries(self) for entry in search_space_entries: worker = getattr(entry, "worker", None) @@ -874,6 +908,7 @@ def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: def _validate_srt_recipes(self: Any) -> Any: + """Return the config, raising ValueError unless every entry names a valid srt recipe.""" entries = _master_search_space_entries(self) if self.srt_recipe_dir is None: raise ValueError(f"srt-slurm configs require {Fields.SRT_RECIPE_DIR.value}") @@ -910,19 +945,23 @@ class SingleNodeMasterConfigEntry(BaseModel): @model_validator(mode="after") def validate_agentic_runner(self) -> Self: + """Return the config, raising ValueError if agentic scenarios use a non-cluster runner.""" _validate_agentic_runner_is_cluster(self.runner, self.scenarios) return self @model_validator(mode="after") def validate_component_metadata_scope(self) -> Self: + """Return the config, raising ValueError if router or KV transfer settings are misplaced.""" return _validate_component_metadata_scope(self) @model_validator(mode="after") def validate_multinode_entry_scope(self) -> Self: + """Return the config, raising ValueError if an entry's workers do not fit its node type.""" return _validate_multinode_entry_scope(self) @model_validator(mode="after") def validate_srt_recipes(self) -> Self: + """Return the config, raising ValueError unless every entry names a valid srt recipe.""" return _validate_srt_recipes(self) @@ -946,27 +985,33 @@ class MultiNodeMasterConfigEntry(BaseModel): @model_validator(mode="after") def validate_agentic_runner(self) -> Self: + """Return the config, raising ValueError if agentic scenarios use a non-cluster runner.""" _validate_agentic_runner_is_cluster(self.runner, self.scenarios) return self @model_validator(mode="after") def validate_component_metadata_scope(self) -> Self: + """Return the config, raising ValueError if router or KV transfer settings are misplaced.""" return _validate_component_metadata_scope(self) @model_validator(mode="after") def validate_multinode_entry_scope(self) -> Self: + """Return the config, raising ValueError if an entry's workers do not fit its node type.""" return _validate_multinode_entry_scope(self) @model_validator(mode="after") def validate_srt_recipes(self) -> Self: + """Return the config, raising ValueError unless every entry names a valid srt recipe.""" return _validate_srt_recipes(self) def srt_recipe_path(multinode: bool, directory: str, reference: str) -> str: + """Return the repository path of a recipe ``reference`` in its single- or multinode dir.""" return f"{MULTI_NODE_RECIPES if multinode else SINGLE_NODE_RECIPES}/{directory}/{reference}" def _zip_length(group: Any) -> int: + """Return the variant count of a zip override group, its longest list.""" if isinstance(group, list): return len(group) if isinstance(group, dict): @@ -975,6 +1020,10 @@ def _zip_length(group: Any) -> int: def _check_srt_recipe(root: Path, reference: str, recipes: dict[str, Any]) -> None: + """Raise ValueError unless the recipe file exists, parses, and has the selected variant. + + Parsed recipes are cached in ``recipes`` by path. + """ path, _, selector = reference.partition(":") if path not in recipes: if not (root / path).is_file(): @@ -994,6 +1043,7 @@ def _check_srt_recipe(root: Path, reference: str, recipes: dict[str, Any]) -> No def config_root(config_files: list[str]) -> Path: + """Return the project directory shared by the config files, or raise ValueError.""" roots = {Path(config_file).resolve().parent.parent for config_file in config_files} if len(roots) != 1: raise ValueError(f"Master configs must share one project directory: {sorted(roots)}") @@ -1001,6 +1051,10 @@ def config_root(config_files: list[str]) -> Path: def srt_recipe_references(master_configs: dict) -> dict[str, list[str]]: + """Return the srt recipe paths each master config key references. + + Raise ValueError when a master config entry fails validation. + """ references = {} for key, entry in master_configs.items(): model = ( @@ -1022,6 +1076,7 @@ def srt_recipe_references(master_configs: dict) -> dict[str, list[str]]: def validate_master_config(master_configs: dict, root: Path) -> dict: + """Return the master configs, raising ValueError if an entry or its recipes are invalid.""" recipes: dict[str, Any] = {} for key, references in srt_recipe_references(master_configs).items(): for reference in references: @@ -1033,6 +1088,7 @@ def validate_master_config(master_configs: dict, root: Path) -> dict: def _validate_runner_labels(labels: dict) -> None: + """Raise ValueError unless every runner label maps to a non-empty list of strings.""" for key, value in labels.items(): if not isinstance(value, list): raise ValueError( @@ -1047,6 +1103,7 @@ def _validate_runner_labels(labels: dict) -> None: def validate_runner_config(runner_configs: dict) -> dict: + """Return the runner config, raising ValueError if its labels or clusters are invalid.""" labels = runner_configs.get("labels") if not isinstance(labels, dict): raise ValueError("Runner config must define a labels mapping") @@ -1093,6 +1150,7 @@ class ChangelogEntry(BaseModel): @model_validator(mode="after") def validate_append_only_mode(self) -> Self: + """Return the entry, raising ValueError if no-evals or append-only selects evals.""" if self.no_evals and ( self.evals_only or self.all_evals or self.eval_min_prefill_ep is not None ): @@ -1129,6 +1187,10 @@ class ChangelogMatrixEntry(BaseModel): def load_config_files(config_files: list[str], validate: bool = True) -> dict: + """Load and merge the master config files, validating them unless ``validate`` is False. + + Raise ValueError for a missing file, bad or duplicate keys, or an invalid config. + """ all_config_data = {} for config_file in config_files: try: @@ -1164,6 +1226,10 @@ def load_config_files(config_files: list[str], validate: bool = True) -> dict: def load_runner_file(runner_file: str, validate: bool = True) -> dict: + """Load the runner config, validating it unless ``validate`` is False. + + Raise ValueError when the file is missing or invalid. + """ try: with open(runner_file) as f: runner_config = yamlio.safe_load(f) From f6600f4b6b0ac198df77a89c7f7d6a6fbad95f02 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:32:23 -0500 Subject: [PATCH 194/203] docs(results): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give every function and method in infx/results a docstring that states its inputs and result, and remove the remaining comments, folding the useful ones into docstrings. 中文:为 infx/results 中的每个函数和方法添加说明输入与输出的文档字符串,并删除其余注释,将有用的注释并入文档字符串。 --- .../infx/results/agentic/__init__.py | 13 ++++++ .../analyze_benchmark_distributions.py | 4 ++ .../infx/results/agentic/artifacts.py | 11 +++++ .../infx/results/agentic/backends/__init__.py | 1 + .../infx/results/agentic/backends/atom.py | 2 + .../infx/results/agentic/backends/base.py | 7 +++ .../results/agentic/backends/dynamo_vllm.py | 4 ++ .../infx/results/agentic/backends/sglang.py | 8 ++++ .../infx/results/agentic/backends/trtllm.py | 7 +++ .../infx/results/agentic/backends/vllm.py | 9 ++++ .../results/agentic/collect_sweep_results.py | 3 ++ inferencex-e2e/infx/results/agentic/common.py | 13 ++++++ .../infx/results/agentic/power_adapter.py | 7 +++ .../results/agentic/process_agentic_result.py | 1 + .../infx/results/agentic/request_metrics.py | 14 ++++++ .../infx/results/agentic/server_metrics.py | 4 ++ .../agentic/validate_agentic_result.py | 1 + inferencex-e2e/infx/results/artifacts.py | 12 ++--- .../infx/results/collect_eval_results.py | 2 + inferencex-e2e/infx/results/collect_events.py | 8 ++-- .../infx/results/collect_results.py | 1 + .../infx/results/compare_results.py | 2 + inferencex-e2e/infx/results/eval_artifacts.py | 3 ++ inferencex-e2e/infx/results/evals.py | 11 +++-- inferencex-e2e/infx/results/fixed_sequence.py | 5 +++ inferencex-e2e/infx/results/identity.py | 6 ++- inferencex-e2e/infx/results/power/common.py | 1 + .../infx/results/power/multinode.py | 44 +++++++++++++++++++ .../infx/results/result_filename.py | 10 +++++ 29 files changed, 201 insertions(+), 13 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 5cf7469691..67ada94c91 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -13,6 +13,7 @@ def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: + """Return ``env[name]`` as an int, or ``default`` when unset or empty.""" value = env.get(name) if value in (None, ""): return default @@ -20,6 +21,7 @@ def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: def _env_bool(env: Mapping[str, str], name: str, default: bool = False) -> bool: + """Return whether ``env[name]`` is a truthy word, or ``default`` when unset or empty.""" value = env.get(name) if value in (None, ""): return default @@ -27,6 +29,7 @@ def _env_bool(env: Mapping[str, str], name: str, default: bool = False) -> bool: def _required_env(env: Mapping[str, str], name: str) -> str: + """Return ``env[name]``, raising SystemExit when it is unset or empty.""" value = env.get(name) if value in (None, ""): raise SystemExit(f"Missing required environment variable: {name}") @@ -34,6 +37,7 @@ def _required_env(env: Mapping[str, str], name: str) -> str: def _optional_component_metadata(env: Mapping[str, str], env_name: str) -> dict[str, str] | None: + """Parse the component metadata JSON in ``env[env_name]``, or None when unset.""" return parse_component_metadata( env.get(env_name), env_name, @@ -44,6 +48,7 @@ def _optional_component_metadata(env: Mapping[str, str], env_name: str) -> dict[ def _optional_kv_offload_backend_metadata( env: Mapping[str, str], env_name: str ) -> dict[str, str] | None: + """Parse the KV offload backend metadata in ``env[env_name]``, where the version may be absent.""" return parse_component_metadata( env.get(env_name), env_name, @@ -55,6 +60,10 @@ def _optional_kv_offload_backend_metadata( def _validate_kv_offload_env( env: Mapping[str, str], ) -> tuple[str, dict[str, str] | None]: + """Return KV_OFFLOADING and the backend metadata from ``env``. + + Raises SystemExit when the backend name and metadata disagree with the offload mode. + """ kv_offloading = _required_env(env, "KV_OFFLOADING") backend_name = env.get("KV_OFFLOAD_BACKEND", "") backend_metadata = _optional_kv_offload_backend_metadata(env, "KV_OFFLOAD_BACKEND_METADATA") @@ -70,6 +79,10 @@ def _validate_kv_offload_env( def _gpu_shape(env: Mapping[str, str]) -> tuple[dict[str, Any], int, int, int, str]: + """Derive parallelism fields, GPU count, TP, EP and DP-attention from ``env``. + + Multinode runs sum the prefill and decode workers. Raises SystemExit on invalid shapes. + """ is_multinode = _env_bool(env, "IS_MULTINODE") tp = _env_int(env, "TP", 1) ep = _env_int(env, "EP_SIZE", 1) diff --git a/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py b/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py index f985c9009b..978f0802ad 100644 --- a/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py +++ b/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py @@ -21,6 +21,7 @@ def load_records(artifacts_dir: Path) -> list[dict]: def _stats(values: list[int]) -> dict[str, float]: + """Return count, mean, median, p75, p90 and p95 of the integer ``values``.""" sv = sorted(values) n = len(sv) return { @@ -34,6 +35,7 @@ def _stats(values: list[int]) -> dict[str, float]: def _fmt(s: dict[str, float]) -> str: + """Format a ``_stats`` dict as one indented summary line.""" return ( f" n={s['n']:,} mean={s['mean']:,.0f} median={s['median']:,} " f"p75={s['p75']:,} p90={s['p90']:,} p95={s['p95']:,}" @@ -41,6 +43,7 @@ def _fmt(s: dict[str, float]) -> str: def analyze(records: list[dict], output_dir: Path) -> None: + """Print ISL/OSL stats for ``records`` and write them to workload_distribution_summary.txt in ``output_dir``.""" output_dir.mkdir(parents=True, exist_ok=True) all_isl: list[int] = [] @@ -77,6 +80,7 @@ def analyze(records: list[dict], output_dir: Path) -> None: def main() -> None: + """Parse the artifacts dir and output dir from argv and write the distribution summary.""" parser = argparse.ArgumentParser(description="Analyze benchmark workload distributions") parser.add_argument("artifacts_dir", help="Path to aiperf_artifacts/ directory") parser.add_argument( diff --git a/inferencex-e2e/infx/results/agentic/artifacts.py b/inferencex-e2e/infx/results/agentic/artifacts.py index a35ee829ae..dfe8a52bcd 100644 --- a/inferencex-e2e/infx/results/agentic/artifacts.py +++ b/inferencex-e2e/infx/results/agentic/artifacts.py @@ -21,11 +21,13 @@ def resolve_artifact_dir(result_dir: Path) -> Path: def load_aggregate(path: Path) -> dict[str, Any]: + """Return the JSON object stored at ``path``.""" with open(path) as f: return json.load(f) def load_records(path: Path) -> list[dict[str, Any]]: + """Return the profiled, error-free records from the profile_export.jsonl at ``path``.""" records, _ = load_records_with_accounting(path) return records @@ -73,6 +75,7 @@ def load_records_with_accounting( def _error_category(error: Any) -> str: + """Reduce a record's ``error`` field to a short category name, ``"unknown"`` when empty.""" if isinstance(error, dict): for key in ("type", "error_type", "code", "class", "status"): value = error.get(key) @@ -90,6 +93,7 @@ def _error_category(error: Any) -> str: def load_server_metrics(path: Path) -> dict[str, Any]: + """Return the server metrics JSON at ``path``, or an empty dict when the file is missing.""" if not path.exists(): return {} with open(path) as f: @@ -97,6 +101,7 @@ def load_server_metrics(path: Path) -> dict[str, Any]: def load_server_log_head(path: Path, max_bytes: int = 64 * 1024 * 1024) -> str | None: + """Return up to ``max_bytes`` of the log at ``path`` as text without NUL bytes, or None if missing.""" if not path.exists(): return None with open(path, "rb") as f: @@ -105,6 +110,7 @@ def load_server_log_head(path: Path, max_bytes: int = 64 * 1024 * 1024) -> str | def find_server_log_paths(result_dir: Path) -> list[Path]: + """List server.log in ``result_dir`` and watchtower-*.out files from it and three parents, deduplicated.""" paths: list[Path] = [] direct = result_dir / "server.log" if direct.is_file(): @@ -127,6 +133,10 @@ def find_server_log_paths(result_dir: Path) -> list[Path]: def _hf_traces_dir(hf_dataset_name: str | None, env: Mapping[str, str]) -> Path | None: + """Return the single cached snapshot dir of the HF dataset named ``hf_dataset_name``, else None. + + The hub cache root comes from HF_HUB_CACHE, HUGGINGFACE_HUB_CACHE or HF_HOME in ``env``. + """ if not hf_dataset_name: return None @@ -146,6 +156,7 @@ def _hf_traces_dir(hf_dataset_name: str | None, env: Mapping[str, str]) -> Path def _iter_trace_blobs(traces_dir: Path) -> Iterator[dict[str, Any]]: + """Yield every JSON object from the *.jsonl and *.json files in ``traces_dir``, skipping unreadable ones.""" for path in sorted(traces_dir.glob("*.jsonl")): try: with open(path) as f: diff --git a/inferencex-e2e/infx/results/agentic/backends/__init__.py b/inferencex-e2e/infx/results/agentic/backends/__init__.py index 0cde104ff8..d873d9c4b3 100644 --- a/inferencex-e2e/infx/results/agentic/backends/__init__.py +++ b/inferencex-e2e/infx/results/agentic/backends/__init__.py @@ -24,6 +24,7 @@ def detect_backend( metrics: dict[str, dict[str, Any]], framework: str, ) -> ServerMetricsBackend | None: + """Return the first backend that claims ``metrics`` for ``framework``, or None.""" for backend in BACKENDS: if backend.matches(metrics, framework): return backend diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 947024b111..9e5c661de2 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -23,6 +23,7 @@ class AtomBackend(ServerMetricsBackend): name = "atom" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for atom metric names, or for an empty scrape when ``framework`` is atom.""" metric_names = set(metrics) return any(name.startswith(("atom:", "atom_")) for name in metric_names) or ( not metrics and framework.lower() == "atom" @@ -34,6 +35,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, prefix and LMCache hit rates and KV usage from ATOM ``metrics``.""" prompt_total = sum_stat( metrics, _atom_counter_names("prompt_tokens"), diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 427a4c1606..8aafdc0885 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -28,6 +28,7 @@ class ServerMetricsBackend: name = "" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return whether this adapter handles ``metrics`` scraped from ``framework``.""" raise NotImplementedError def populate( @@ -36,6 +37,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill the ``flat`` and ``nested`` server metric dicts in place from ``metrics``.""" raise NotImplementedError def gpu_kv_capacity_tokens( @@ -43,10 +45,12 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], # noqa: ARG002 server_logs: Iterable[str | None], # noqa: ARG002 ) -> int | None: + """Return the total GPU KV cache capacity in tokens from ``metrics`` or ``server_logs``, None by default.""" return None def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: + """Return the nested server metrics skeleton with every value None, tagged ``present`` and ``metric_count``.""" return { "present": present, "adapter": "none", @@ -98,6 +102,7 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: def apply_profile_totals(flat: dict[str, Any], records: list[dict[str, Any]]) -> None: + """Fill missing prompt and generation totals in ``flat`` from per-record lengths and set the completed count.""" input_tokens = _record_token_sum(records, "input_sequence_length") output_tokens = _record_token_sum(records, "output_sequence_length") if flat["total_prompt_tokens"] is None and input_tokens is not None: @@ -108,12 +113,14 @@ def apply_profile_totals(flat: dict[str, Any], records: list[dict[str, Any]]) -> def counter_int(value: float | None) -> int | None: + """Round a counter ``value`` to an int, keeping None.""" if value is None: return None return round(value) def _record_token_sum(records: list[dict[str, Any]], metric_name: str) -> int | None: + """Sum the integer ``metric_name`` values across ``records``, or None when no record has one.""" total = 0 found = False for record in records: diff --git a/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py b/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py index 6edae36cbb..5e09446461 100644 --- a/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py @@ -13,6 +13,7 @@ class DynamoVllmBackend(VllmBackend): name = "dynamo-vllm" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for dynamo frameworks or dynamo_ metrics, except dynamo-sglang with sglang metrics.""" metric_names = set(metrics) framework = framework.lower() if framework == "dynamo-sglang" and any( @@ -27,6 +28,7 @@ def prompt_generation_totals( self, metrics: dict[str, dict[str, Any]], ) -> tuple[float | None, float | None]: + """Return prompt and generation token totals, preferring Dynamo frontend counters over vLLM ones.""" prompt_total = first_counter_total( metrics, ["dynamo_frontend_input_sequence_tokens", "vllm:prompt_tokens"], @@ -43,6 +45,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill the vLLM fields, then the Dynamo frontend and router fields, from ``metrics``.""" super().populate(metrics, flat, nested) self._populate_dynamo(metrics, flat, nested) @@ -52,6 +55,7 @@ def _populate_dynamo( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Add Dynamo GPU usage, frontend and router hit rates where vLLM metrics left gaps.""" dynamo_gpu_usage = normalize_fraction( gauge_stat( metrics, diff --git a/inferencex-e2e/infx/results/agentic/backends/sglang.py b/inferencex-e2e/infx/results/agentic/backends/sglang.py index c49d0ea038..a6990a16d0 100644 --- a/inferencex-e2e/infx/results/agentic/backends/sglang.py +++ b/inferencex-e2e/infx/results/agentic/backends/sglang.py @@ -26,6 +26,7 @@ class SglangBackend(ServerMetricsBackend): _DP_SIZE_RE = re.compile(r"\bdp_size=(?P\d+)\b") def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for sglang: metric names, or for an empty scrape when ``framework`` is sglang.""" metric_names = set(metrics) return any(name.startswith("sglang:") for name in metric_names) or ( not metrics and framework.lower() == "sglang" @@ -37,6 +38,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, cache hit rates by source and GPU/host KV usage from SGLang ``metrics``.""" prompt_total = sum_stat( metrics, "sglang:prompt_tokens", @@ -146,6 +148,7 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], # noqa: ARG002 server_logs: Iterable[str | None], ) -> int | None: + """Return the summed SGLang KV pool size parsed from ``server_logs``, ignoring ``metrics``.""" return sum_server_log_capacities( server_logs, self.kv_cache_pool_tokens_from_server_log, @@ -153,6 +156,10 @@ def gpu_kv_capacity_tokens( @classmethod def kv_cache_pool_tokens_from_server_log(cls, server_log: str | None) -> int | None: + """Return the KV pool tokens from max_total_num_tokens lines in ``server_log``, or None. + + Sums distinct DP/TP/EP ranks and scales a single logged rank by dp_size. + """ if not server_log: return None @@ -187,6 +194,7 @@ def kv_cache_pool_tokens_from_server_log(cls, server_log: str | None) -> int | N @classmethod def _dp_size(cls, server_log: str) -> int | None: + """Return the positive dp_size logged in ``server_log``, or None.""" match = cls._DP_SIZE_RE.search(server_log) if not match: return None diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index aba19a03c6..cb85924d28 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -20,6 +20,7 @@ class TrtllmBackend(ServerMetricsBackend): name = "trtllm" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for trtllm_ metric names, or for an empty scrape from a TRT-LLM ``framework``.""" metric_names = set(metrics) return any(name.startswith("trtllm_") for name in metric_names) or ( not metrics and framework.lower() in ("trtllm", "dynamo-trt", "dynamo-trtllm") @@ -31,6 +32,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, KV hit rate, usage and offload bytes and per-endpoint sources from ``metrics``.""" prompt_total = _first_counter_total( metrics, ["dynamo_frontend_input_sequence_tokens", "trtllm_prompt_tokens_total"], @@ -130,6 +132,7 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], server_logs: Iterable[str | None], ) -> int | None: + """Return KV max blocks times tokens per block from ``metrics``, or None if either is missing.""" del server_logs max_blocks = sum_stat( metrics, @@ -151,6 +154,7 @@ def _first_counter_total( metrics: dict[str, dict[str, Any]], metric_names: list[str], ) -> float | None: + """Return the first non-None counter total among ``metric_names`` in ``metrics``.""" for metric_name in metric_names: value = sum_stat( metrics, @@ -163,6 +167,7 @@ def _first_counter_total( def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Return per-endpoint token, hit rate and KV usage rows from TRT-LLM ``metrics``, sorted by endpoint.""" endpoints: set[str] = set() roles: dict[str, str] = {} for entry in metrics.values(): @@ -186,6 +191,7 @@ def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: for endpoint in sorted(endpoints): def series_filter(series: dict, endpoint: str = endpoint) -> bool: + """Return whether ``series`` was scraped from ``endpoint``.""" return str(series.get("endpoint_url", "")) == endpoint prompt_tokens = sum_stat( @@ -233,6 +239,7 @@ def series_filter(series: dict, endpoint: str = endpoint) -> bool: def _normalize_role(mode: str) -> str: + """Map a disaggregation ``mode`` label to prefill, decode or combined, else return it lowercased.""" lowered = mode.lower() if lowered in ("prefill", "context", "ctx"): return "prefill" diff --git a/inferencex-e2e/infx/results/agentic/backends/vllm.py b/inferencex-e2e/infx/results/agentic/backends/vllm.py index 5baf5a47f2..bbafcfd2b8 100644 --- a/inferencex-e2e/infx/results/agentic/backends/vllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/vllm.py @@ -26,6 +26,7 @@ class VllmBackend(ServerMetricsBackend): _GPU_KV_SIZE_RE = re.compile(r"GPU KV cache size:\s*(?P[\d,]+)\s*tokens") def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for vllm: metric names, or for an empty scrape when ``framework`` is vllm.""" metric_names = set(metrics) return any(name.startswith("vllm:") for name in metric_names) or ( not metrics and framework.lower() == "vllm" @@ -37,6 +38,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, prefix and external hit rates, KV usage, offload stats and sources from ``metrics``.""" prompt_total, generation_total = self.prompt_generation_totals(metrics) flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) @@ -177,6 +179,7 @@ def prompt_generation_totals( self, metrics: dict[str, dict[str, Any]], ) -> tuple[float | None, float | None]: + """Return the vLLM prompt and generation token totals from ``metrics``.""" prompt_total = sum_stat( metrics, "vllm:prompt_tokens", @@ -194,6 +197,7 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], # noqa: ARG002 server_logs: Iterable[str | None], ) -> int | None: + """Return the summed vLLM GPU KV cache size parsed from ``server_logs``, ignoring ``metrics``.""" return sum_server_log_capacities( server_logs, self.kv_cache_pool_tokens_from_server_log, @@ -201,6 +205,7 @@ def gpu_kv_capacity_tokens( @classmethod def kv_cache_pool_tokens_from_server_log(cls, server_log: str | None) -> int | None: + """Return the GPU KV cache size in tokens from ``server_log``, summed per EngineCore, or None.""" if not server_log: return None @@ -239,6 +244,7 @@ def first_counter_total( metrics: dict[str, dict[str, Any]], metric_names: list[str], ) -> float | None: + """Return the first non-None counter total among ``metric_names`` in ``metrics``.""" for metric_name in metric_names: value = sum_stat( metrics, @@ -251,6 +257,7 @@ def first_counter_total( def _vllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Return per-source token, hit rate and KV usage rows from vLLM ``metrics``, sorted by source id.""" source_ids = set() for metric_name in ( "vllm:prompt_tokens", @@ -269,6 +276,7 @@ def _vllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: continue def series_filter(series: dict, source_id: str = source_id) -> bool: + """Return whether ``series`` belongs to ``source_id``.""" return _source_id(series) == source_id prompt_tokens = sum_stat(metrics, "vllm:prompt_tokens", series_filter=series_filter) @@ -297,6 +305,7 @@ def series_filter(series: dict, source_id: str = source_id) -> bool: def _source_id(series: dict[str, Any]) -> str: + """Build a ``role|endpoint|key=value`` id for ``series`` from its component, endpoint and rank labels.""" labels = series.get("labels") if isinstance(series.get("labels"), dict) else {} component = str(labels.get("dynamo_component") or labels.get("component") or "") if component == "prefill": diff --git a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py index da1ee90af8..1713a9763a 100644 --- a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py +++ b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py @@ -54,6 +54,7 @@ def _load_aiperf_summary_csv(csv_path: Path) -> dict | None: break def metric_stat(metric_name: str, stat: str) -> float: + """Return the ``stat`` column of ``metric_name`` as a float, or 0 when missing or unparsable.""" if metric_name in per_metric: try: return float(per_metric[metric_name].get(stat, 0)) @@ -62,6 +63,7 @@ def metric_stat(metric_name: str, stat: str) -> float: return 0 def scalar_val(metric_name: str) -> float: + """Return the scalar row ``metric_name`` as a float, or 0 when missing or unparsable.""" if metric_name in scalars: try: return float(scalars[metric_name]) @@ -219,6 +221,7 @@ def load_experiment(exp_dir: Path) -> dict | None: def main() -> None: + """Write summary.csv to the output dir for every experiment under the artifacts dir from argv, then plot overviews.""" if len(sys.argv) < 3: print(f"Usage: {sys.argv[0]} ") sys.exit(1) diff --git a/inferencex-e2e/infx/results/agentic/common.py b/inferencex-e2e/infx/results/agentic/common.py index eb498c9fd6..26bb6ea368 100644 --- a/inferencex-e2e/infx/results/agentic/common.py +++ b/inferencex-e2e/infx/results/agentic/common.py @@ -9,6 +9,7 @@ def percentile(data: list[float], p: float) -> float: + """Return the linearly interpolated ``p``th percentile of ``data``, or 0.0 when empty.""" if not data: return 0.0 sorted_data = sorted(data) @@ -21,6 +22,7 @@ def percentile(data: list[float], p: float) -> float: def stats_for(prefix: str, values: list[float]) -> dict[str, float]: + """Return mean, p50, p75, p90, p95 and std of ``values`` keyed with ``prefix``, or {} when empty.""" if not values: return {} return { @@ -34,6 +36,7 @@ def stats_for(prefix: str, values: list[float]) -> dict[str, float]: def to_float(value: Any) -> float | None: + """Return ``value`` as a float, or None when it is None or not numeric.""" if value is None: return None try: @@ -43,6 +46,7 @@ def to_float(value: Any) -> float | None: def to_int(value: Any) -> int | None: + """Return ``value`` as an int, or None when it is None or not convertible.""" if value is None: return None try: @@ -52,6 +56,7 @@ def to_int(value: Any) -> int | None: def rate(numerator: float | None, denominator: float | None) -> float | None: + """Return ``numerator / denominator``, or None when either is None or the denominator is not positive.""" if numerator is None or denominator is None: return None if denominator <= 0: @@ -98,6 +103,7 @@ def metric_series( metrics: dict[str, dict[str, Any]], metric_names: str | list[str], ) -> list[dict[str, Any]]: + """Return every series dict of the named metric or metrics in ``metrics``.""" names = [metric_names] if isinstance(metric_names, str) else metric_names out: list[dict[str, Any]] = [] for name in names: @@ -115,6 +121,7 @@ def series_stat( series: dict[str, Any], preferred_keys: tuple[str, ...] = ("total", "sum", "max", "avg"), ) -> float | None: + """Return the first numeric stat of ``series`` among ``preferred_keys``, or None.""" stats = series.get("stats") if not isinstance(stats, dict): return None @@ -132,6 +139,7 @@ def sum_stat( preferred_keys: tuple[str, ...] = ("total", "sum", "max", "avg"), series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: + """Sum the preferred stat over matching series of ``metric_names``, or None when none has one.""" total = 0.0 found = False for series in metric_series(metrics, metric_names): @@ -153,6 +161,7 @@ def gauge_stat( combine: str = "max", series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: + """Combine the preferred stat of matching series by max, avg or sum per ``combine``, or None.""" values: list[float] = [] for series in metric_series(metrics, metric_names): if series_filter is not None and not series_filter(series): @@ -170,6 +179,7 @@ def gauge_stat( def label_value(series: dict[str, Any], key: str) -> str | None: + """Return the label ``key`` of ``series`` as a string, or None.""" labels = series.get("labels") if not isinstance(labels, dict): return None @@ -180,6 +190,7 @@ def label_value(series: dict[str, Any], key: str) -> str | None: def label_equals(key: str, value: str) -> Callable[[dict[str, Any]], bool]: + """Return a series filter that keeps series whose label ``key`` equals ``value``.""" return lambda series: label_value(series, key) == value @@ -190,6 +201,7 @@ def sum_by_label( *, preferred_keys: tuple[str, ...] = ("total", "sum", "max", "avg"), ) -> dict[str, float]: + """Sum the preferred stat of ``metric_names`` grouped by the first present label among ``label_keys``.""" keys = [label_keys] if isinstance(label_keys, str) else label_keys out: dict[str, float] = {} for series in metric_series(metrics, metric_names): @@ -215,6 +227,7 @@ def sum_server_log_capacities( logs: Iterable[str | None], parser: Callable[[str | None], int | None], ) -> int | None: + """Sum the capacities ``parser`` extracts from each of ``logs``, or None when none parse.""" total = 0 found = False for text in logs: diff --git a/inferencex-e2e/infx/results/agentic/power_adapter.py b/inferencex-e2e/infx/results/agentic/power_adapter.py index 13eaee27ca..59860c74e7 100644 --- a/inferencex-e2e/infx/results/agentic/power_adapter.py +++ b/inferencex-e2e/infx/results/agentic/power_adapter.py @@ -142,6 +142,7 @@ def build_power_window( def _fail_multinode_adapter(message: str, *, require_power: bool) -> int: + """Log ``message`` to stderr and return exit code 1 when ``require_power`` is set, else 0.""" print(f"[agentx_power] {message}", file=sys.stderr) return 1 if require_power else 0 @@ -194,6 +195,7 @@ def _window_payload( end: float | None, duration: float | None, ) -> dict[str, Any]: + """Build the formal custom-benchmark window JSON for ``result_path`` with the given status and times.""" return { "schema_version": 1, "benchmark_type": "custom", @@ -283,6 +285,7 @@ def _record_multinode_adapter_failure( validation_result: Path, reasons: list[str], ) -> None: + """Mark ``agg_result`` power-invalid and always write ``validation_result`` with ``reasons``.""" try: aggregate = json.loads(agg_result.read_text(encoding="utf-8")) if not isinstance(aggregate, dict): @@ -300,6 +303,7 @@ def _record_multinode_adapter_failure( def _write_multinode_failure_validation(validation_result: Path, reasons: list[str]) -> None: + """Write a power-invalid validation JSON with ``reasons`` to ``validation_result``.""" _write_json_atomic( validation_result, { @@ -311,6 +315,7 @@ def _write_multinode_failure_validation(validation_result: Path, reasons: list[s def _formal_result_for_directory(result_dir: Path) -> Path | None: + """Return the single agentic_power_concurrency_*.json in ``result_dir``, or None if absent or ambiguous.""" candidates = [ path for path in result_dir.glob("agentic_power_concurrency_*.json") @@ -320,6 +325,7 @@ def _formal_result_for_directory(result_dir: Path) -> Path | None: def _gpu_count(value: Any) -> int | None: + """Return ``value`` when it is a non-negative int (not bool), else None.""" if isinstance(value, bool) or not isinstance(value, int) or value < 0: return None return value @@ -433,6 +439,7 @@ def run_native_agentic_power( def main() -> int: + """Run the window, missing-contract or power-join mode chosen by argv and return the exit code.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--result-dir", type=Path, required=True) parser.add_argument("--agg-result", type=Path) diff --git a/inferencex-e2e/infx/results/agentic/process_agentic_result.py b/inferencex-e2e/infx/results/agentic/process_agentic_result.py index a44dda3f68..d7dac83644 100644 --- a/inferencex-e2e/infx/results/agentic/process_agentic_result.py +++ b/inferencex-e2e/infx/results/agentic/process_agentic_result.py @@ -23,6 +23,7 @@ def main() -> int: + """Build the AgentX aggregate from RESULT_DIR artifacts, write it to AGENTIC_OUTPUT_DIR and return the exit code.""" result_filename = os.environ.get("RESULT_FILENAME", "") if not result_filename: print("ERROR: RESULT_FILENAME env var not set", file=sys.stderr) diff --git a/inferencex-e2e/infx/results/agentic/request_metrics.py b/inferencex-e2e/infx/results/agentic/request_metrics.py index 66075e50e2..0586e4184a 100644 --- a/inferencex-e2e/infx/results/agentic/request_metrics.py +++ b/inferencex-e2e/infx/results/agentic/request_metrics.py @@ -12,6 +12,7 @@ def _metric_value(record: dict[str, Any], key: str) -> Any: + """Return the value of metric ``key`` in ``record``, unwrapping ``{"value": ...}`` dicts.""" metric = record.get("metrics", {}).get(key) if isinstance(metric, dict): return metric.get("value") @@ -19,6 +20,7 @@ def _metric_value(record: dict[str, Any], key: str) -> Any: def extract_per_record_floats(records: list[dict[str, Any]], key: str) -> list[float]: + """Return the float values of metric ``key`` across ``records``, skipping missing ones.""" out: list[float] = [] for record in records: value = to_float(_metric_value(record, key)) @@ -28,6 +30,7 @@ def extract_per_record_floats(records: list[dict[str, Any]], key: str) -> list[f def extract_per_record_ints(records: list[dict[str, Any]], key: str) -> list[int]: + """Return the int values of metric ``key`` across ``records``, skipping missing ones.""" out: list[int] = [] for record in records: value = to_int(_metric_value(record, key)) @@ -37,6 +40,7 @@ def extract_per_record_ints(records: list[dict[str, Any]], key: str) -> list[int def _ms_to_s(values_ms: Iterable[float]) -> list[float]: + """Convert finite positive millisecond values to seconds, dropping the rest.""" return [ value / 1000.0 for value in values_ms @@ -45,6 +49,7 @@ def _ms_to_s(values_ms: Iterable[float]) -> list[float]: def _distribution(prefix: str, values: list[int]) -> dict[str, float]: + """Return mean, p50, p75, p90, p95 and std of integer ``values`` keyed with ``prefix``, or {} when empty.""" if not values: return {} return { @@ -58,6 +63,7 @@ def _distribution(prefix: str, values: list[int]) -> dict[str, float]: def _nest_stats(prefix: str, flat: dict[str, Any]) -> dict[str, Any]: + """Return the entries of ``flat`` ending in ``_`` with that suffix stripped.""" suffix = f"_{prefix}" return {key[: -len(suffix)]: value for key, value in flat.items() if key.endswith(suffix)} @@ -115,6 +121,7 @@ def _e2e_normalized_interactivity_stats( def compute_latency_stats( records: list[dict[str, Any]], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested TTFT, E2EL, ITL, TPOT and interactivity stats in seconds for ``records``.""" ttfts = _ms_to_s(extract_per_record_floats(records, "time_to_first_token")) e2els = _ms_to_s(extract_per_record_floats(records, "request_latency")) itls = _ms_to_s(extract_per_record_floats(records, "inter_token_latency")) @@ -161,6 +168,7 @@ def compute_latency_stats( def compute_qps_stats( records: list[dict[str, Any]], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested completions-per-second stats over 1 s windows of ``records``' end times.""" ends_ns = [ int(record["metadata"]["request_end_ns"]) for record in records @@ -192,6 +200,7 @@ def compute_qps_stats( def compute_workload_stats( records: list[dict[str, Any]], traces: Iterable[dict[str, Any]] ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested input, actual output and trace-expected output token distributions.""" input_tokens = extract_per_record_ints(records, "input_sequence_length") output_tokens = extract_per_record_ints(records, "output_sequence_length") @@ -214,6 +223,7 @@ def compute_workload_stats( def compute_throughput_stats( records: list[dict[str, Any]], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested input, output and total tokens per second over the ``records`` time span.""" input_tokens = extract_per_record_ints(records, "input_sequence_length") output_tokens = extract_per_record_ints(records, "output_sequence_length") starts_ns = [ @@ -254,6 +264,7 @@ def _aiperf_percent_metric_as_rate( aggregate: dict[str, Any], metric_name: str, ) -> float | None: + """Return aiperf ``metric_name`` from ``aggregate`` as a 0..1 rate, or None when absent.""" metric = aggregate.get(metric_name) if not isinstance(metric, dict): return None @@ -274,6 +285,7 @@ def compute_cache_stats( records: list[dict[str, Any]], # noqa: ARG001 aggregate: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested theoretical prefix cache hit rate from the aiperf ``aggregate``.""" flat: dict[str, Any] = { "theoretical_cache_hit_rate": _aiperf_percent_metric_as_rate( aggregate, @@ -292,6 +304,7 @@ def compute_request_metrics( *, traces: Iterable[dict[str, Any]] = (), ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested QPS, latency, token, cache and throughput metrics for ``records``.""" aggregate = aggregate or {} flat: dict[str, Any] = {} nested: dict[str, Any] = {} @@ -340,6 +353,7 @@ def _trace_output_lengths(traces: Iterable[dict[str, Any]]) -> dict[str, list[in def _expected_output_lengths( records: list[dict[str, Any]], traces: Iterable[dict[str, Any]] ) -> list[int]: + """Return the trace-declared output length for each record's conversation turn, skipping unmatched ones.""" metadata = _trace_output_lengths(traces) if not metadata: return [] diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index d14962a61f..0ba5ffdb55 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -21,6 +21,10 @@ def compute_server_metrics( records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: + """Return flat and nested server metrics plus warnings from aiperf's ``server_metrics`` export. + + Token totals fall back to ``records``. Raises ValueError when no backend adapter matches. + """ metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() warnings: list[str] = [] diff --git a/inferencex-e2e/infx/results/agentic/validate_agentic_result.py b/inferencex-e2e/infx/results/agentic/validate_agentic_result.py index 1d7f76df02..6e5a114c48 100644 --- a/inferencex-e2e/infx/results/agentic/validate_agentic_result.py +++ b/inferencex-e2e/infx/results/agentic/validate_agentic_result.py @@ -89,6 +89,7 @@ def validate_result(artifact_dir: Path, failed_request_threshold: float) -> list def main() -> int: + """Validate the artifact dir from argv against the error threshold and return 1 on any error, else 0.""" parser = argparse.ArgumentParser() parser.add_argument("artifact_dir", type=Path) parser.add_argument( diff --git a/inferencex-e2e/infx/results/artifacts.py b/inferencex-e2e/infx/results/artifacts.py index 2f45b1d232..187ed1b8c5 100644 --- a/inferencex-e2e/infx/results/artifacts.py +++ b/inferencex-e2e/infx/results/artifacts.py @@ -257,7 +257,7 @@ def duplicate_identity_errors( label: str, identities: Iterable[tuple[Any, ...]], ) -> list[str]: - """Reject duplicate rows that set equality would otherwise hide.""" + """Return errors naming each ``identities`` entry seen more than once, which set equality would hide.""" counts = Counter(identities) duplicates = {identity: count for identity, count in counts.items() if count > 1} if not duplicates: @@ -278,7 +278,7 @@ def duplicate_identity_errors( def validate_fixed_artifacts( artifacts_dir: Path, ) -> list[str]: - """Validate fixed-sequence aggregate rows for duplicate identities.""" + """Return duplicate-identity errors for the fixed-sequence rows in ``artifacts_dir``.""" actual_rows = actual_benchmark_key_rows(artifacts_dir) return duplicate_identity_errors("fixed-sequence", actual_rows) @@ -286,7 +286,10 @@ def validate_fixed_artifacts( def validate_agentic_artifacts( artifacts_dir: Path, ) -> list[str]: - """Validate agentic point, raw, and aggregate artifacts agree.""" + """Return errors where the agentic point, raw and aggregate artifacts in ``artifacts_dir`` disagree. + + Raw dirs of attempts replaced by a later rerun still download and are ignored. + """ point_files = agentic_point_files(artifacts_dir) point_rows = agentic_keys_from_paths(point_files) errors = duplicate_identity_errors("agentic point", point_rows) @@ -306,7 +309,6 @@ def validate_agentic_artifacts( point_names = { path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") for path in point_files } - # A rerun attempt's results replace an earlier attempt whose raw dir still downloads. superseded_names = { path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") for path in artifacts_dir.glob("bmk_agentic_*/**/*.json") @@ -328,7 +330,7 @@ def validate_agentic_artifacts( def validate_run_stats(artifacts_dir: Path, required: bool) -> list[str]: - """Require run-stats when fixed-sequence collection should have run.""" + """Return an error when ``required`` and ``artifacts_dir`` holds no run-stats JSON, else [].""" if not required: return [] if list((artifacts_dir / "run-stats").glob("*.json")): diff --git a/inferencex-e2e/infx/results/collect_eval_results.py b/inferencex-e2e/infx/results/collect_eval_results.py index e11d02d7dd..870889727b 100644 --- a/inferencex-e2e/infx/results/collect_eval_results.py +++ b/inferencex-e2e/infx/results/collect_eval_results.py @@ -85,6 +85,7 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> list[Path]: def _read_results(directory: Path) -> dict[Path, dict[str, Any]]: + """Map each readable eval result JSON in ``directory``, except meta_env.json, to its parsed content.""" return read_eval_results( (path for path in directory.glob("*.json") if path.name != "meta_env.json"), skip_errors=(Exception,), @@ -134,6 +135,7 @@ def collect_eval_rows(root: Path) -> list[dict[str, Any]]: def main() -> None: + """Print single-node and multinode eval tables for the results dir in argv and write agg_eval_.json.""" if len(sys.argv) < 3: print("Usage: collect_eval_results.py [sort_by: model_prefix|hw]") sys.exit(1) diff --git a/inferencex-e2e/infx/results/collect_events.py b/inferencex-e2e/infx/results/collect_events.py index f17338c56f..f272f50ff8 100644 --- a/inferencex-e2e/infx/results/collect_events.py +++ b/inferencex-e2e/infx/results/collect_events.py @@ -23,8 +23,6 @@ FILENAME = "job_event.json" SCHEMA_VERSION = 2 NO_RECORD = "NoLauncherRecord" -# The names benchmark-tmpl.yml and benchmark-multinode-tmpl.yml give their jobs: an optional -# klaud marker, the priority, model prefix, precision and runner, then the concurrency last. _BENCHMARK_JOB = re.compile( r"(?:^|/ )(?:klaud \| )?p[0-9.]+ \| (?P\S+) (?P\S+) (?P\S+)" r" .* c(?P[0-9]+(?:x[0-9]+)*)(?: \| (?Peval|eval-only))?$" @@ -51,7 +49,10 @@ def read_events(directory: Path) -> list[dict[str, Any]]: def reconcile( events: list[dict[str, Any]], jobs: Iterable[dict[str, Any]], *, run_url: str ) -> list[dict[str, Any]]: - """``events`` with their jobs' outcomes, then a record for each benchmark job without one.""" + """``events`` with their jobs' outcomes, then a record for each benchmark job without one. + + Benchmark jobs are recognized by the names benchmark-tmpl.yml and benchmark-multinode-tmpl.yml give them. + """ events = [dict(event) for event in events] starts = [_time(event.get("started_at")) for event in events] synthesized = [] @@ -164,6 +165,7 @@ def _time(value: object) -> datetime | None: def main(argv: list[str] | None = None) -> int: + """Write the job events under the artifacts dir in ``argv`` as JSON Lines, reconciled with ``--run`` jobs, and return 0.""" parser = argparse.ArgumentParser( prog="python -m infx.results.collect_events", description=__doc__ ) diff --git a/inferencex-e2e/infx/results/collect_results.py b/inferencex-e2e/infx/results/collect_results.py index 8c748dae73..5ad1c8a631 100644 --- a/inferencex-e2e/infx/results/collect_results.py +++ b/inferencex-e2e/infx/results/collect_results.py @@ -4,6 +4,7 @@ def main() -> None: + """Concatenate every JSON result under the results dir in argv into agg_.json.""" results_dir = Path(sys.argv[1]) exp_name = sys.argv[2] diff --git a/inferencex-e2e/infx/results/compare_results.py b/inferencex-e2e/infx/results/compare_results.py index a38e3619ab..d7b19cafc6 100644 --- a/inferencex-e2e/infx/results/compare_results.py +++ b/inferencex-e2e/infx/results/compare_results.py @@ -11,6 +11,7 @@ def parse_bool(value: object) -> bool: + """Return ``value`` if it is a bool, else whether its string form is ``"true"`` in any case.""" if isinstance(value, bool): return value return str(value).lower() == "true" @@ -184,6 +185,7 @@ def compute_metric_delta( def main() -> None: + """Print markdown tables comparing fixed-sequence results in the argv dir to DATABASE_URL baselines.""" if len(sys.argv) < 2: print("Usage: python compare_results.py ") sys.exit(1) diff --git a/inferencex-e2e/infx/results/eval_artifacts.py b/inferencex-e2e/infx/results/eval_artifacts.py index 1308d697bf..9ef9a09bde 100644 --- a/inferencex-e2e/infx/results/eval_artifacts.py +++ b/inferencex-e2e/infx/results/eval_artifacts.py @@ -195,10 +195,12 @@ class _RawEvalArtifact: @cached_property def results(self) -> dict[Path, dict[str, Any]]: + """Parsed results*.json files of this raw dir, keyed by path.""" return read_eval_results(self.path.glob("results*.json")) @classmethod def read(cls, path: Path) -> _RawEvalArtifact: + """Load the raw eval dir at ``path``, recording meta_env.json problems in ``errors``.""" artifact = cls(path, {}, [], False, []) prefix = f"raw eval artifact {path.name!r}" meta_path = path / "meta_env.json" @@ -405,6 +407,7 @@ def _prune_raw_eval_dir( name = artifact_dir.name def superseded(key: tuple[Any, ...]) -> bool: + """Return whether a dir other than this one won identity ``key``.""" winner = winners.get(key) return winner is not None and winner.parent.name != name diff --git a/inferencex-e2e/infx/results/evals.py b/inferencex-e2e/infx/results/evals.py index 3c1f56bef7..e95fdbfd44 100644 --- a/inferencex-e2e/infx/results/evals.py +++ b/inferencex-e2e/infx/results/evals.py @@ -27,6 +27,7 @@ def read_eval_results( *, skip_errors: tuple[type[Exception], ...] = (OSError, UnicodeDecodeError, json.JSONDecodeError), ) -> dict[Path, dict[str, Any]]: + """Map each of ``paths`` holding a recognized eval result to its JSON, skipping ``skip_errors`` failures.""" results = {} for path in paths: try: @@ -131,6 +132,7 @@ def is_valid_effective_count(value: object) -> bool: def _is_standard_error(value: object) -> bool: + """Return whether ``value`` is a finite non-negative number, excluding booleans.""" return ( not isinstance(value, bool) and isinstance(value, (int, float)) @@ -149,6 +151,7 @@ def metric_family(name: str) -> str | None: def _primary_metric(metrics: dict[str, Any]) -> str | None: + """Return the first of strict, accuracy and flex set in ``metrics``, or None.""" return next((name for name in _SCORE_NAMES if metrics.get(name) is not None), None) @@ -249,7 +252,10 @@ def as_bool(x: Any, default: bool = False) -> bool: def build_row(meta: dict[str, Any], m: dict[str, Any]) -> dict[str, Any]: - """Build a result row from metadata and extracted metrics.""" + """Build a result row from metadata and extracted metrics. + + Parallelism sizes in ``meta`` are copied because raw artifact identities include them. + """ is_multinode = as_bool(meta.get("is_multinode"), False) prefill_tp = as_int(meta.get("prefill_tp", meta.get("tp", 1)), 1) prefill_ep = as_int(meta.get("prefill_ep", meta.get("ep", 1)), 1) @@ -307,7 +313,6 @@ def build_row(meta: dict[str, Any], m: dict[str, Any]) -> dict[str, Any]: if "eval_suite" in meta: row["eval_suite"] = meta["eval_suite"] - # Raw artifact identities include these sizes, so rows must carry them to match. for key in _PARALLELISM_SIZE_KEYS: if key in meta: row[key] = as_int(meta[key], 1) @@ -330,6 +335,7 @@ def build_rows( Primary scores prefer strict, accuracy, then flexible metrics. An invalid primary produces a failed row rather than falling back to a secondary score. + Non-numeric stderrs, such as lm-eval's "N/A" for one-sample metrics, become None. Inputs are not modified; discovery, concurrency selection and writes belong to the caller. """ @@ -346,7 +352,6 @@ def build_rows( "message": f"invalid primary score: {score!r}", } for name in _SCORE_NAMES: - # lm-eval reports a one-sample metric's stderr as the string "N/A". if not _is_standard_error(metrics[f"{name}_se"]): metrics[f"{name}_se"] = None rows.append(build_row(meta, metrics)) diff --git a/inferencex-e2e/infx/results/fixed_sequence.py b/inferencex-e2e/infx/results/fixed_sequence.py index 2a3a511230..e9a19c4adb 100644 --- a/inferencex-e2e/infx/results/fixed_sequence.py +++ b/inferencex-e2e/infx/results/fixed_sequence.py @@ -296,6 +296,10 @@ def aggregate_power_result( def process_result(env: Mapping[str, str]) -> int: + """Write agg_.json from the benchmark JSON and ``env``, join power when configured. + + Returns 1 when power is required and fails or the benchmark failed, else 0. + """ require_environment(env, _BASE_ENV_VARS) result_filename = env["RESULT_FILENAME"] bench_path = Path(f"{result_filename}.json") @@ -409,6 +413,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int: def main() -> int: + """Process every multinode point with ``--all``, else the single result named by the environment.""" return ( process_multinode_results(os.environ) if sys.argv[1:] == ["--all"] diff --git a/inferencex-e2e/infx/results/identity.py b/inferencex-e2e/infx/results/identity.py index 412466707f..cc9ea75ff0 100644 --- a/inferencex-e2e/infx/results/identity.py +++ b/inferencex-e2e/infx/results/identity.py @@ -2,7 +2,6 @@ from __future__ import annotations -# InferenceX-app FRAMEWORK_ALIASES: ingestion stores these config frameworks under other names. FRAMEWORK_ALIASES = { "atom-disagg": "mooncake-atom", "sglang-disagg": "mori-sglang", @@ -13,7 +12,10 @@ def public_framework(framework: str) -> str: - """The framework name the database and public API hold for a config's ``framework``.""" + """The framework name the database and public API hold for a config's ``framework``. + + The aliases mirror InferenceX-app's FRAMEWORK_ALIASES used at ingestion. + """ framework = framework.lower() return FRAMEWORK_ALIASES.get(framework, framework) diff --git a/inferencex-e2e/infx/results/power/common.py b/inferencex-e2e/infx/results/power/common.py index 528a94db65..3a8fb651ee 100644 --- a/inferencex-e2e/infx/results/power/common.py +++ b/inferencex-e2e/infx/results/power/common.py @@ -27,6 +27,7 @@ class BenchmarkData: @property def integration_duration_s(self) -> float: + """Seconds between the benchmark start and end timestamps.""" return self.end_unix - self.start_unix diff --git a/inferencex-e2e/infx/results/power/multinode.py b/inferencex-e2e/infx/results/power/multinode.py index b118c1245e..c857ed4e64 100644 --- a/inferencex-e2e/infx/results/power/multinode.py +++ b/inferencex-e2e/infx/results/power/multinode.py @@ -113,10 +113,12 @@ def _dedupe(values: list[str]) -> tuple[str, ...]: + """Return ``values`` as a tuple without duplicates, keeping first-seen order.""" return tuple(dict.fromkeys(values)) def _is_safe_relative_subpath(value: str) -> bool: + """Return whether ``value`` is a non-empty relative POSIX path with no ``..`` or empty parts.""" if not value or value.startswith(("/", "~")): return False parts = PurePosixPath(value).parts @@ -124,6 +126,7 @@ def _is_safe_relative_subpath(value: str) -> bool: def _stays_below(root: Path, relative: str) -> bool: + """Return whether ``root / relative`` resolves inside ``root``.""" try: (root / relative).resolve().relative_to(root.resolve()) except (ValueError, OSError): @@ -132,10 +135,12 @@ def _stays_below(root: Path, relative: str) -> bool: def _is_finite(value: Any) -> bool: + """Return whether ``value`` is a finite int or float, excluding booleans.""" return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) def _is_positive_finite(value: Any) -> bool: + """Return whether ``value`` is finite and greater than zero.""" return _is_finite(value) and value > 0 @@ -155,6 +160,7 @@ class ExpectedDevice: @property def key(self) -> tuple[str, int]: + """The (hostname, gpu_index) pair identifying this device.""" return (self.hostname, self.gpu_index) @@ -165,6 +171,7 @@ class ExpectedWindow: @property def key(self) -> tuple[str, int]: + """The (benchmark_type, concurrency) pair identifying this window.""" return (self.benchmark_type, self.concurrency) @@ -179,6 +186,7 @@ class SampleRow: @property def key(self) -> tuple[int, str, int]: + """The (scrape_seq, hostname, gpu_index) triple identifying this sample.""" return (self.scrape_seq, self.hostname, self.gpu_index) @@ -193,9 +201,11 @@ class ObservedDevice: @property def key(self) -> tuple[str, int]: + """The (hostname, gpu_index) pair identifying this device.""" return (self.hostname, self.gpu_index) def to_dict(self) -> dict: + """Return this device as a JSON-ready dict without its sample times.""" return { "hostname": self.hostname, "gpu_index": self.gpu_index, @@ -206,24 +216,31 @@ def to_dict(self) -> dict: def _text(value: Any, label: str) -> str: + """Return ``value`` if it is a non-empty string, else raise ValueError naming ``label``.""" if not isinstance(value, str) or not value: raise ValueError(f"{label} is not a non-empty string: {value!r}") return value def _whole(value: Any, label: str, *, minimum: int) -> int: + """Return ``value`` if it is an int of at least ``minimum``, else raise ValueError naming ``label``.""" if not isinstance(value, int) or isinstance(value, bool) or value < minimum: raise ValueError(f"{label} is not an integer >= {minimum}: {value!r}") return value def _role(value: Any) -> str: + """Return ``value`` if it is a known worker role, else raise ValueError.""" if value not in WORKER_ROLES: raise ValueError(f"worker_role is not one of {WORKER_ROLES}: {value!r}") return value def _parse_expected_devices(manifest: dict) -> list[ExpectedDevice]: + """Parse the manifest's expected_devices into ExpectedDevice entries. + + Raises TypeError, ValueError or KeyError on malformed entries. + """ entries = manifest.get("expected_devices") or [] if not isinstance(entries, list): raise TypeError(f"expected_devices is not a list: {entries!r}") @@ -256,6 +273,10 @@ def _parse_expected_devices(manifest: dict) -> list[ExpectedDevice]: def _parse_expected_windows(manifest: dict) -> list[ExpectedWindow]: + """Parse the manifest's expected_windows into ExpectedWindow entries. + + Raises TypeError, ValueError or KeyError on malformed entries. + """ entries = manifest.get("expected_windows") or [] if not isinstance(entries, list): raise TypeError(f"expected_windows is not a list: {entries!r}") @@ -368,6 +389,7 @@ def _parse_sample_row(raw: list[str], expected_version: int) -> SampleRow | None def read_samples(path: Path) -> tuple[tuple[SampleRow, ...], tuple[str, ...]]: + """Return the valid unique sample rows of the samples CSV at ``path`` and the reasons it is unsound.""" if not path.is_file(): return (), ("samples_csv_missing",) @@ -417,6 +439,7 @@ def read_samples(path: Path) -> tuple[tuple[SampleRow, ...], tuple[str, ...]]: def derive_observed_devices(rows: tuple[SampleRow, ...]) -> list[ObservedDevice]: + """Group sample ``rows`` per (hostname, gpu_index) into ObservedDevice entries sorted by key.""" ordered: dict[tuple[str, int], list[SampleRow]] = {} for row in rows: ordered.setdefault((row.hostname, row.gpu_index), []).append(row) @@ -441,6 +464,7 @@ def derive_observed_devices(rows: tuple[SampleRow, ...]) -> list[ObservedDevice] def _resolve_roles( devices: list[ExpectedDevice], ) -> tuple[dict[tuple[str, int], str], list[str]]: + """Map each expected device to its single worker role, or return {} and a conflict reason.""" roles: dict[tuple[str, int], str] = {} for device in devices: distinct = {assignment.worker_role for assignment in device.assignments} @@ -453,6 +477,7 @@ def _resolve_roles( def _resolve_het_groups( devices: list[ExpectedDevice], ) -> tuple[dict[str, int | None], list[str]]: + """Map each expected host to its single het group, or return {} and a conflict reason.""" groups: dict[str, int | None] = {} for device in devices: distinct = {assignment.het_group for assignment in device.assignments} @@ -469,6 +494,7 @@ def _validate_devices( expected: list[ExpectedDevice], observed: list[ObservedDevice], ) -> tuple[list[str], dict[tuple[str, int], str]]: + """Compare ``expected`` and ``observed`` devices and return the mismatch reasons and the role per device.""" reasons: list[str] = [] expected_keys = {device.key for device in expected} observed_keys = {device.key for device in observed} @@ -555,6 +581,7 @@ class ParsedWindow: def _window_status_invariants_hold(status: str, end: Any, duration: Any, reason: Any) -> bool: + """Return whether ``end``, ``duration`` and ``reason`` are consistent with window ``status``.""" if status == WINDOW_STATUS_RUNNING: return end is None and duration is None and reason is None if status == "interrupted": @@ -569,6 +596,10 @@ def _window_status_invariants_hold(status: str, end: Any, duration: Any, reason: def _parse_window( path: Path, relative: str, result_root: Path ) -> tuple[ParsedWindow | None, list[str]]: + """Parse the window JSON at ``path`` whose result must stay below ``result_root``. + + Returns the window, or None with the reason codes when it is malformed. + """ try: payload = json.loads(path.read_text()) except (OSError, ValueError): @@ -621,6 +652,10 @@ def _scan_windows( result_root: Path, artifact_errors: list[dict], ) -> tuple[dict[tuple[str, int], ParsedWindow], set[tuple[str, int]]]: + """Parse every window file in ``windows_dir`` keyed by (type, concurrency), dropping duplicated keys. + + Returns the windows and the duplicated keys, appending file problems to ``artifact_errors``. + """ parsed: dict[tuple[str, int], ParsedWindow] = {} duplicates: set[tuple[str, int]] = set() if not windows_dir.is_dir(): @@ -672,6 +707,7 @@ def _scan_windows( def _check_window_result(window: ParsedWindow, result_root: Path) -> list[str]: + """Return reasons the window's result JSON under ``result_root`` is missing or disagrees with ``window``.""" result_file = result_root / window.result_path if not result_file.is_file(): return ["measurement_window_result_missing"] @@ -697,6 +733,7 @@ def _check_window_result(window: ParsedWindow, result_root: Path) -> list[str]: def _bracketing_sequence(times: tuple[float, ...], start: float, end: float) -> list[float] | None: + """Return the sample ``times`` from the last at or before ``start`` to the first at or after ``end``, or None.""" before = [value for value in times if value <= start] after = [value for value in times if value >= end] if not before or not after: @@ -711,6 +748,7 @@ def _check_coverage( expected_device_keys: set[tuple[str, int]], observed_devices: list[ObservedDevice], ) -> tuple[dict[str, float], list[str]]: + """Return the largest sample gap per expected device across ``start``..``end`` and the coverage reasons.""" by_key = {device.key: device for device in observed_devices} gaps: dict[str, float] = {} reasons: list[str] = [] @@ -821,6 +859,7 @@ def _check_stored_evidence( validations: list[dict], artifact_errors: list[dict], ) -> list[str]: + """Return mismatches between the ``manifest``'s stored evidence and the values recomputed from disk.""" failures: list[str] = [] stored_rows = manifest.get("sample_row_count") @@ -894,12 +933,14 @@ class MultinodePowerAudit: def _canonical_sha256(payload: dict) -> str: + """Return the SHA-256 hex digest of ``payload`` as sorted, compact JSON.""" return hashlib.sha256( json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() def _add_reason(audit: MultinodePowerAudit, reason: str, detail: str | None = None) -> None: + """Record ``reason`` once on ``audit`` and append ``detail`` to its failures when given.""" _append_reason(audit.reasons, reason) if detail: audit.failures.append(detail) @@ -1239,6 +1280,7 @@ def _select_window_for_result( def _patch_agg(agg_path: Path, audit: MultinodePowerAudit) -> None: + """Write the ``audit`` verdict and power metrics into the aggregate JSON at ``agg_path``.""" patch_power_metrics( agg_path, metric_keys=_ALL_POWER_METRIC_KEYS, @@ -1254,6 +1296,7 @@ def _sidecar_payload( bench_result: Path, benchmark: BenchmarkData | None, ) -> dict: + """Build the power validation sidecar JSON from ``audit`` and the benchmark inputs.""" return { "schema_version": 1, "power_valid": audit.power_valid, @@ -1383,6 +1426,7 @@ def run( def main() -> int: + """Parse the multinode power options from argv and return the exit code of ``run``.""" parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--power-dir", diff --git a/inferencex-e2e/infx/results/result_filename.py b/inferencex-e2e/infx/results/result_filename.py index c4093589ad..ad3e421532 100644 --- a/inferencex-e2e/infx/results/result_filename.py +++ b/inferencex-e2e/infx/results/result_filename.py @@ -8,6 +8,10 @@ def compact(value: str, budget: int) -> str: + """Return ``value`` if it fits ``budget`` UTF-8 bytes, else a truncated prefix plus a hash suffix. + + Raises ValueError for empty values, path separators or a budget too small for the suffix. + """ if not value or any(char in value for char in "/\\\r\n\x00"): raise ValueError("Expected a nonempty filename component") encoded = value.encode("utf-8") @@ -20,6 +24,7 @@ def compact(value: str, budget: int) -> str: def result_stem(base: str, fingerprint: str = "") -> str: + """Return the result basename for ``base`` and an optional recipe ``fingerprint``, capped at 120 bytes.""" value = base + (f"_recipe-{fingerprint}" if fingerprint else "") return compact(value, 120) @@ -27,6 +32,10 @@ def result_stem(base: str, fingerprint: str = "") -> str: def point_filename( stem: str, config: str, conc: str, gpus: str, ctx: str = "", gen: str = "" ) -> str: + """Return the point result filename for ``stem``, ``config`` and numeric point fields within 255 bytes. + + Raises ValueError when a point field is not decimal. + """ numbers = [conc, gpus, *([ctx, gen] if ctx or gen else [])] if any(not number.isascii() or not number.isdecimal() for number in numbers): raise ValueError("Expected numeric point identity") @@ -36,6 +45,7 @@ def point_filename( def main() -> None: + """Print a point filename for ``--point``, else the result stem from RESULT_FILENAME_BASE.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--point", nargs=6, metavar=("STEM", "CONFIG", "CONC", "GPUS", "CTX", "GEN") From 0f88fc62abcc6801658cc8f77ca898ad5d2f8a9f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:32:22 -0500 Subject: [PATCH 195/203] docs(launch): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give every function and method in infx/launch, infx/clusters, infx/srt_slurm and infx/golden_al_distribution a short docstring naming its inputs and result, and delete the remaining # comments. Facts a reader needs (Slurm signal and time quirks, SGLang method names, DFlash and Qwen3.5 draft windows) move into the docstrings. Code is unchanged apart from docstrings. 中文:为 infx/launch、infx/clusters、infx/srt_slurm 和 infx/golden_al_distribution 中的每个函数和方法补充简短的文档字符串,说明其输入与结果,并删除剩余的 # 注释。 读者需要的信息(Slurm 信号与时间的特殊行为、SGLang 方法命名、DFlash 与 Qwen3.5 的草稿窗口)已移入文档字符串。除文档字符串外代码未改动。 --- inferencex-e2e/infx/clusters/__init__.py | 3 ++ .../infx/golden_al_distribution/__main__.py | 6 ++++ .../infx/golden_al_distribution/curves.py | 12 +++++-- inferencex-e2e/infx/launch/__main__.py | 3 ++ inferencex-e2e/infx/launch/artifacts.py | 10 ++++-- .../infx/launch/backends/__init__.py | 1 + inferencex-e2e/infx/launch/backends/base.py | 2 ++ .../infx/launch/backends/slurm/__init__.py | 2 +- .../infx/launch/backends/slurm/cli.py | 9 +++-- .../infx/launch/backends/slurm/squash.py | 2 ++ inferencex-e2e/infx/launch/drivers/script.py | 2 ++ .../infx/launch/drivers/srt/__init__.py | 4 +++ .../infx/launch/drivers/srt/checkout.py | 4 +++ .../infx/launch/drivers/srt/collect.py | 1 + .../infx/launch/drivers/srt/lanes.py | 1 + .../infx/launch/drivers/srt/models.py | 1 + inferencex-e2e/infx/launch/drivers/srt/run.py | 3 ++ .../infx/launch/drivers/srt/submit.py | 1 + inferencex-e2e/infx/launch/event.py | 34 ++++++++++++++++--- inferencex-e2e/infx/launch/evidence.py | 6 ++-- inferencex-e2e/infx/launch/lifecycle.py | 11 ++++++ inferencex-e2e/infx/launch/policy.py | 11 ++++-- inferencex-e2e/infx/launch/request.py | 5 +++ inferencex-e2e/infx/srt_slurm/single_node.py | 2 +- .../infx/srt_slurm/synthetic_acceptance.py | 15 +++++--- inferencex-e2e/infx/srt_slurm/variants.py | 3 ++ inferencex-e2e/infx/srt_slurm/workload.py | 6 ++++ 27 files changed, 138 insertions(+), 22 deletions(-) diff --git a/inferencex-e2e/infx/clusters/__init__.py b/inferencex-e2e/infx/clusters/__init__.py index 0a06183c5e..305b67e948 100644 --- a/inferencex-e2e/infx/clusters/__init__.py +++ b/inferencex-e2e/infx/clusters/__init__.py @@ -83,14 +83,17 @@ class Cluster(Record): @property def id(self) -> str: + """Return the cluster id this record was bound to, or an empty string.""" return self._id def bind_id(self, cluster_id: str) -> None: + """Record ``cluster_id`` as this cluster's id.""" self._id = cluster_id @field_validator("env") @classmethod def _exportable(cls, env: dict[str, str]) -> dict[str, str]: + """Return ``env`` unchanged, or raise ValueError if any value contains a comma.""" if names := sorted(name for name, value in env.items() if "," in value): raise ValueError(f"env values cannot contain ',' (srun --export splits on it): {names}") return env diff --git a/inferencex-e2e/infx/golden_al_distribution/__main__.py b/inferencex-e2e/infx/golden_al_distribution/__main__.py index 363bfb5bcf..15a20a7dfa 100644 --- a/inferencex-e2e/infx/golden_al_distribution/__main__.py +++ b/inferencex-e2e/infx/golden_al_distribution/__main__.py @@ -26,6 +26,7 @@ def token_range(tokens: list[int]) -> str: + """Render draft lengths as ``a-b`` if contiguous, else comma-separated, or ``-`` if empty.""" if not tokens: return "-" if tokens == list(range(tokens[0], tokens[-1] + 1)) and len(tokens) > 1: @@ -34,6 +35,7 @@ def token_range(tokens: list[int]) -> str: def curve_json(curve: Curve) -> dict: + """Return a curve as a JSON-ready dict of its name, model and per-mode AL values.""" return { "curve": curve.name, "model": curve.model, @@ -45,6 +47,7 @@ def curve_json(curve: Curve) -> dict: def list_command(args: argparse.Namespace) -> None: + """Print every committed curve with its draft-length ranges, as a table or JSON.""" curves = list_curves(args.golden_dir) if args.json: print(json.dumps([curve_json(curve) for curve in curves], indent=2)) @@ -62,6 +65,7 @@ def list_command(args: argparse.Namespace) -> None: def show_command(args: argparse.Namespace) -> None: + """Print one curve's AL per draft length and thinking mode, as a table or JSON.""" curve = load_curve(args.curve, args.golden_dir) if args.json: print(json.dumps(curve_json(curve), indent=2)) @@ -85,6 +89,7 @@ def show_command(args: argparse.Namespace) -> None: def lookup_command(args: argparse.Namespace) -> None: + """Print the golden AL for the model, method, draft length and thinking mode in ``args``.""" spec = {"method": args.method, "num_speculative_tokens": args.tokens} if args.draft_model: spec["model"] = args.draft_model @@ -107,6 +112,7 @@ def lookup_command(args: argparse.Namespace) -> None: def main(argv: list[str] | None = None) -> int: + """Parse ``argv``, run the chosen subcommand and return the exit code.""" parser = argparse.ArgumentParser( prog="python -m infx.golden_al_distribution", description=__doc__, diff --git a/inferencex-e2e/infx/golden_al_distribution/curves.py b/inferencex-e2e/infx/golden_al_distribution/curves.py index 27c660ce9d..4618bb1533 100644 --- a/inferencex-e2e/infx/golden_al_distribution/curves.py +++ b/inferencex-e2e/infx/golden_al_distribution/curves.py @@ -25,9 +25,11 @@ class Curve: modes: Mapping[str, Mapping[int, Any]] def tokens(self, thinking: str) -> list[int]: + """Return the sorted draft lengths measured for one thinking mode.""" return sorted(self.modes.get(thinking) or {}) def acceptance(self, thinking: str, tokens: int) -> float: + """Return the AL for a thinking mode and draft length, or raise ValueError if invalid.""" try: value = float(self.modes[thinking][tokens]) except (KeyError, TypeError, ValueError) as error: @@ -40,9 +42,12 @@ def acceptance(self, thinking: str, tokens: int) -> float: def curve_name(model: str, spec: Mapping[str, Any]) -> str: - """Map an InferenceX model prefix and speculative config to a curve file stem.""" + """Map an InferenceX model prefix and speculative config to a curve file stem. + + SGLang names native model MTP EAGLE or NEXTN, so those map to the model's own head curve. + Kimi DSpark has one curve per draft sampler and raises ValueError for any other sampler. + """ method = str(spec.get("method", "")).lower() - # SGLang calls native model MTP EAGLE/NEXTN; the curve describes the model's head. if method in ("eagle", "nextn"): method = "eagle3" if model == "minimaxm3" else "mtp" curve = f"{model}_{method}" @@ -52,7 +57,6 @@ def curve_name(model: str, spec: Mapping[str, Any]) -> str: if "gqa" in str(spec.get("model", "")).lower(): curve += "_gqa" elif model == "kimik3" and method == "dspark": - # Kimi has distinct measured curves; require the recipe to choose its sampler. sampling = spec.get("draft_sample_method") if sampling == "probabilistic": curve += "_probabilistic_sample_method_block_rejection_sample_method" @@ -64,6 +68,7 @@ def curve_name(model: str, spec: Mapping[str, Any]) -> str: def load_curve(name: str, golden_dir: Path = GOLDEN_DIR) -> Curve: + """Load ``.yaml`` from ``golden_dir``, or raise ValueError if missing or malformed.""" if not CURVE_NAME.fullmatch(name): raise ValueError(f"Invalid golden curve identity: {name!r}") path = golden_dir / f"{name}.yaml" @@ -82,6 +87,7 @@ def load_curve(name: str, golden_dir: Path = GOLDEN_DIR) -> Curve: def list_curves(golden_dir: Path = GOLDEN_DIR) -> list[Curve]: + """Return every committed curve in ``golden_dir``, sorted by name.""" return [load_curve(path.stem, golden_dir) for path in sorted(golden_dir.glob("*.yaml"))] diff --git a/inferencex-e2e/infx/launch/__main__.py b/inferencex-e2e/infx/launch/__main__.py index f6398710fc..7cd2bffb14 100644 --- a/inferencex-e2e/infx/launch/__main__.py +++ b/inferencex-e2e/infx/launch/__main__.py @@ -39,6 +39,7 @@ def launch(cluster: Cluster, request: LaunchRequest, event: JobEventBuilder | No def _run(runner_config: Path | None) -> int: + """Launch the point the environment describes, emit its job record and return its exit code.""" event = JobEventBuilder.begin(os.environ) rc = 1 try: @@ -64,6 +65,7 @@ def _run(runner_config: Path | None) -> int: def _cleanup(runner_config: Path | None) -> int: + """Remove leftovers of earlier launches on ``RUNNER_NAME`` and return 0, or 1 if it is unset.""" runner = os.environ.get("RUNNER_NAME", "") if not runner: print("ERROR: RUNNER_NAME is required", file=sys.stderr) @@ -80,6 +82,7 @@ def _cleanup(runner_config: Path | None) -> int: def main(argv: list[str] | None = None) -> int: + """Parse ``argv``, run the ``run`` or ``cleanup`` command and return its exit code.""" parser = argparse.ArgumentParser(prog="python -m infx.launch", description=__doc__) parser.add_argument( "--runner-config", diff --git a/inferencex-e2e/infx/launch/artifacts.py b/inferencex-e2e/infx/launch/artifacts.py index 44b7c88a8a..9579d005e9 100644 --- a/inferencex-e2e/infx/launch/artifacts.py +++ b/inferencex-e2e/infx/launch/artifacts.py @@ -30,10 +30,12 @@ class ArtifactError(RuntimeError): def _say(message: str) -> None: + """Print ``message`` to stdout and flush.""" print(message, flush=True) def _warn(message: str) -> None: + """Print ``message`` to stderr and flush.""" print(message, file=sys.stderr, flush=True) @@ -61,6 +63,7 @@ def copy_to_workspace(source: Path, destination: Path) -> None: def _point_fields(filename: str) -> list[str]: + """Return the concurrency, GPU, ctx and gen fields of a result filename, blank if absent.""" return [match.group(1) if (match := p.search(filename)) else "" for p in _POINT_FIELDS] @@ -77,7 +80,10 @@ def _result_subdirs(logs_dir: Path) -> list[Path]: def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename: str) -> None: - """Copy srt-slurm's ``results_concurrency_*.json`` points under bounded workspace names.""" + """Copy srt-slurm's ``results_concurrency_*.json`` points under bounded workspace names. + + Skips benchmark_serving's ``.pytorch.json`` sidecars, which share a point's name prefix. + """ subdirs = _result_subdirs(logs_dir) if not subdirs: _warn(f"WARNING: no result subdirectories found in {logs_dir}") @@ -85,7 +91,6 @@ def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename _say(f"Processing result subdirectory: {subdir}") config_name = subdir.name for result_file in sorted(subdir.rglob("results_concurrency_*.json")): - # benchmark_serving's PyTorch-format sidecar shares the point's name prefix. if not result_file.is_file() or result_file.name.endswith(".pytorch.json"): continue concurrency, gpus, ctx, gen = _point_fields(result_file.name) @@ -104,6 +109,7 @@ def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename def _top_level_files(directory: Path, pattern: str = "*") -> list[Path]: + """Return regular, non-symlink files directly in ``directory`` matching ``pattern``, sorted.""" return sorted( entry for entry in directory.iterdir() diff --git a/inferencex-e2e/infx/launch/backends/__init__.py b/inferencex-e2e/infx/launch/backends/__init__.py index 54971195fc..69cf8bd021 100644 --- a/inferencex-e2e/infx/launch/backends/__init__.py +++ b/inferencex-e2e/infx/launch/backends/__init__.py @@ -15,5 +15,6 @@ def backend_class(scheduler: str) -> type[Backend]: + """Import and return the backend class registered for ``scheduler``, or raise KeyError.""" module, _, name = BACKENDS[scheduler].partition(":") return getattr(importlib.import_module(module), name) diff --git a/inferencex-e2e/infx/launch/backends/base.py b/inferencex-e2e/infx/launch/backends/base.py index e13e6c00df..18a10888d4 100644 --- a/inferencex-e2e/infx/launch/backends/base.py +++ b/inferencex-e2e/infx/launch/backends/base.py @@ -57,6 +57,7 @@ class JobStatus: @property def succeeded(self) -> bool: + """Return whether the job ended in the succeeded state.""" return self.state is JobState.SUCCEEDED @@ -115,6 +116,7 @@ class Backend(ABC): """Runs containers and follows jobs on one cluster's scheduler for one launch.""" def __init__(self, cluster: Cluster, request: LaunchRequest, life: Lifecycle) -> None: + """Bind the backend to ``cluster``, the launch ``request`` and its ``life`` cycle.""" self.cluster = cluster self.request = request self.life = life diff --git a/inferencex-e2e/infx/launch/backends/slurm/__init__.py b/inferencex-e2e/infx/launch/backends/slurm/__init__.py index 9dbc979165..4b1c7eae21 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/__init__.py +++ b/inferencex-e2e/infx/launch/backends/slurm/__init__.py @@ -48,7 +48,6 @@ from infx.launch.request import LaunchRequest CANCEL_POLL_S = 10.0 -# End states that lose the job to the cluster rather than fail its workload. _LOST_STATES = frozenset({"BOOT_FAIL", "CANCELLED", "NODE_FAIL", "PREEMPTED"}) @@ -398,6 +397,7 @@ def _allocate(self, gpus: int, time_min: int) -> Job: returned = False def cancel_unreturned() -> None: + """Cancel the runner's queued request by name if ``salloc`` never returned a job.""" if not returned: cli.cancel_named([self.request.runner_name], wait=False) diff --git a/inferencex-e2e/infx/launch/backends/slurm/cli.py b/inferencex-e2e/infx/launch/backends/slurm/cli.py index 79eaf6f00b..4e3f5dd0a4 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/cli.py +++ b/inferencex-e2e/infx/launch/backends/slurm/cli.py @@ -37,6 +37,7 @@ class ContainerSpec: env: dict[str, str] = field(default_factory=dict) def srun_args(self) -> list[str]: + """Return the ``srun`` container flags for this spec.""" args = [f"--container-image={self.image}"] if self.mounts: args.append( @@ -61,6 +62,7 @@ class Resources: exclude: Sequence[str] = () def args(self) -> list[str]: + """Return the ``salloc``/``sbatch`` flags requesting these resources on one node.""" args = [f"--partition={self.partition}"] if self.account: args.append(f"--account={self.account}") @@ -205,6 +207,7 @@ def cancel_named(names: Sequence[str], *, wait: bool = True) -> None: def _query(argv: list[str], env: Mapping[str, str] | None = None) -> str: + """Run a Slurm query and return its stdout, or an empty string if it fails or is absent.""" try: result = proc.run(argv, env=env, capture=True) except FileNotFoundError: @@ -233,7 +236,8 @@ def _time(text: str) -> datetime | None: def observe(job: Job) -> JobStatus: """One reading of the allocation: accounting (``sacct -X``), else the controller's record. - Never raises; a job neither knows reads as ``UNKNOWN``. + Never raises; a job neither knows reads as ``UNKNOWN``. Start and end times are dropped while + the job is not yet started or finished, since scontrol projects them from the time limit. """ env = {**os.environ, "SLURM_TIME_FORMAT": "%s"} argv = ["sacct", "-X", "-n", "-P", "-j", job.id] @@ -253,7 +257,6 @@ def observe(job: Job) -> JobStatus: for name in ("JobState", "ExitCode", "NodeList", "SubmitTime", "StartTime", "EndTime") ) nodes, state = nodes.strip(), state.strip() - # scontrol projects StartTime for pending jobs and EndTime from the time limit until the end. started = "" if state == "PENDING" else started ended = "" if state in {"PENDING", "CONFIGURING", "RUNNING"} else ended return replace( @@ -269,10 +272,10 @@ def job_status(state: str, exit_code: str) -> JobStatus: """Only ``COMPLETED`` with ``0:0`` succeeds; any other state after the job ran failed. Slurm's ``ExitCode`` is ``status:signal``, and a death by signal N reads as 128 + N. + Only signals 1 to 64 count, so OUT_OF_MEMORY's ``0:125`` marker is not read as a signal. """ code, _, signal = exit_code.partition(":") raw = f"{state}|{exit_code}" - # Linux signals stop at 64. OUT_OF_MEMORY reports the marker 0:125, not a signal. if signal.isdigit() and 0 < int(signal) <= 64: exit_status: int | None = 128 + int(signal) else: diff --git a/inferencex-e2e/infx/launch/backends/slurm/squash.py b/inferencex-e2e/infx/launch/backends/slurm/squash.py index d40a579e32..3989ae1586 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/squash.py +++ b/inferencex-e2e/infx/launch/backends/slurm/squash.py @@ -60,6 +60,7 @@ def squash_path(image: str, policy: SquashPolicy) -> Path: def lock_path(image: str, policy: SquashPolicy) -> Path: + """Return the lock file guarding the squash file of ``image`` under the policy's lock layout.""" path = squash_path(image, policy) if policy.lock_file == "locks-dir": return path.parent / ".locks" / f"{squash_key(image)}.lock" @@ -100,6 +101,7 @@ def registry_reference(image: str) -> str: def is_valid_squash(path: Path) -> bool: + """Return whether ``path`` is readable and ``unsquashfs`` can list it.""" if not os.access(path, os.R_OK): return False try: diff --git a/inferencex-e2e/infx/launch/drivers/script.py b/inferencex-e2e/infx/launch/drivers/script.py index 2b191b1217..995a35fd29 100644 --- a/inferencex-e2e/infx/launch/drivers/script.py +++ b/inferencex-e2e/infx/launch/drivers/script.py @@ -33,6 +33,7 @@ def container_workspace(image: str) -> PurePosixPath: + """Return ``/ix`` for images with an editable install there, else ``/workspace``.""" if any(fnmatch.fnmatchcase(image, glob) for glob in EDITABLE_INSTALL_IMAGE_GLOBS): return PurePosixPath("/ix") return PurePosixPath("/workspace") @@ -66,6 +67,7 @@ def script_outputs(request: ScriptRequest, workdir: PurePosixPath) -> tuple[Pure def run(launch: Launch) -> int: + """Run the request's benchmark script in one container and return its exit code.""" request = ScriptRequest.from_env(launch.request.env) cluster, backend = launch.cluster, launch.backend time_limit = policy.salloc_time_limit(cluster.id, request) diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index 5b4ae0e292..1b0a3b0a96 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -208,6 +208,7 @@ def run_multinode(launch: Launch) -> int: def _srt_settings(cluster: Cluster | None) -> SrtSlurmSettings | None: + """Return the cluster's srt-slurm settings, or None if it is absent or not a Slurm cluster.""" settings = cluster.scheduler_settings if cluster is not None else None return settings.srt_slurm if isinstance(settings, SlurmSettings) else None @@ -217,15 +218,18 @@ def table_problems(clusters: Mapping[str, Cluster], only: str | None = None) -> problems: list[str] = [] def scoped(cluster_id: str) -> bool: + """Return whether ``cluster_id`` is within the checked scope.""" return only in (None, cluster_id) def profile(table: str, cluster_id: str) -> SrtSlurmSettings | None: + """Return the srt-slurm settings of ``cluster_id``, recording a problem if it has none.""" srt = _srt_settings(clusters.get(cluster_id)) if srt is None: problems.append(f"{table}[{cluster_id!r}]: no srt-slurm cluster {cluster_id!r}") return srt def volume(where: str, cluster_id: str, name: str) -> None: + """Record a problem at ``where`` if ``cluster_id`` has no volume ``name``.""" if name not in clusters[cluster_id].scheduler_settings.volumes: problems.append(f"{where}: no volume {name!r}") diff --git a/inferencex-e2e/infx/launch/drivers/srt/checkout.py b/inferencex-e2e/infx/launch/drivers/srt/checkout.py index da425e0b0f..e7ceb84c7f 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/checkout.py +++ b/inferencex-e2e/infx/launch/drivers/srt/checkout.py @@ -36,10 +36,12 @@ class Checkout: @property def venv(self) -> Path: + """Return the checkout's virtualenv directory.""" return self.root / ".venv" def _git(*args: str | Path, capture: bool = False) -> str: + """Run ``git`` and return stripped stdout if ``capture``, or raise LaunchError on failure.""" try: result = proc.run(["git", *args], check=True, capture=capture) except (OSError, subprocess.CalledProcessError) as error: @@ -48,6 +50,7 @@ def _git(*args: str | Path, capture: bool = False) -> str: def _checked(result: subprocess.CompletedProcess[str], action: str) -> None: + """Raise LaunchError naming ``action`` if ``result`` exited non-zero.""" if result.returncode: raise LaunchError(f"{action} failed (exit {result.returncode})") @@ -161,6 +164,7 @@ def _installed_version(venv: Path, name: str) -> str | None: def _run_logged(argv: list[str], log: Path, *, env: dict[str, str], cwd: Path) -> int: + """Echo and run ``argv`` in ``cwd``, append its output to ``log`` and return its exit code.""" echo(argv, env) with log.open("a") as handle: return subprocess.run( diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index 108fb88b3d..228021b558 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -164,6 +164,7 @@ def collect( logs_staged = False def stage_logs_on_exit() -> None: + """Fetch and stage the job's logs when the launch ends before collect staged them.""" if not logs_staged: _stage_logs(run, backend.fetch_outputs(job, fetched) / "logs", power) diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 4ffbeca23b..f33ae62222 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -107,6 +107,7 @@ class SrtLane: def srt_lane(cluster_id: str, path: LaunchPath) -> SrtLane: + """Return the srt-slurm lane for a cluster and launch path, or raise LaunchError if none.""" try: return SRT_LANES[(cluster_id, path)] except KeyError: diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 5f0e9b33cb..f52d8b62c5 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -117,6 +117,7 @@ def checkpoint(cluster: Cluster, request: LaunchRequest) -> Checkpoint | None: entries = cluster.models.entries def node_local(key: str) -> bool: + """Return whether model entry ``key`` lives on a node-local volume.""" return volumes[entries[key].root].visibility == "node-local" key = _override(cluster, request, "entry") diff --git a/inferencex-e2e/infx/launch/drivers/srt/run.py b/inferencex-e2e/infx/launch/drivers/srt/run.py index 80effcf96e..d11983b760 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/run.py +++ b/inferencex-e2e/infx/launch/drivers/srt/run.py @@ -52,9 +52,11 @@ def create( @property def workspace(self) -> Path: + """Return the request's runner workspace directory.""" return self.request.workspace def prepend_path(self, directory: Path) -> None: + """Put ``directory`` first on the children's ``PATH``.""" self.env["PATH"] = os.pathsep.join(filter(None, (str(directory), self.env.get("PATH")))) @@ -66,6 +68,7 @@ def require(request: LaunchRequest, *names: str) -> None: def slurm_backend(launch: Launch) -> SlurmBackend: + """Return the launch's Slurm backend, or raise TypeError if it uses another backend.""" if not isinstance(launch.backend, SlurmBackend): raise TypeError( f"{launch.path} needs the Slurm backend, got {type(launch.backend).__name__}" diff --git a/inferencex-e2e/infx/launch/drivers/srt/submit.py b/inferencex-e2e/infx/launch/drivers/srt/submit.py index b7d6735619..19787679aa 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/submit.py +++ b/inferencex-e2e/infx/launch/drivers/srt/submit.py @@ -205,6 +205,7 @@ def cancel(self, backend: SlurmBackend) -> None: backend.cancel(job) def adopted(self) -> SlurmJob: + """Return the submitted job, or raise LaunchError if srtctl reported none.""" if self.job is None: raise LaunchError("srtctl reported no submitted job") return self.job diff --git a/inferencex-e2e/infx/launch/event.py b/inferencex-e2e/infx/launch/event.py index b8ec37664a..cf8237b255 100644 --- a/inferencex-e2e/infx/launch/event.py +++ b/inferencex-e2e/infx/launch/event.py @@ -118,6 +118,7 @@ class Metrics(_Record): ) @classmethod def _rounded(cls, value: float | None) -> float | None: + """Round a metric to three decimals, keeping None.""" return None if value is None else round(value, 3) @@ -208,6 +209,7 @@ def __init__( *, annotate: bool = False, ) -> None: + """Start timing a launch that records into ``event`` and tracks ``workspace`` files.""" self.event = event if event is not None else JobEvent() self.workspace = workspace self._annotate = annotate @@ -223,7 +225,11 @@ def __init__( @classmethod def begin(cls, env: Mapping[str, str]) -> Self: - """Identify the job from the workflow environment; never raises.""" + """Identify the job from the workflow environment; never raises. + + AgentX replays traces, so agentic runs leave ISL and OSL unset. A batch-wrapped launch + never annotates on GitHub, because its wrapper is the job's exit boundary. + """ scenario = _text(env, "SCENARIO_TYPE") multinode = env.get("IS_MULTINODE") == "true" eval_only = env.get("EVAL_ONLY") == "true" @@ -245,7 +251,6 @@ def begin(cls, env: Mapping[str, str]) -> Self: for word in env.get("CONC_LIST", "").split() if word.isascii() and word.isdecimal() ], - # AgentX replays traces; the workflow's ISL/OSL of 0 only fill the fixed shape. isl=None if scenario == AGENTIC else _number(env, "ISL"), osl=None if scenario == AGENTIC else _number(env, "OSL"), topology=_topology(env, multinode=multinode), @@ -261,15 +266,16 @@ def begin(cls, env: Mapping[str, str]) -> Self: kv_offload_backend=_text(env, "KV_OFFLOAD_BACKEND"), ) workspace = Path(env["GITHUB_WORKSPACE"]) if env.get("GITHUB_WORKSPACE") else None - # A batch-wrapped launch is not the job's exit boundary; the wrapper annotates. annotate = env.get("GITHUB_ACTIONS") == "true" and env.get(BATCH_REENTRY_ENV) != "1" return cls(event, workspace, annotate=annotate) def set(self, **fields: object) -> None: + """Set the given fields on the event.""" for name, value in fields.items(): setattr(self.event, name, value) def set_versions(self, **versions: str | None) -> None: + """Set the given component versions on the event.""" for name, value in versions.items(): setattr(self.event.versions, name, value) @@ -290,6 +296,7 @@ def stage(self, name: str) -> Iterator[None]: @property def current_stage(self) -> str: + """Return the innermost running stage, else the last one, else ``prepare``.""" return self._active[-1] if self._active else self._last or "prepare" def evidence(self, log: Path) -> None: @@ -324,10 +331,12 @@ def error(self, error: BaseException, *, stage: str | None = None, report: bool self.fail(kind, str(error), stage=stage, retriable=retriable, report=report) def interrupted(self, signum: int) -> None: + """Record a retriable failure for the received signal ``signum``.""" message = f"received {signal.Signals(signum).name}" self.fail("Interrupted", message, retriable=True, report=False) def exited(self, returncode: int) -> None: + """Record a failure if ``returncode`` is non-zero.""" if returncode: self.fail("ExitStatus", f"exited {returncode}", report=False) @@ -344,13 +353,17 @@ def adopt_reentry(self) -> None: self._reentry = inner def finish(self, returncode: int) -> JobEvent: + """Return the completed record for a launch that exited ``returncode``. + + After a batch re-entry the wrapped launch's record is the base, since it knows the + benchmark job, and this launch adds its batch job, queue time and ``batch_`` stages. + """ self._lap() self.exited(returncode) record = self.event.model_copy(deep=True) stages = dict(self._stages) failure = self._failure if (inner := self._reentry) is not None: - # The wrapped launch knows the benchmark job; this one only the batch around it. outer = record record = inner.model_copy(deep=True) record.launch_path = outer.launch_path @@ -412,6 +425,7 @@ def emit(self, returncode: int) -> None: print(f"WARNING: could not write {path}: {error}", file=sys.stderr) def _lap(self) -> None: + """Charge the time since the last lap to the innermost active stage.""" now = time.monotonic() if self._active: name = self._active[-1] @@ -419,6 +433,7 @@ def _lap(self) -> None: self._since = now def _shown(self, path: Path | None) -> str | None: + """Return ``path`` relative to the workspace when inside it, else as given, or None.""" if path is None: return None if self.workspace is not None and path.is_relative_to(self.workspace): @@ -456,6 +471,7 @@ def annotation(error: JobError) -> str: def _escape_data(text: str) -> str: + """Escape ``%``, CR and LF in ``text`` for a GitHub Actions workflow command.""" return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") @@ -474,19 +490,23 @@ def _files(directory: Path | None) -> dict[str, tuple[int, int]]: def _text(env: Mapping[str, str], name: str) -> str | None: + """Return the variable ``name`` from ``env``, or None if unset or empty.""" return env.get(name) or None def _number(env: Mapping[str, str], name: str) -> int | None: + """Return the variable ``name`` as a non-negative integer, or None if it is not one.""" value = env.get(name, "") return int(value) if value.isascii() and value.isdecimal() else None def _flag(env: Mapping[str, str], name: str) -> bool | None: + """Return the variable ``name`` as a bool when it is ``true`` or ``false``, else None.""" return {"true": True, "false": False}.get(env.get(name, "")) def _topology(env: Mapping[str, str], *, multinode: bool) -> Topology: + """Return single-node parallelism, or per-role topology when ``multinode``.""" if not multinode: return Topology( tp=_number(env, "TP"), @@ -508,6 +528,7 @@ def _topology(env: Mapping[str, str], *, multinode: bool) -> Topology: def _versions(env: Mapping[str, str], *, agentx: bool) -> Versions: + """Return checkout, workflow and image versions, plus the AgentX harness if ``agentx``.""" head = None with contextlib.suppress(OSError): head = _git_head(repository_root()) @@ -527,6 +548,7 @@ def _versions(env: Mapping[str, str], *, agentx: bool) -> Versions: def _github(env: Mapping[str, str]) -> GitHub: + """Return the run and job URLs and the event, PR, test and ref details from ``env``.""" server, repository, run = ( _text(env, name) for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID") ) @@ -633,16 +655,19 @@ def _agentx_point(name: str, result: dict[str, Any]) -> Metrics: def _mapping(parent: dict[str, Any], key: str) -> dict[str, Any]: + """Return ``parent[key]`` if it is a dict, else an empty dict.""" value = parent.get(key) return value if isinstance(value, dict) else {} def _count(parent: dict[str, Any], key: str) -> int | None: + """Return ``parent[key]`` if it is a non-bool int, else None.""" value = parent.get(key) return value if isinstance(value, int) and not isinstance(value, bool) else None def _value(parent: dict[str, Any], key: str) -> float | None: + """Return ``parent[key]`` as a float if it is a finite non-bool number, else None.""" value = parent.get(key) if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value): return None @@ -650,4 +675,5 @@ def _value(parent: dict[str, Any], key: str) -> float | None: def _milliseconds(seconds: float | None) -> float | None: + """Convert ``seconds`` to milliseconds, keeping None.""" return None if seconds is None else seconds * 1000 diff --git a/inferencex-e2e/infx/launch/evidence.py b/inferencex-e2e/infx/launch/evidence.py index 9a3f62b455..9e935768a8 100644 --- a/inferencex-e2e/infx/launch/evidence.py +++ b/inferencex-e2e/infx/launch/evidence.py @@ -18,7 +18,6 @@ _ESCAPES = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])") _NOISE = re.compile(r"warn|^\+ ", re.IGNORECASE) -# Most specific first: a resource fault names the cause better than the exception it raises. _SIGNATURES = ( re.compile( r"out of memory|OutOfMemoryError|\bOOM\b|oom[-_ ]?kill|CUDA error|\bNCCL\w* ?error\b" @@ -60,7 +59,10 @@ def error_detail(log: Path) -> str | None: def _cause(handle: BinaryIO) -> str | None: - """The first line of the most specific signature rank in the scanned head of ``handle``.""" + """The first line of the most specific signature rank in the scanned head of ``handle``. + + Resource faults rank first, since they name the cause better than the exception they raise. + """ best: tuple[int, str] | None = None scanned = 0 while scanned < SCAN_BYTES and (raw := handle.readline(_LINE_BYTES)): diff --git a/inferencex-e2e/infx/launch/lifecycle.py b/inferencex-e2e/infx/launch/lifecycle.py index fae7f60700..7993ae1ef3 100644 --- a/inferencex-e2e/infx/launch/lifecycle.py +++ b/inferencex-e2e/infx/launch/lifecycle.py @@ -17,6 +17,7 @@ class _Interrupted(BaseException): def __init__(self, signum: int) -> None: + """Carry the received signal number ``signum``.""" super().__init__(signum) self.signum = signum @@ -30,6 +31,7 @@ class Lifecycle: """ def __init__(self, event: JobEventBuilder | None = None) -> None: + """Start with no cleanups and exit code 0, recording into ``event`` or a fresh one.""" self.event = event if event is not None else JobEventBuilder() self._stack = ExitStack() self._rc = 0 @@ -40,11 +42,13 @@ def __init__(self, event: JobEventBuilder | None = None) -> None: @property def returncode(self) -> int: + """Return the first recorded nonzero code, else 1 if a cleanup failed, else 0.""" if self._rc: return self._rc return 1 if self._cleanup_failed else 0 def record(self, rc: int) -> None: + """Keep ``rc`` as the exit code unless it is 0 or a nonzero code is already recorded.""" if rc and not self._rc: self._rc = rc @@ -52,6 +56,7 @@ def callback(self, fn: Callable[..., Any], /, *args: Any, **kwargs: Any) -> None """Register a cleanup; one that raises or returns a nonzero int fails the launch.""" def run() -> None: + """Call the cleanup and record a failure if it raises or returns a nonzero int.""" try: result = fn(*args, **kwargs) except Exception as error: # noqa: BLE001 @@ -68,6 +73,7 @@ def run() -> None: self._stack.callback(run) def _on_signal(self, signum: int, frame: FrameType | None) -> None: + """Defer ``signum`` while exiting, else ignore later signals and raise ``_Interrupted``.""" if self._exiting or (frame is not None and frame.f_code is _EXIT_CODE): self._signum = self._signum or signum return @@ -76,6 +82,7 @@ def _on_signal(self, signum: int, frame: FrameType | None) -> None: raise _Interrupted(signum) def __enter__(self) -> Self: + """Install the signal handlers and return this lifecycle.""" for sig in _SIGNALS: self._previous[sig] = signal.signal(sig, self._on_signal) return self @@ -86,6 +93,10 @@ def __exit__( exc: BaseException | None, tb: TracebackType | None, ) -> bool: + """Record how the body ended, run the cleanups and restore the signal handlers. + + Raises ``SystemExit(128 + signum)`` after a signal, else returns False to propagate errors. + """ self._exiting = True mask = signal.pthread_sigmask(signal.SIG_BLOCK, _SIGNALS) if isinstance(exc, _Interrupted): diff --git a/inferencex-e2e/infx/launch/policy.py b/inferencex-e2e/infx/launch/policy.py index 308febe46f..9a7b964e9f 100644 --- a/inferencex-e2e/infx/launch/policy.py +++ b/inferencex-e2e/infx/launch/policy.py @@ -21,6 +21,7 @@ def any_of(*values: str) -> frozenset[str]: + """Return ``values`` as a frozenset for a ``Match`` field.""" return frozenset(values) @@ -37,6 +38,7 @@ class Match: model_glob: str | None = None def __call__(self, request: LaunchRequest) -> bool: + """Return whether ``request`` satisfies every set field.""" return ( (self.prefixes is None or request.model_prefix in self.prefixes) and (self.precisions is None or request.precision in self.precisions) @@ -76,6 +78,7 @@ class LaunchPath(StrEnum): def launch_path(cluster_id: str, request: LaunchRequest) -> LaunchPath: + """Return the launch path that runs ``request`` on the cluster ``cluster_id``.""" if request.is_multinode: if any(lane(request) for lane in NATIVE_SRT_LANES.get(cluster_id, ())): return LaunchPath.SRT_NATIVE @@ -134,14 +137,17 @@ def runtime_env( def point_settings(request: LaunchRequest) -> frozenset[str]: - """The names the point's NAME=value additional-settings set; the workflow exports them.""" + """The names the point's NAME=value additional-settings set; the workflow exports them. + + A null value, which the workflow sends for a role without additional-settings, sets none. + Raises ``RequestError`` if a value is not a JSON list. + """ names: set[str] = set() for name in ("PREFILL_ADDITIONAL_SETTINGS", "DECODE_ADDITIONAL_SETTINGS"): try: values = json.loads(request.env.get(name) or "[]") except ValueError: continue - # The workflow passes toJSON of a role without additional-settings, which is null. if values is None: continue if not isinstance(values, list): @@ -155,6 +161,7 @@ def table_problems(clusters: Mapping[str, Cluster], only: str | None = None) -> """Rows of this module's tables that ``clusters`` contradicts; only ``only``'s rows if given.""" def keys(table: Mapping[str, object]) -> list[str]: + """Return the keys of ``table`` within the checked scope.""" return [key for key in table if only in (None, key)] tables: dict[str, Mapping[str, object]] = { diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index 2c463a062d..2c37dd67ad 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -21,14 +21,17 @@ class RequestError(ValueError): @classmethod def missing(cls, *names: str) -> RequestError: + """Return an error naming the unset environment variables ``names``.""" return cls(f"required environment variables are not set: {', '.join(names)}") def _equals(*truthy: str) -> BeforeValidator: + """Return a validator reading a string as True when it is one of ``truthy``.""" return BeforeValidator(lambda value: value if isinstance(value, bool) else value in truthy) def _int_list(value: Any) -> Any: + """Split a whitespace-separated string into a list, passing other values through.""" return value.split() if isinstance(value, str) else value @@ -39,6 +42,7 @@ def _int_list(value: Any) -> Any: def _describe(error: ValidationError) -> str: + """Render a validation error as one message listing missing then invalid inputs.""" missing: list[str] = [] invalid: list[str] = [] for detail in error.errors(include_url=False, include_input=False): @@ -113,6 +117,7 @@ def from_env(cls, env: Mapping[str, str] | None = None) -> Self: @property def workspace(self) -> Path: + """Return ``GITHUB_WORKSPACE``, or raise ``RequestError`` if it is unset.""" if self.github_workspace is None: raise RequestError.missing("GITHUB_WORKSPACE") return self.github_workspace diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index a902db78c9..f84bd6e8e4 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -24,7 +24,6 @@ ) SINGLE_NODE_ENGINES = {**ENGINES, "atom": "atom"} -# SGLang spells decode context parallelism --dcp-size, with the vLLM name as an alias. DCP_FLAGS = { "sglang": ("dcp-size", "decode-context-parallel-size"), "vllm": ("decode-context-parallel-size",), @@ -216,6 +215,7 @@ def submission_fields(path: Path) -> tuple[str, str]: def main() -> None: + """Run ``prepare`` to bind a recipe, or ``submission`` to print a job's id and output dir.""" parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) prepare = commands.add_parser("prepare") diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 51607eca11..e7da9b72bb 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -44,6 +44,11 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: + """Return a role's speculative config for ``engine`` as method, draft length and draft model. + + Empty when the role does not speculate. DFlash's draft length is its verify block minus the + current token. Raises ValueError on a malformed or conflicting config. + """ args = role.get("args", {}) if engine == "tilert": if args.get("with-mtp") is not True: @@ -81,7 +86,6 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: if not algorithm: return {} if algorithm == "dflash": - # DFlash verifies a block of the current token plus its drafts. width = args.get("speculative-num-draft-tokens") return { "method": "dflash", @@ -135,7 +139,11 @@ def _simulated_length( *, golden_dir: Path = GOLDEN_DIR, ) -> float | None: - """The golden AL ``build_overrides`` sets on non-adaptive speculative roles, else None.""" + """The golden AL ``build_overrides`` sets on non-adaptive speculative roles, else None. + + Raises ValueError unless a Qwen3.5 SGLang MTP generation role's window is its draft steps + plus one verification token. + """ engine = ENGINES.get(framework) if engine is None: return None @@ -154,7 +162,6 @@ def _simulated_length( for name, role in roles.items() if name in ("agg", "prefill", "decode") and spec_parameters(role, engine) ): - # build_overrides leaves adaptive verification real, so no golden AL is simulated. return None al = golden_length(environment["MODEL_PREFIX"], spec, environment["THINKING_MODE"], golden_dir) if ( @@ -162,7 +169,6 @@ def _simulated_length( and environment["MODEL_PREFIX"] == "qwen3.5" and spec.get("method") in ("eagle", "nextn", "mtp") ): - # The generation role's MTP window includes one verification token. drafts = generation.get("args", {}).get("speculative-num-draft-tokens") expected = spec["num_speculative_tokens"] + 1 if not isinstance(drafts, int) or isinstance(drafts, bool) or drafts != expected: @@ -352,6 +358,7 @@ def plan_commands( def main(argv: list[str] | None = None) -> int: + """Plan srtctl commands with golden acceptance from ``argv``, run them and return the rc.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--record", type=Path, help="write the plan's golden acceptance length here as JSON" diff --git a/inferencex-e2e/infx/srt_slurm/variants.py b/inferencex-e2e/infx/srt_slurm/variants.py index 7e8ae68bd8..c9c0d43cc1 100644 --- a/inferencex-e2e/infx/srt_slurm/variants.py +++ b/inferencex-e2e/infx/srt_slurm/variants.py @@ -29,6 +29,7 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] def _list_lengths(group: dict[str, Any]) -> list[int]: + """Return the length of every list value in ``group``, nested mappings included.""" lengths = [] for value in group.values(): if isinstance(value, list): @@ -50,6 +51,7 @@ def _zip_length(group: dict[str, Any]) -> int: def _zip_slice(group: dict[str, Any], index: int) -> dict[str, Any]: + """Return ``group`` with each list replaced by its ``index`` item, or its only item.""" sliced = {} for key, value in group.items(): if isinstance(value, list): @@ -81,6 +83,7 @@ def _block_variants(raw: dict[str, Any], key: str) -> list[tuple[str, dict[str, def without_nulls(value: Any) -> Any: + """Return ``value`` with null-valued keys removed from every nested mapping.""" if isinstance(value, dict): return {key: without_nulls(item) for key, item in value.items() if item is not None} return value diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 0b17d98364..e5eb375757 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -109,6 +109,7 @@ def merge_blocks(shared: Mapping[str, Any], fragment: Mapping[str, Any]) -> dict def _lookup(block: Any, key: tuple[str, ...]) -> tuple[bool, Any]: + """Follow ``key`` into nested mappings and return whether it is present and its value.""" for part in key: if not isinstance(block, Mapping) or part not in block: return False, None @@ -159,6 +160,7 @@ def check_fragment(raw: Mapping[str, Any], source: Path, *, agentic: bool, multi def _block(path: Path) -> dict[str, Any]: + """Load the YAML mapping at ``path``, or raise ValueError if it is not a mapping.""" block = yamlio.safe_load(path.read_text()) if not isinstance(block, dict): raise ValueError(f"{path}: shared block must be a mapping") @@ -210,6 +212,7 @@ def compose_recipe( def _required(environment: Mapping[str, str], name: str) -> str: + """Return the input ``name`` from ``environment``, or raise ValueError if it is blank.""" value = environment.get(name, "") if not value.strip(): raise ValueError(f"Missing workload input: {name}") @@ -217,6 +220,7 @@ def _required(environment: Mapping[str, str], name: str) -> str: def _positive(value: str, name: str) -> int: + """Return ``value`` as a positive int, or raise ValueError naming ``name``.""" if not value.isascii() or not value.isdecimal() or int(value) <= 0: raise ValueError(f"{name} must be a positive integer: {value!r}") return int(value) @@ -408,6 +412,7 @@ def add_fabric_argument(parser: argparse.ArgumentParser) -> None: def _assignment(text: str) -> tuple[str, str]: + """Split a ``NAME=VALUE`` argument, or raise ArgumentTypeError if malformed.""" name, separator, value = text.partition("=") if not separator or not name: raise argparse.ArgumentTypeError(f"expected NAME=VALUE, got {text!r}") @@ -415,6 +420,7 @@ def _assignment(text: str) -> tuple[str, str]: def main(argv: list[str] | None = None) -> None: + """Bind the multi-node recipe variant ``argv`` names and write it to the output path.""" parser = argparse.ArgumentParser(description="Bind one multi-node recipe variant") parser.add_argument("recipe", help="fragment[:selector]") parser.add_argument("output", type=Path) From e02dbf086d3be0d34ee0b1a43af80eb655650d22 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:33:55 -0500 Subject: [PATCH 196/203] docs(workflows): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give every function and method in infx/workflows and infx/evals a short docstring naming its inputs, result and notable errors. Remove the remaining comments and fold the four that explained upstream behavior into the docstrings of the functions they described. Code is unchanged. 中文:为 infx/workflows 与 infx/evals 中的每个函数和方法补充简短的文档字符串, 说明其输入、返回值及重要异常。删除剩余注释,并将四处说明上游行为的注释 并入对应函数的文档字符串。代码逻辑未改变。 --- .../infx/evals/_kimi_verifier_archive.py | 9 +++ inferencex-e2e/infx/evals/bfcl_adapter.py | 65 ++++++++++++++++++- inferencex-e2e/infx/evals/kimi_vendor_eval.py | 24 +++++++ .../infx/evals/minimax_m3_full_eval.py | 33 +++++++++- .../infx/evals/minimax_provider_eval.py | 19 ++++++ .../evals/patches/lm_eval_sitecustomize.py | 9 +++ inferencex-e2e/infx/evals/utils.py | 4 ++ inferencex-e2e/infx/evals/validate_scores.py | 4 ++ .../infx/workflows/benchmark_schema.py | 9 +++ .../infx/workflows/calc_success_rate.py | 7 ++ inferencex-e2e/infx/workflows/ci_priority.py | 20 ++++++ .../infx/workflows/merge_with_reuse.py | 28 +++++++- .../infx/workflows/pareto_coverage.py | 10 +++ .../infx/workflows/require_launcher.py | 4 ++ inferencex-e2e/infx/workflows/reuse.py | 5 ++ .../infx/workflows/reuse_comment.py | 8 +++ .../infx/workflows/signoff_publish.py | 14 ++++ .../infx/workflows/signoff_resolve.py | 12 ++++ .../infx/workflows/signoff_scope.py | 13 ++++ .../infx/workflows/signoff_validate.py | 4 ++ .../infx/workflows/stage_results.py | 17 +++++ .../infx/workflows/stage_results_callback.py | 9 +++ inferencex-e2e/infx/workflows/sweep_runs.py | 1 + .../infx/workflows/validate_perf_changelog.py | 7 +- 24 files changed, 326 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py index 3cb7743e83..2bb37c0416 100644 --- a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py @@ -14,6 +14,10 @@ def archive_member_parts(name: str) -> tuple[str, ...]: + """Split an archive member name into its path parts. + + Raises ValueError if the name is absolute, empty or escapes the archive root. + """ if not name or "\x00" in name or "\\" in name or name.startswith("/"): raise ValueError(f"unsafe archive member path: {name!r}") normalized = name.rstrip("/") @@ -24,6 +28,11 @@ def archive_member_parts(name: str) -> tuple[str, ...]: def main() -> None: + """Download the pinned verifier archive, check its SHA256 and extract the required files. + + Takes repo URL, commit SHA, archive SHA256 and checkout directory from argv. Exits 1 after + printing the failed stage. + """ repo_url, verifier_ref, expected_archive_sha256, checkout_dir_arg = sys.argv[1:] checkout_dir = Path(checkout_dir_arg) stage = "derive the pinned archive URL" diff --git a/inferencex-e2e/infx/evals/bfcl_adapter.py b/inferencex-e2e/infx/evals/bfcl_adapter.py index 17078989f4..b11f290016 100644 --- a/inferencex-e2e/infx/evals/bfcl_adapter.py +++ b/inferencex-e2e/infx/evals/bfcl_adapter.py @@ -75,13 +75,16 @@ class SuiteSpec: @property def leaf_categories(self) -> tuple[str, ...]: + """Leaf category names in suite order.""" return tuple(category for category, _ in self.expected_leaf_counts) @property def expected_sample_count(self) -> int: + """Total number of cases across all leaf categories.""" return sum(count for _, count in self.expected_leaf_counts) def projected_task(self, category: str) -> str: + """Return the projected lm-eval task name for category.""" if self.name == TASK_NAME: return f"bfcl_{category}" return f"{self.name}_{category}" @@ -162,7 +165,9 @@ def __call__( base_url: str, api_key: str, num_threads: int, - ) -> None: ... + ) -> None: + """Run BFCL generation and evaluation for model against base_url under project_root.""" + ... @dataclass(frozen=True) @@ -177,6 +182,7 @@ class CategoryScore: total_count: int def as_dict(self) -> dict[str, Any]: + """Return the score as a JSON-ready dict with a per-case pass or fail entry.""" failure_ids = {record["id"] for record in self.records} return { "category": self.category, @@ -196,12 +202,17 @@ def as_dict(self) -> dict[str, Any]: def _nonempty_string(value: str) -> str: + """Return value stripped of whitespace, raising ArgumentTypeError if it is blank.""" if not value.strip(): raise argparse.ArgumentTypeError("must be a non-empty string") return value.strip() def _absolute_http_url(value: str) -> str: + """Return value without trailing slashes. + + Raises ArgumentTypeError unless it is an absolute HTTP(S) API root URL. + """ normalized = _nonempty_string(value).rstrip("/") parsed = urllib.parse.urlsplit(normalized) if ( @@ -221,6 +232,7 @@ def _absolute_http_url(value: str) -> str: def _positive_int(value: str) -> int: + """Parse value as an int, raising ArgumentTypeError unless it is greater than 0.""" try: parsed = int(value) except ValueError as exc: @@ -234,6 +246,7 @@ def _source_details( suite: SuiteSpec, # noqa: ARG001 case_ids_by_category: Mapping[str, tuple[str, ...]], ) -> dict[str, Any]: + """Return the pinned upstream source metadata plus the case IDs selected per category.""" return { "url": UPSTREAM_SOURCE, "repository": UPSTREAM_REPOSITORY, @@ -250,12 +263,14 @@ def _source_details( def _error_dict(error: BaseException) -> dict[str, str]: + """Return the type name and message of error as a dict.""" return {"type": type(error).__name__, "message": str(error)} def _expected_category_details( case_ids_by_category: Mapping[str, tuple[str, ...]], ) -> list[dict[str, Any]]: + """Return placeholder diagnostics that mark every expected case in each category as failed.""" return [ { "category": category, @@ -276,6 +291,7 @@ def _diagnostics( case_ids_by_category: Mapping[str, tuple[str, ...]], scores: Sequence[CategoryScore] | None = None, ) -> dict[str, Any]: + """Return source metadata and per-category diagnostics, with placeholders if scores is None.""" return { "source": _source_details(suite, case_ids_by_category), "categories": ( @@ -296,6 +312,7 @@ def _native_report( scores: Sequence[CategoryScore] | None, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the native BFCL report for model from scores, recording integration_error if given.""" correct_count = sum(score.correct_count for score in scores or ()) total_count = sum(score.total_count for score in scores or ()) accuracy = correct_count / total_count if total_count else 0.0 @@ -332,6 +349,10 @@ def _compatibility_result( scores: Sequence[CategoryScore] | None, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with suite and per-category accuracies. + + Records integration_error in the result when given. + """ score_by_category = {score.category: score for score in scores or ()} total_count = sum(score.total_count for score in scores or ()) correct_count = sum(score.correct_count for score in scores or ()) @@ -398,6 +419,7 @@ def _compatibility_result( def _write_json(path: Path, value: Mapping[str, Any]) -> None: + """Write value to path as indented UTF-8 JSON.""" path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -437,6 +459,7 @@ def _write_upstream_attribution(project_root: Path) -> None: def _prepare_output_paths(output_dir: Path) -> tuple[Path, Path]: + """Create output_dir, delete stale reports and return the (native, compatibility) paths.""" output_dir.mkdir(parents=True, exist_ok=True) native_path = output_dir / NATIVE_REPORT_FILENAME native_path.unlink(missing_ok=True) @@ -467,6 +490,7 @@ def _bounded_openai_handler(stock_handler: type[Any]) -> type[Any]: class BoundedOpenAICompletionsHandler(stock_handler): def _build_client_kwargs(self) -> dict[str, Any]: + """Return the stock client kwargs with the full-suite timeout and retry limits.""" kwargs = super()._build_client_kwargs() kwargs.update( timeout=FULL_SUITE_REQUEST_TIMEOUT_SECONDS, @@ -478,6 +502,7 @@ def _build_client_kwargs(self) -> dict[str, Any]: def _write_id_map(project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]]) -> None: + """Write the selected case IDs per category to project_root/test_case_ids_to_generate.json.""" project_root.mkdir(parents=True, exist_ok=True) _write_json( project_root / "test_case_ids_to_generate.json", @@ -518,6 +543,11 @@ def _load_dataset_helpers() -> tuple[ def _build_suite_case_ids( suite: SuiteSpec, ) -> dict[str, tuple[str, ...]]: + """Return the case IDs per leaf category that suite evaluates. + + Splits each category limit evenly across its leaves. Raises ValueError on malformed or duplicate + dataset entries or unexpected counts. + """ if suite is SMOKE_SUITE: return dict(SMOKE_CASE_IDS) @@ -561,6 +591,10 @@ def _build_suite_case_ids( def _read_selected_suite( project_root: Path, ) -> tuple[SuiteSpec, dict[str, tuple[str, ...]]]: + """Read the case ID map under project_root and return the suite it matches with those IDs. + + Raises ValueError if the map is malformed or matches no supported suite. + """ raw = json.loads((project_root / "test_case_ids_to_generate.json").read_text(encoding="utf-8")) if not isinstance(raw, Mapping): raise ValueError("BFCL test-case ID map must be a JSON object") @@ -590,7 +624,11 @@ def _run_upstream( api_key: str, num_threads: int, ) -> None: - """Lazily load and invoke the pinned BFCL API against an existing server.""" + """Lazily load and invoke the pinned BFCL API against an existing server. + + Also registers the model config under its result-dir name with every "_" turned + into "/", since that is how BFCL evaluation looks configs up. + """ suite, case_ids_by_category = _read_selected_suite(project_root) _apply_suite_runtime_limits(suite) os.environ["BFCL_PROJECT_ROOT"] = str(project_root) @@ -623,7 +661,6 @@ def _run_upstream( underscore_to_dot=True, ) bfcl_model_config.MODEL_CONFIG_MAPPING[model] = model_config - # BFCL evaluation looks configs up by result dir name with every "_" turned into "/". bfcl_model_config.MODEL_CONFIG_MAPPING[model.replace("/", "_").replace("_", "/")] = model_config categories = list(suite.generation_categories) @@ -653,6 +690,10 @@ def _validate_generated_results( project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]] = SMOKE_CASE_IDS, ) -> None: + """Check each category has one result file under project_root with the expected case IDs. + + Raises ValueError on malformed files and RuntimeError if any case hit an inference error. + """ for category, case_ids in case_ids_by_category.items(): matches = sorted(project_root.glob(f"result/**/BFCL_v4_{category}_result.json")) if len(matches) != 1: @@ -693,6 +734,11 @@ def _validate_generated_results( def _validate_header(*, category: str, header: Any, expected_count: int) -> tuple[float, int, int]: + """Check a category score header against expected_count and return its counts. + + Returns (accuracy, correct count, total count). Raises ValueError if fields are missing, + mistyped or inconsistent. + """ if not isinstance(header, Mapping): raise ValueError(f"{category} score header must be a JSON object") missing = {"accuracy", "correct_count", "total_count"} - header.keys() @@ -727,6 +773,11 @@ def _collect_scores( project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]] = SMOKE_CASE_IDS, ) -> list[CategoryScore]: + """Read and validate each category's BFCL score file under project_root. + + Returns one CategoryScore per category. Raises ValueError if a file is missing, malformed or + disagrees with its header. + """ scores: list[CategoryScore] = [] for category, case_ids in case_ids_by_category.items(): matches = sorted(project_root.glob(f"score/**/BFCL_v4_{category}_score.json")) @@ -961,6 +1012,10 @@ def install_runtime(download_dir: Path, uv: str) -> None: def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse CLI arguments from argv. + + Enforces the options each mode needs and defaults --num-threads per suite. + """ parser = argparse.ArgumentParser(description="Run a pinned BFCL V4 OpenAI completions suite.") parser.add_argument( "--install-runtime", @@ -1000,6 +1055,10 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Install the runtime, record an integration error or run the suite. + + Returns 0 on pass, 1 on failure or after recording an error. + """ args = parse_args(argv) if args.install_runtime is not None: try: diff --git a/inferencex-e2e/infx/evals/kimi_vendor_eval.py b/inferencex-e2e/infx/evals/kimi_vendor_eval.py index 9cc9d5f4be..711ea19620 100755 --- a/inferencex-e2e/infx/evals/kimi_vendor_eval.py +++ b/inferencex-e2e/infx/evals/kimi_vendor_eval.py @@ -85,12 +85,17 @@ def build_pytest_command( def _mapping(value: Any, name: str) -> Mapping[str, Any]: + """Return value if it is a mapping, else raise ValueError naming it as name.""" if not isinstance(value, Mapping): raise ValueError(f"{name} must be an object") return value def _expected_total(task_name: str) -> int: + """Return the expected record count for task_name. + + Raises ValueError for an unsupported task. + """ try: return EXPECTED_TOTALS[task_name] except KeyError as exc: @@ -122,6 +127,11 @@ def _project_report( *, task_name: str = TASK_NAME, ) -> tuple[dict[str, Any], bool]: + """Validate a native verifier report and convert it to a compatibility result. + + Returns (result, True if every record passed). Raises ValueError if the report is malformed or + incomplete. + """ expected_total = _expected_total(task_name) root = _mapping(report, "report") summary = _mapping(root.get("summary"), "report.summary") @@ -244,6 +254,10 @@ def _compatibility_result( n_samples: int, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with score over n_samples. + + Records integration_error in the result when given. + """ expected_total = _expected_total(task_name) result: dict[str, Any] = { "result_format": RESULT_FORMAT, @@ -310,6 +324,7 @@ def _write_native_failure( def _write_compatibility(path: Path, result: Mapping[str, Any]) -> None: + """Write result to path as indented JSON.""" path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") @@ -420,6 +435,7 @@ def run_evaluation( def _positive_int(value: str) -> int: + """Parse value as an int, raising ArgumentTypeError unless it is at least 1.""" parsed = int(value) if parsed < 1: raise argparse.ArgumentTypeError("must be a positive integer") @@ -427,6 +443,10 @@ def _positive_int(value: str) -> int: def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse CLI arguments from argv. + + Requires --verifier-dir and --base-url unless --integration-error is set. + """ parser = argparse.ArgumentParser( description="Run the pinned stock Kimi Vendor Verifier tool-schema evaluation." ) @@ -463,6 +483,10 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Run the Kimi verifier eval, or record the given integration error. + + Returns 0 on pass or after recording an error, 1 on failure. + """ args = parse_args(argv) if args.integration_error is not None: args.output_dir.mkdir(parents=True, exist_ok=True) diff --git a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py index b0b3eb1077..2be094dc68 100755 --- a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py @@ -58,7 +58,8 @@ class FullSuiteError(RuntimeError): class _RejectRedirects(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args: Any, **kwargs: Any) -> None: # noqa: ARG002 - return None + """Refuse every redirect by returning None.""" + return None # noqa: RET501 _NO_REDIRECT_OPENER = urllib.request.build_opener(_RejectRedirects()) @@ -91,6 +92,10 @@ def verify_source_content(relative_path: str, content: bytes) -> None: def _fetch_source(relative_path: str) -> bytes: + """Download the pinned source file at relative_path, verify its hash and return its bytes. + + Retries transient failures and raises FullSuiteError once every attempt fails. + """ url = source_url(relative_path) request = urllib.request.Request( # noqa: S310 url, @@ -131,6 +136,10 @@ def _fetch_source(relative_path: str) -> bytes: def _validate_sample(content: bytes) -> None: + """Check that pinned sample.jsonl content has the expected number of JSON object rows. + + Raises ValueError otherwise. + """ try: text = content.decode("utf-8") except UnicodeDecodeError as exc: @@ -226,6 +235,7 @@ def build_verifier_command( def _error_dict(error: BaseException) -> dict[str, str]: + """Return the type name and message of error as a dict.""" return {"type": type(error).__name__, "message": str(error)} @@ -236,6 +246,10 @@ def _compatibility_result( effective: int, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with score over effective samples. + + Records integration_error in the result when given. + """ result: dict[str, Any] = { "result_format": RESULT_FORMAT, "eval_adapter": ADAPTER_NAME, @@ -272,6 +286,7 @@ def _compatibility_result( def _compatibility_path(output_dir: Path) -> Path: + """Delete stale compatibility results in output_dir and return a new timestamped result path.""" for stale_path in output_dir.glob(COMPATIBILITY_GLOB): stale_path.unlink() timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%f") @@ -279,10 +294,15 @@ def _compatibility_path(output_dir: Path) -> Path: def _write_json(path: Path, value: Mapping[str, Any]) -> None: + """Write value to path as indented UTF-8 JSON.""" path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _read_native_report(path: Path) -> dict[str, Any]: + """Load the native summary JSON object at path. + + Raises FullSuiteError if it is missing or invalid. + """ try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: @@ -293,6 +313,10 @@ def _read_native_report(path: Path) -> dict[str, Any]: def _read_native_results(path: Path) -> list[dict[str, Any]]: + """Load the native JSONL result rows at path. + + Raises FullSuiteError unless there are exactly the expected number of object rows. + """ try: lines = path.read_text(encoding="utf-8").removesuffix("\n").split("\n") except OSError as exc: @@ -314,12 +338,17 @@ def _read_native_results(path: Path) -> list[dict[str, Any]]: def _count(value: Any, name: str) -> int: + """Return value if it is a non-negative int, else raise FullSuiteError naming field name.""" if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise FullSuiteError(f"native summary {name} must be a non-negative integer") return value def _rate(value: Any, name: str) -> float: + """Return value as a float if it is a finite rate in [0, 1]. + + Raises FullSuiteError naming field name otherwise. + """ if ( isinstance(value, bool) or not isinstance(value, (int, float)) @@ -460,6 +489,7 @@ def run_full_suite( def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse argv into the prepare-source, run, project or failure subcommand and its options.""" parser = argparse.ArgumentParser( description="Run or project the pinned full MiniMax M3 verifier." ) @@ -488,6 +518,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Run the chosen subcommand and return 0 on success or 1 on failure.""" args = parse_args(argv) if args.command == "prepare-source": try: diff --git a/inferencex-e2e/infx/evals/minimax_provider_eval.py b/inferencex-e2e/infx/evals/minimax_provider_eval.py index 8f6c07d9d2..8f1a0c49cb 100755 --- a/inferencex-e2e/infx/evals/minimax_provider_eval.py +++ b/inferencex-e2e/infx/evals/minimax_provider_eval.py @@ -48,6 +48,7 @@ class SmokeSuiteError(RuntimeError): def _mapping(value: Any, name: str) -> Mapping[str, Any]: + """Return value if it is a mapping, else raise ValueError naming it as name.""" if not isinstance(value, Mapping): raise ValueError(f"{name} must be an object") return value @@ -82,6 +83,10 @@ def load_fixture(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: def _normalized_base_url(value: str) -> str: + """Return value stripped of whitespace and trailing slashes. + + Raises ValueError unless it is an absolute HTTP(S) URL without query or fragment. + """ if not isinstance(value, str) or not value.strip(): raise ValueError("base_url must be a non-empty string") normalized = value.strip().rstrip("/") @@ -147,10 +152,15 @@ def build_verifier_command( def _error_dict(error: BaseException) -> dict[str, str]: + """Return the type name and message of error as a dict.""" return {"type": type(error).__name__, "message": str(error)} def _rate(value: Any, name: str) -> float: + """Return value as a float if it is a finite rate in [0, 1]. + + Raises SmokeSuiteError naming field name otherwise. + """ if ( isinstance(value, bool) or not isinstance(value, (int, float)) @@ -162,12 +172,14 @@ def _rate(value: Any, name: str) -> float: def _nonnegative_count(value: Any, name: str) -> int: + """Return value if it is a non-negative int, else raise SmokeSuiteError naming field name.""" if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise SmokeSuiteError(f"native summary {name} must be a non-negative integer") return value def _compatibility_path(output_dir: Path) -> Path: + """Delete stale compatibility results in output_dir and return a new timestamped result path.""" for stale_path in output_dir.glob(COMPATIBILITY_GLOB): stale_path.unlink() timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%f") @@ -181,6 +193,10 @@ def _compatibility_result( effective: int, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with score over effective samples. + + Records integration_error in the result when given. + """ result: dict[str, Any] = { "result_format": RESULT_FORMAT, "eval_adapter": ADAPTER_NAME, @@ -218,6 +234,7 @@ def _compatibility_result( def _write_json(path: Path, value: Mapping[str, Any]) -> None: + """Write value to path as indented UTF-8 JSON.""" path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -351,6 +368,7 @@ def run_evaluation( def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse argv into the run or failure subcommand and its options.""" parser = argparse.ArgumentParser( description="Run the pinned MiniMax M3 smoke through stock verify.py." ) @@ -373,6 +391,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Run the smoke eval, or record the given failure, and return 0 on success or 1 on failure.""" args = parse_args(argv) if args.command == "failure": publish_failure( diff --git a/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py b/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py index 881a7b6c8a..fe8c17325e 100644 --- a/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py +++ b/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py @@ -7,6 +7,10 @@ def _parse_generations(outputs: Any, **kwargs: Any) -> list[Any]: # noqa: ARG001 + """Return the text of every choice in chat completion outputs. + + Falls back to reasoning_content when content is empty. + """ results = [] if not isinstance(outputs, list): outputs = [outputs] @@ -42,6 +46,11 @@ def _apply_chat_template( chat_history: list[dict[str, Any]], add_generation_prompt: bool = True, ) -> Any: + """Render chat_history for the configured tokenizer backend. + + Returns a templated string for local HF tokenizers, the raw messages for remote ones, or a + JSON chat string otherwise. + """ if self.tokenizer_backend == "huggingface" and self.tokenized_requests: return self.tokenizer.apply_chat_template( chat_history, diff --git a/inferencex-e2e/infx/evals/utils.py b/inferencex-e2e/infx/evals/utils.py index 29f55f8f5b..997927281d 100644 --- a/inferencex-e2e/infx/evals/utils.py +++ b/inferencex-e2e/infx/evals/utils.py @@ -8,6 +8,10 @@ def process_docs( dataset: datasets.Dataset, n_repeats: int = 2, seed: int = 3407 ) -> datasets.Dataset: + """Return dataset repeated n_repeats times with answers shuffled into A to D by seed. + + Each row gets its correct letter in answer and its repeat index in repeat_id. + """ rng = random.Random(seed) # noqa: S311 docs = list(dataset) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index e586720fa6..ad826b93e2 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -247,6 +247,10 @@ def validate_batch_manifest( def main() -> int: + """Check each eval result file against its score threshold, printing PASS or FAIL per metric. + + Returns 0 if all metrics pass, 1 if any fail or none matched. + """ for _stream in (sys.stdout, sys.stderr): with contextlib.suppress(AttributeError, ValueError): _stream.reconfigure(line_buffering=True) diff --git a/inferencex-e2e/infx/workflows/benchmark_schema.py b/inferencex-e2e/infx/workflows/benchmark_schema.py index e1952347ff..38ccb662ee 100644 --- a/inferencex-e2e/infx/workflows/benchmark_schema.py +++ b/inferencex-e2e/infx/workflows/benchmark_schema.py @@ -58,6 +58,11 @@ def _validate_rows( multinode: bool | None = None, agentic: bool | None = None, ) -> None: + """Validate each row in rows against its workflow schema. + + multinode and agentic force the schema choice when set, otherwise each row decides. Raises + ValueError located by path. + """ if not isinstance(rows, list): raise ValueError(f"{path}: expected a list of matrix rows") for index, row in enumerate(rows): @@ -106,6 +111,10 @@ def validate_matrix(matrix: object, *, plan: bool = False) -> None: def main() -> None: + """Validate the matrix JSON on stdin and echo it unchanged to stdout. + + Exits with a usage error if the matrix is invalid. + """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--plan", diff --git a/inferencex-e2e/infx/workflows/calc_success_rate.py b/inferencex-e2e/infx/workflows/calc_success_rate.py index 1a3f415e95..8efbadfdab 100644 --- a/inferencex-e2e/infx/workflows/calc_success_rate.py +++ b/inferencex-e2e/infx/workflows/calc_success_rate.py @@ -16,6 +16,7 @@ def load_hardware_labels() -> list[str]: def build_hardware_match_patterns( hardware_labels: Iterable[str], ) -> dict[str, tuple[re.Pattern[str], ...]]: + """Map each hardware label to whole-word regexes for it and its cluster label.""" return { hardware: tuple( re.compile(rf"(? str | None: + """Return the first hardware label whose patterns match job_name, or None if nothing matches. + + Builds patterns from runners.yaml when match_patterns is omitted. + """ job_lower = job_name.lower() if match_patterns is None: match_patterns = build_hardware_match_patterns(load_hardware_labels()) @@ -39,6 +44,7 @@ def extract_hardware_from_name( def calculate_hardware_success_rates() -> dict[str, dict[str, int]]: + """Count successful and total non-skipped jobs per hardware in the current Actions run.""" hardware_labels = load_hardware_labels() patterns = build_hardware_match_patterns(hardware_labels) run_id = int(os.environ["GITHUB_RUN_ID"]) @@ -82,6 +88,7 @@ def print_success_rates(success_rates: dict[str, dict[str, int]] | None) -> None def main() -> None: + """Print per-hardware success rates for this run and write them to .json.""" run_stats = calculate_hardware_success_rates() print_success_rates(run_stats) diff --git a/inferencex-e2e/infx/workflows/ci_priority.py b/inferencex-e2e/infx/workflows/ci_priority.py index 85ce5f05e0..8f3b5a7220 100755 --- a/inferencex-e2e/infx/workflows/ci_priority.py +++ b/inferencex-e2e/infx/workflows/ci_priority.py @@ -42,10 +42,15 @@ class PriorityContext: def _decimal(value: Any) -> Decimal: + """Convert value to an exact Decimal via its string form.""" return Decimal(str(value)) def load_policy(path: str | Path) -> dict[str, Any]: + """Load the priority policy YAML at path. + + Raises ValueError on an unsupported version or unknown keys. + """ with Path(path).open() as policy_file: policy = yaml.safe_load(policy_file) if policy.get("version") != 1: @@ -59,6 +64,10 @@ def load_policy(path: str | Path) -> dict[str, Any]: def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal: + """Return the adjustment for the first prefix matching value, or 0 if none matches. + + A prefix matches when value equals it or starts with it followed by a hyphen. + """ for prefix, adjustment in adjustments.items(): if value == prefix or value.startswith(f"{prefix}-"): return _decimal(adjustment) @@ -88,6 +97,10 @@ def _entry_from_criteria( policy: dict[str, Any], entry: dict[str, Any], ) -> dict[str, Any]: + """Project entry onto only the fields named in criteria, for priority grouping. + + Raises ValueError if criteria contains names the policy does not support. + """ unknown = criteria - supported_criteria(policy) if unknown: raise ValueError(f"Unknown CI priority criteria: {sorted(unknown)}") @@ -175,10 +188,12 @@ def calculate_priority( def format_priority(score: Decimal) -> str: + """Format score with three decimals.""" return f"{score:.3f}" def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: + """Return a 32-hex-character hash of value, namespace and path for queue grouping.""" canonical = json.dumps(value, sort_keys=True, separators=(",", ":")) material = f"{namespace}:{'/'.join(path)}:{canonical}".encode() return hashlib.sha256(material).hexdigest()[:32] @@ -211,6 +226,10 @@ def annotate_jobs( def _criteria_from_json(raw_criteria: str) -> frozenset[str] | None: + """Parse --criteria-json into a set of names, or None when empty or null. + + Raises ValueError unless it is a JSON array of strings. + """ if not raw_criteria: return None value = json.loads(raw_criteria) @@ -222,6 +241,7 @@ def _criteria_from_json(raw_criteria: str) -> frozenset[str] | None: def main() -> int: + """Annotate the matrix JSON from --input or stdin with priorities, print it and return 0.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--policy", default=str(repository_root() / "configs/ci-priority.yaml")) parser.add_argument("--event-name", default="") diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index 698085c033..f541a9d64d 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -40,8 +40,6 @@ from github import Auth, Github, GithubException, RateLimitExceededException - # PyGithub surfaces network failures as requests exceptions, which are not builtin ones, - # and raises RetryError once its own 5xx and rate-limit retries run out. _NETWORK_ERRORS: tuple[type[Exception], ...] = ( ConnectionError, TimeoutError, @@ -120,14 +118,17 @@ def changelog_path() -> str: def log(msg: str) -> None: + """Print msg as a progress line.""" print(f"{_CYAN_BOLD}→{_RESET} {msg}") def ok(msg: str) -> None: + """Print msg as a success line.""" print(f"{_GREEN_BOLD}✓{_RESET} {msg}") def die(msg: str) -> int: + """Print msg as an error on stderr and return exit code 1.""" print(f"{_RED_BOLD}✗{_RESET} {msg}", file=sys.stderr) return 1 @@ -167,6 +168,8 @@ def _is_transient_error(exc: Exception) -> bool: Covers server errors (5xx), rate limits (429, and 403 rate-limit responses), and connection-level failures from requests/urllib3. + PyGithub reports network failures as requests exceptions and raises + RetryError once its own retries run out, so both count as transient. """ if isinstance(exc, RateLimitExceededException): return True @@ -198,17 +201,21 @@ class GitOps: """ def __init__(self, repo: gitpython.Repo | None = None) -> None: + """Wrap repo, or the repository containing the current directory.""" self.repo = repo or gitpython.Repo(".", search_parent_directories=True) def is_clean(self) -> bool: + """Return True if the working tree has no changes or untracked files.""" return not self.repo.is_dirty(untracked_files=True) def current_ref(self) -> str: + """Return the current branch name, or the commit SHA when HEAD is detached.""" if self.repo.head.is_detached: return self.repo.head.commit.hexsha return self.repo.active_branch.name def rev_parse(self, ref: str = "HEAD") -> str: + """Return the full commit SHA that ref resolves to.""" return self.repo.rev_parse(ref).hexsha def diff_name_only_unmerged(self) -> str: @@ -239,9 +246,11 @@ def show_stage(self, stage: int, path: str) -> bytes: ) def fetch(self, *args: str) -> None: + """Run git fetch with args.""" self.repo.git.fetch(*args) def checkout(self, *args: str) -> None: + """Run git checkout with args.""" self.repo.git.checkout(*args) def merge(self, *args: str) -> int: @@ -253,19 +262,24 @@ def merge(self, *args: str) -> int: return exc.status or 1 def add(self, *paths: str) -> None: + """Stage paths with git add.""" self.repo.git.add(*paths) def commit(self, *args: str) -> None: + """Run git commit with args.""" self.repo.git.commit(*args) def push(self, *args: str) -> None: + """Run git push with args.""" self.repo.git.push(*args) def branch_delete(self, name: str) -> None: + """Force-delete local branch name, ignoring git errors.""" with contextlib.suppress(gitpython.GitCommandError): self.repo.git.branch("-D", name) def merge_abort(self) -> None: + """Abort an in-progress merge, ignoring git errors.""" with contextlib.suppress(gitpython.GitCommandError): self.repo.git.merge("--abort") @@ -446,6 +460,8 @@ def wait_for_check( Returns 0 on success, 1 on failure or timeout. Transient API errors are retried with backoff until the timeout expires. + Skipped runs are ignored when another exists, because unrelated label + events re-run the workflow with this job skipped on the same SHA. """ log(f"Waiting for {check_name} on {sha[:8]}") deadline = time.monotonic() + timeout @@ -455,7 +471,6 @@ def wait_for_check( try: check_runs = list(commit.get_check_runs()) matching = [cr for cr in check_runs if cr.name == check_name] - # Unrelated label events re-run the workflow with this job skipped on the same SHA. matching = [cr for cr in matching if cr.conclusion != "skipped"] or matching if matching: matching.sort(key=_check_run_recency) @@ -529,6 +544,7 @@ class _MergeState: __slots__ = ("local_branch",) def __init__(self) -> None: + """Start with no local branch recorded.""" self.local_branch = "" @@ -560,6 +576,7 @@ def merge_pr( state = _MergeState() def cleanup() -> None: + """Abort any merge, restore the original branch and delete the temporary local branch.""" git_ops.merge_abort() with contextlib.suppress(Exception): git_ops.checkout("--quiet", original_branch) @@ -599,6 +616,10 @@ def _merge_pr_inner( git_ops: GitOps, state: _MergeState, ) -> int: + """Merge PR number pr once its reusable sweep, changelog and checks are in place. + + Returns 0 on success, non-zero on failure. Records the local branch in state for cleanup. + """ gh_repo = gh.get_repo(repo) pull = gh_repo.get_pull(pr) @@ -750,6 +771,7 @@ def _poll_pr_head( def main() -> int: + """Merge the PR named on the command line using env settings and return the exit code.""" if not _WORKFLOWS_AVAILABLE: print(_INSTALL_HINT, file=sys.stderr) return 1 diff --git a/inferencex-e2e/infx/workflows/pareto_coverage.py b/inferencex-e2e/infx/workflows/pareto_coverage.py index 5e94eb0715..ae9aa7d4d3 100644 --- a/inferencex-e2e/infx/workflows/pareto_coverage.py +++ b/inferencex-e2e/infx/workflows/pareto_coverage.py @@ -17,10 +17,15 @@ def _positive_finite(value: Any) -> bool: + """Return True if value is a positive finite int or float.""" return type(value) in (int, float) and value > 0 and math.isfinite(value) def assess_curve(curve: dict[str, Any]) -> dict[str, Any]: + """Compute the Pareto frontier of one curve and return its coverage summary. + + Raises ValueError if the curve lacks a key or a points list. + """ key, points = curve.get("key"), curve.get("points") if not isinstance(key, str) or not key.strip() or not isinstance(points, list): raise ValueError("Each curve needs a nonempty key and a points array") @@ -59,6 +64,10 @@ def assess_curve(curve: dict[str, Any]) -> dict[str, Any]: def assess_coverage(curves: Any) -> list[dict[str, Any]]: + """Assess every curve in curves and return their summaries. + + Raises ValueError on empty input, non-object curves or duplicate keys. + """ if not isinstance(curves, list) or not curves: raise ValueError("Provide every affected curve; an empty list cannot prove coverage") results = [] @@ -75,6 +84,7 @@ def assess_coverage(curves: Any) -> list[dict[str, Any]]: def main() -> None: + """Read curves JSON from stdin and print the coverage report, exiting 1 on invalid input.""" try: results = assess_coverage(json.load(sys.stdin)) print(json.dumps(results, indent=2, allow_nan=False)) diff --git a/inferencex-e2e/infx/workflows/require_launcher.py b/inferencex-e2e/infx/workflows/require_launcher.py index 6147512874..72ded041f0 100644 --- a/inferencex-e2e/infx/workflows/require_launcher.py +++ b/inferencex-e2e/infx/workflows/require_launcher.py @@ -15,6 +15,10 @@ def main(argv: list[str] | None = None) -> int: + """Check that the checkout in argv contains the Python launcher. + + Returns 0 if it does, else prints a CI error and returns 1. + """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("checkout", type=Path, help="repository checkout of the measured revision") checkout = parser.parse_args(argv).checkout diff --git a/inferencex-e2e/infx/workflows/reuse.py b/inferencex-e2e/infx/workflows/reuse.py index 599933cb1a..8fe862aa4b 100644 --- a/inferencex-e2e/infx/workflows/reuse.py +++ b/inferencex-e2e/infx/workflows/reuse.py @@ -119,6 +119,7 @@ def find_reuse_authorization( command: str, allowed_author_associations: set[str], ) -> tuple[bool, int | None]: + """Return whether an allowed reuse command exists on the PR, and the run ID it pins if any.""" comment, pinned_run_id = find_reuse_request( repo, pr_number, @@ -250,6 +251,10 @@ def resolve_reusable_run( def main() -> int: + """Decide from CLI arguments whether the sweep can reuse a prior run. + + Writes the outputs and returns the exit code. Raises RuntimeError if no GitHub token is set. + """ parser = argparse.ArgumentParser() parser.add_argument("--repo", required=True) parser.add_argument("--commit-sha", required=True) diff --git a/inferencex-e2e/infx/workflows/reuse_comment.py b/inferencex-e2e/infx/workflows/reuse_comment.py index f0a7bab6b0..4fff9184e5 100644 --- a/inferencex-e2e/infx/workflows/reuse_comment.py +++ b/inferencex-e2e/infx/workflows/reuse_comment.py @@ -49,6 +49,10 @@ def warn_outside_pr( def acknowledge(repo: str, event: dict[str, Any], token: str) -> int: + """React to a reuse command comment on a PR and report whether it was accepted. + + Returns 0 when accepted or not applicable, 1 when the request is rejected. + """ if event.get("action") not in {"created", "edited"} or not event.get("issue", {}).get( "pull_request" ): @@ -122,6 +126,10 @@ def acknowledge(repo: str, event: dict[str, Any], token: str) -> int: def main() -> int: + """Acknowledge the comment from the GitHub Actions event and return the exit code. + + Raises RuntimeError if no GitHub token is set. + """ token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") if not token: raise RuntimeError("GH_TOKEN or GITHUB_TOKEN is required") diff --git a/inferencex-e2e/infx/workflows/signoff_publish.py b/inferencex-e2e/infx/workflows/signoff_publish.py index 00d6769469..0d74da42c3 100644 --- a/inferencex-e2e/infx/workflows/signoff_publish.py +++ b/inferencex-e2e/infx/workflows/signoff_publish.py @@ -41,6 +41,7 @@ def normalize_verdict(verdict: str) -> str: + """Return verdict with pass emojis in N/A rows replaced by the neutral marker.""" return NA_ROW_WITH_PASS_EMOJI.sub(r"\1➖\2", verdict) # noqa: RUF001 @@ -97,18 +98,25 @@ def verdict_errors(verdict: str) -> list[str]: def marker(signoff_key: str) -> str: + """Return the hidden HTML marker for signoff_key, raising ValueError if the key is invalid.""" if SIGNOFF_KEY.fullmatch(signoff_key) is None: raise ValueError("Invalid sign-off key") return f"" def _is_verdict_for(comment: dict[str, Any], signoff_key: str) -> bool: + """Return True if comment is a verifier bot's verdict for signoff_key.""" author = comment.get("user") or {} body = comment.get("body") or "" return author.get("login") in VERIFIER_AUTHORS and body.startswith(f"{marker(signoff_key)}\n") def verdict_body(verdict: str, head_sha: str, signoff_key: str) -> tuple[str, str]: + """Build the PR comment body and status for a verdict on head_sha. + + Returns (body, status) where status is success, warning or failure. Invalid verdicts become a + failure notice. + """ verdict = normalize_verdict(verdict) checks, errors = parse_verdict(verdict) if errors: @@ -143,6 +151,11 @@ def publish( *, verification_succeeded: bool, ) -> None: + """Post or update the PR verdict comment for signoff_key on head_sha. + + Uses the verdict file only when verification succeeded, and reposts if the old comment was + deleted. + """ verdict = "" if verification_succeeded and verdict_path.exists(): verdict = verdict_path.read_text(encoding="utf-8").strip() @@ -171,6 +184,7 @@ def publish( def main() -> None: + """Publish the sign-off verdict using settings from the environment.""" publish( os.environ["GITHUB_REPOSITORY"], os.environ["GH_TOKEN"], diff --git a/inferencex-e2e/infx/workflows/signoff_resolve.py b/inferencex-e2e/infx/workflows/signoff_resolve.py index 20d322c667..915db906eb 100644 --- a/inferencex-e2e/infx/workflows/signoff_resolve.py +++ b/inferencex-e2e/infx/workflows/signoff_resolve.py @@ -14,6 +14,7 @@ def _positive_id(value: Any) -> int: + """Return value as an int, raising ValueError unless it is a positive decimal ID.""" if isinstance(value, bool) or not re.fullmatch(r"[1-9][0-9]*", str(value)): raise ValueError("Expected a positive GitHub resource ID") return int(value) @@ -22,6 +23,11 @@ def _positive_id(value: Any) -> int: def _reference( event_name: str, event: dict[str, Any], repo: str, token: str ) -> tuple[int, str, str, str, str]: + """Locate the sign-off comment for an event and check it belongs to the PR. + + Returns (PR number, author, comment kind, sign-off key, gh command to fetch the body). Raises + ValueError on a mismatched or invalid reference. + """ if event_name == "workflow_dispatch": inputs = event["inputs"] match = re.fullmatch( @@ -83,6 +89,11 @@ def resolve( scoped_head_sha: str, token: str, ) -> dict[str, str]: + """Decide whether an event is a trusted sign-off request and return the workflow outputs. + + Returns {"proceed": "false"} for ignored events. Raises ValueError for bad input and + RuntimeError if the PR head moved. + """ if event_name not in { "issue_comment", "pull_request_review", @@ -131,6 +142,7 @@ def resolve( def main() -> None: + """Resolve the sign-off from the GitHub Actions environment and write the step outputs.""" event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) outputs = resolve( os.environ["GITHUB_REPOSITORY"], diff --git a/inferencex-e2e/infx/workflows/signoff_scope.py b/inferencex-e2e/infx/workflows/signoff_scope.py index d307294263..59005b2e08 100644 --- a/inferencex-e2e/infx/workflows/signoff_scope.py +++ b/inferencex-e2e/infx/workflows/signoff_scope.py @@ -15,6 +15,11 @@ def required_owners(repo: str, pr: dict[str, Any], token: str) -> list[str]: + """Return the base CODEOWNERS owners of the PR's changed files who must sign off. + + Skips the core team and admin users. Raises RuntimeError if files or CODEOWNERS cannot be read + reliably. + """ files = github.paginate(repo, f"/pulls/{pr['number']}/files", token, "") if len({file["filename"] for file in files}) != pr["changed_files"]: raise RuntimeError("Incomplete changed-file list; cannot determine sign-off scope") @@ -52,6 +57,10 @@ def required_owners(repo: str, pr: dict[str, Any], token: str) -> list[str]: def check_scope(repo: str, number: int, token: str) -> dict[str, str]: + """Return workflow outputs saying whether PR number needs a sign-off. + + Raises RuntimeError if the PR changed during the check. + """ pr = github.api(repo, f"/pulls/{number}", token) required = required_owners(repo, pr, token) current = github.api(repo, f"/pulls/{number}", token) @@ -75,6 +84,10 @@ def check_scope(repo: str, number: int, token: str) -> dict[str, str]: def main() -> None: + """Find the PR from the GitHub event, check its sign-off scope and write the step outputs. + + Raises ValueError if a dispatch pr-number does not match comment_url. + """ event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) number = (event.get("pull_request") or event.get("issue") or {}).get("number") if number is None: diff --git a/inferencex-e2e/infx/workflows/signoff_validate.py b/inferencex-e2e/infx/workflows/signoff_validate.py index 37be3c5813..e9c54d568f 100644 --- a/inferencex-e2e/infx/workflows/signoff_validate.py +++ b/inferencex-e2e/infx/workflows/signoff_validate.py @@ -17,6 +17,10 @@ def main(argv: list[str] | None = None) -> int: + """Validate the verdict file named in argv and print the result. + + Returns 0 if valid, 1 if invalid, 2 on usage or read errors. + """ args = sys.argv[1:] if argv is None else argv if len(args) != 1: print("usage: python -m infx.workflows.signoff_validate ", file=sys.stderr) diff --git a/inferencex-e2e/infx/workflows/stage_results.py b/inferencex-e2e/infx/workflows/stage_results.py index b8c99e3287..680343fdd1 100644 --- a/inferencex-e2e/infx/workflows/stage_results.py +++ b/inferencex-e2e/infx/workflows/stage_results.py @@ -23,6 +23,10 @@ def timestamp(value: str | None) -> float: + """Return the POSIX time of an ISO-8601 value, or infinity if it is invalid. + + Naive times are treated as UTC. + """ try: parsed = datetime.fromisoformat(value or "") return (parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)).timestamp() @@ -31,6 +35,7 @@ def timestamp(value: str | None) -> float: def full_sweep_at(events: list[dict[str, Any]], created_at: str) -> bool: + """Return True if a full-sweep label was on the PR at created_at, replaying label events.""" created = timestamp(created_at) if created == float("inf"): return False @@ -50,10 +55,15 @@ def full_sweep_at(events: list[dict[str, Any]], created_at: str) -> bool: def resolve_run( repo: str, pr_number: int, pull: dict[str, Any], token: str, requested_id: str | None ) -> dict[str, Any]: + """Return the requested_id run, or else the first eligible completed sweep run on the PR. + + Raises RuntimeError naming why no run qualifies. + """ timeline = github.paginate(repo, f"/issues/{pr_number}/timeline", token) commits = sweep_runs.pr_commit_shas(repo, pr_number, token) def rejection(run: dict[str, Any]) -> str: + """Return why run cannot be staged, or an empty string if it can.""" if ( run.get("path") != ".github/workflows/run-sweep.yml" or run.get("event") != "pull_request" @@ -98,10 +108,16 @@ def use_command(body: str) -> re.Match[str] | None: def request(repo: str, event: dict[str, Any], token: str) -> dict[str, str] | None: + """Validate a /stage-results comment event and return the staging outputs. + + Returns None if the comment is not a staging request. Raises RuntimeError after posting a reply + when the request is rejected. + """ pr_number = event["issue"]["number"] actor = event["comment"]["user"]["login"] def reject(reason: str, comment: str) -> NoReturn: + """Post comment on the PR and raise RuntimeError with reason.""" github.api( repo, f"/issues/{pr_number}/comments", token, method="POST", data={"body": comment} ) @@ -159,6 +175,7 @@ def reject(reason: str, comment: str) -> NoReturn: def main() -> None: + """Handle the comment event from the environment and append staging outputs to GITHUB_OUTPUT.""" event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) outputs = request(os.environ["GITHUB_REPOSITORY"], event, os.environ["GH_TOKEN"]) if outputs is None: diff --git a/inferencex-e2e/infx/workflows/stage_results_callback.py b/inferencex-e2e/infx/workflows/stage_results_callback.py index 91ce241f1b..9e4ce8210f 100644 --- a/inferencex-e2e/infx/workflows/stage_results_callback.py +++ b/inferencex-e2e/infx/workflows/stage_results_callback.py @@ -16,6 +16,10 @@ def chart_links(payload: dict[str, Any], run_id: str, run_date: str) -> list[str]: + """Return markdown chart links from payload, or a default link for run_id on run_date. + + Raises RuntimeError if any link is malformed or not on the staging site. + """ links = payload.get("chart-links") if links is None: return [f"{STAGING_SITE}/inference?i_dates={quote(f'{run_date}~r{run_id}')}"] @@ -47,6 +51,10 @@ def chart_links(payload: dict[str, Any], run_id: str, run_date: str) -> list[str def report(repo: str, event: dict[str, Any], token: str, server_url: str) -> None: + """Post or update the PR comment for a staging callback event. + + Raises RuntimeError if the payload is malformed or from an unexpected source. + """ payload = event["client_payload"] if payload.get("source-repository") != "SemiAnalysisAI/InferenceX-app": raise RuntimeError("Unsupported staging callback source") @@ -103,6 +111,7 @@ def report(repo: str, event: dict[str, Any], token: str, server_url: str) -> Non def main() -> None: + """Report the staging callback from the GitHub Actions environment.""" event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) report( os.environ["GITHUB_REPOSITORY"], diff --git a/inferencex-e2e/infx/workflows/sweep_runs.py b/inferencex-e2e/infx/workflows/sweep_runs.py index 1c916e41ea..d0d5c61f38 100644 --- a/inferencex-e2e/infx/workflows/sweep_runs.py +++ b/inferencex-e2e/infx/workflows/sweep_runs.py @@ -16,6 +16,7 @@ def completed_pr_runs( repo: str, workflow_id: str, head_branch: str, token: str ) -> list[dict[str, Any]]: + """Return every completed pull_request run of workflow_id on head_branch.""" workflow = urllib.parse.quote(workflow_id, safe="") return github.paginate( repo, diff --git a/inferencex-e2e/infx/workflows/validate_perf_changelog.py b/inferencex-e2e/infx/workflows/validate_perf_changelog.py index f322f1505e..643ae4e0e6 100644 --- a/inferencex-e2e/infx/workflows/validate_perf_changelog.py +++ b/inferencex-e2e/infx/workflows/validate_perf_changelog.py @@ -46,7 +46,11 @@ def read_git_file(ref: str, path: str) -> bytes: def parse_changelog(raw: bytes, label: str) -> list[dict[str, Any]]: - """Validate file-level invariants and return raw YAML entry mappings.""" + """Validate file-level invariants and return raw YAML entry mappings. + + Rejects NEL, LS and PS because YAML breaks lines there while the + byte-level append checks only see newlines. + """ if not raw.endswith(b"\n"): raise ChangelogValidationError(f"{label} does not end with a newline") if b"\r" in raw: @@ -60,7 +64,6 @@ def parse_changelog(raw: bytes, label: str) -> list[dict[str, Any]]: text = raw.decode("utf-8") except UnicodeDecodeError as exc: raise ChangelogValidationError(f"{label} is not UTF-8: {exc}") from exc - # YAML also breaks lines at NEL, LS and PS; the byte-level append checks only see "\n". if any(line_break in text for line_break in "\x85\u2028\u2029"): raise ChangelogValidationError(f"{label} contains a line break other than newline") From 6c6f3cbb07277078d54053105892f9e5890d39c4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:33:55 -0500 Subject: [PATCH 197/203] docs(bench): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete the # comments in infx/bench and infx/bench_serving and give every function and method a short docstring that names its inputs and result. Notes a reader needs, such as TGI and MII sampling limits and the huggingface-hub pin, now live in those docstrings. SPDX license tags stay. 中文:删除 infx/bench 与 infx/bench_serving 中的 # 注释,为每个函数和方法 补充简短的文档字符串,说明输入与返回值。读者需要的信息(例如 TGI 和 MII 的采样限制、huggingface-hub 版本约束)已并入相应的文档字符串。SPDX 许可证标记保留。 --- inferencex-e2e/infx/bench/agentic/run.py | 12 +++-- inferencex-e2e/infx/bench/agentic/venv.py | 5 ++ inferencex-e2e/infx/bench/eval/__init__.py | 5 ++ inferencex-e2e/infx/bench/eval/lm_eval.py | 9 ++-- inferencex-e2e/infx/bench/eval/meta.py | 5 ++ inferencex-e2e/infx/bench/eval/stage.py | 1 + inferencex-e2e/infx/bench/eval/vendor.py | 8 +++ inferencex-e2e/infx/bench/fixed_seq.py | 9 ++-- inferencex-e2e/infx/bench/proc.py | 12 ++++- inferencex-e2e/infx/bench/server.py | 4 +- .../bench_serving/backend_request_func.py | 50 ++++++++++-------- .../infx/bench_serving/benchmark_outcome.py | 5 +- .../infx/bench_serving/benchmark_serving.py | 52 +++++++++++-------- .../infx/bench_serving/benchmark_utils.py | 6 +-- .../infx/bench_serving/encoding_dsv4.py | 27 ++-------- 15 files changed, 124 insertions(+), 86 deletions(-) diff --git a/inferencex-e2e/infx/bench/agentic/run.py b/inferencex-e2e/infx/bench/agentic/run.py index e4fda327af..0926883c18 100644 --- a/inferencex-e2e/infx/bench/agentic/run.py +++ b/inferencex-e2e/infx/bench/agentic/run.py @@ -77,7 +77,7 @@ def from_env(cls, env: Mapping[str, str]) -> Plan: def _require_single_point(env: Mapping[str, str], conc: str) -> None: - """AgentX measures one concurrency per fresh server deployment.""" + """Raise ``InputError`` unless ``CONC_LIST`` is unset or equals ``conc``, as AgentX measures one concurrency per server.""" points = env.get("CONC_LIST") if points is not None and points != conc: raise inputs.InputError( @@ -88,7 +88,7 @@ def _require_single_point(env: Mapping[str, str], conc: str) -> None: def _validate_kv_offload(env: Mapping[str, str]) -> None: - """The served KV-offload configuration, as the matrix ``kv-offloading`` field allows.""" + """Raise ``InputError`` unless ``env``'s KV-offload settings match a matrix ``kv-offloading`` value.""" mode = inputs.require("KV_OFFLOADING", env=env)["KV_OFFLOADING"] backend = env.get("KV_OFFLOAD_BACKEND") if mode == "none": @@ -105,6 +105,7 @@ def _validate_kv_offload(env: Mapping[str, str]) -> None: def _power_mode(switches: Mapping[str, str], env: Mapping[str, str]) -> PowerMode: + """Pick the power mode from ``ENABLE_AGENTX_POWER`` and the srt-slurm window directory.""" if switches.get("ENABLE_AGENTX_POWER") not in TRUE_VALUES: return "off" return "window" if env.get("SRT_MEASUREMENT_WINDOW_DIR") else "missing" @@ -144,6 +145,7 @@ def execute(plan: Plan, runtime: Runtime, environ: Mapping[str, str]) -> int: def _download_traces(cfg: ReplayConfig, runtime: Runtime, env: Mapping[str, str]) -> int: + """Download ``cfg``'s trace dataset with the runtime's ``hf`` and return its status.""" print(f"Loading traces via aiperf public-dataset: {cfg.loader} ({cfg.dataset})", flush=True) rc = proc.call([str(runtime.hf), "download", "--repo-type", "dataset", cfg.dataset], env) if rc: @@ -195,21 +197,24 @@ def _power_audit(plan: Plan, replay_rc: int) -> list[str] | None: def _window(plan: Plan, state: str) -> list[str]: + """Power adapter arguments that write the multinode window in ``state``.""" return ["--concurrency", str(plan.replay.concurrency), "--write-multinode-window", state] def _power_adapter(plan: Plan, python: str, env: Mapping[str, str], *args: str) -> int: + """Run the power adapter for ``plan`` with ``args`` and return its status.""" strict = ["--require-power"] if plan.require_power else [] adapter = ["agentic.power_adapter", "--result-dir", str(plan.result_dir), *args, *strict] return _results(python, env, *adapter) def _results(python: str, env: Mapping[str, str], module: str, *args: str) -> int: + """Run ``infx.results.`` with ``args`` under ``python`` and return its status.""" return proc.call([python, "-m", f"infx.results.{module}", *args], env) def _check_server_metrics(artifact_dir: Path, prefix: str | None) -> None: - """Opt-in: fail rather than publish trace charts without the engine's metrics.""" + """When ``prefix`` is set, raise ``BenchError`` unless ``artifact_dir`` has non-empty metrics with that prefix.""" if not prefix: return exported = artifact_dir / "server_metrics_export.json" @@ -235,6 +240,7 @@ def _contains(path: Path, needle: bytes) -> bool: def _nonempty(path: Path) -> bool: + """Whether ``path`` is a file with at least one byte.""" return path.is_file() and path.stat().st_size > 0 diff --git a/inferencex-e2e/infx/bench/agentic/venv.py b/inferencex-e2e/infx/bench/agentic/venv.py index 779df64348..9f256a8a87 100644 --- a/inferencex-e2e/infx/bench/agentic/venv.py +++ b/inferencex-e2e/infx/bench/agentic/venv.py @@ -35,18 +35,22 @@ def for_job(cls, environ: Mapping[str, str]) -> Runtime: @property def venv(self) -> Path: + """The venv directory under ``root``.""" return self.root / "venv" @property def python(self) -> Path: + """The venv's python interpreter.""" return self.venv / "bin" / "python" @property def agentx(self) -> Path: + """The venv's ``agentx`` executable.""" return self.venv / "bin" / "agentx" @property def hf(self) -> Path: + """The venv's ``hf`` executable.""" return self.venv / "bin" / "hf" def active(self) -> bool: @@ -83,4 +87,5 @@ def bootstrap(runtime: Runtime) -> int: def _executable(path: Path) -> bool: + """Whether ``path`` is a file this process may execute.""" return path.is_file() and os.access(path, os.X_OK) diff --git a/inferencex-e2e/infx/bench/eval/__init__.py b/inferencex-e2e/infx/bench/eval/__init__.py index 6e91b056cb..ad947efad3 100644 --- a/inferencex-e2e/infx/bench/eval/__init__.py +++ b/inferencex-e2e/infx/bench/eval/__init__.py @@ -53,6 +53,7 @@ def main(argv: list[str]) -> int: def _server_root(endpoint: str) -> str: + """Strip trailing slashes from ``endpoint``, raising ``InputError`` unless it is an http(s) root.""" root = endpoint.rstrip("/") try: parts = urllib.parse.urlsplit(root) @@ -65,6 +66,7 @@ def _server_root(endpoint: str) -> str: def _concurrencies(values: str) -> list[int]: + """Parse the space-separated ``values`` into positive ints, raising ``InputError`` if none.""" concurrencies = [env.parse_positive_int("--concurrency", value) for value in values.split()] if not concurrencies: raise env.InputError("--concurrency must name at least one concurrency") @@ -144,6 +146,7 @@ def _record( suite: str, batch: Mapping[str, Sequence[int]] | None = None, ) -> bool: + """Write ``meta_env.json`` into ``destination`` and return whether that succeeded.""" path = destination / "meta_env.json" try: destination.mkdir(parents=True, exist_ok=True) @@ -161,6 +164,7 @@ def _single( destination: Path, environ: Mapping[str, str], ) -> int: + """Run one eval at ``conc``, stage and record it, and return the exit code.""" outcome, staged = _run(framework, context, conc, destination, "") recorded = _record(destination, environ, conc, outcome.suite) if outcome.returncode: @@ -179,6 +183,7 @@ def _batched( destination: Path, environ: Mapping[str, str], ) -> int: + """Run lm-eval at each concurrency and record the batch manifest, returning 1 if that fails.""" completed: list[int] = [] failed: list[int] = [] suite = "" diff --git a/inferencex-e2e/infx/bench/eval/lm_eval.py b/inferencex-e2e/infx/bench/eval/lm_eval.py index 98eafba81e..7f146b7fe8 100644 --- a/inferencex-e2e/infx/bench/eval/lm_eval.py +++ b/inferencex-e2e/infx/bench/eval/lm_eval.py @@ -24,7 +24,10 @@ def install(environ: Mapping[str, str]) -> None: - """Install lm-eval at ``REF`` into this interpreter; failures only warn, as images may ship it.""" + """Install lm-eval at ``REF`` into this interpreter. Failures only warn, as images may ship it. + + The ``huggingface-hub`` range keeps lm-eval from upgrading it past what transformers 5.x accepts. + """ try: uv.find() except env.BenchError as error: @@ -32,8 +35,6 @@ def install(environ: Mapping[str, str]) -> None: return if "atom" in environ.get("IMAGE", ""): _pip(environ, "uninstall", "torchvision") - # lm-eval does not depend on transformers, so without this range its resolution - # upgrades huggingface-hub past what transformers 5.x accepts. _pip(environ, "install", "lm-eval[api]", "huggingface-hub>=1.5,<2") pinned = ("install", "--no-deps", "--reinstall") git = shutil.which("git", path=environ.get("PATH")) @@ -43,6 +44,7 @@ def install(environ: Mapping[str, str]) -> None: def _pip(environ: Mapping[str, str], command: str, *args: str) -> bool: + """Run ``uv pip ``, warning on failure, and return whether it succeeded.""" argv = uv.pip(command, "-q", "--no-cache", "--break-system-packages", *args) rc = proc.call(argv, environ) if rc: @@ -98,6 +100,7 @@ def max_output_tokens(context: int) -> int: def tasks(environ: Mapping[str, str]) -> str: + """``EVAL_TASKS_DIR``, else the default task set.""" return env.optional("EVAL_TASKS_DIR", env=environ) or DEFAULT_TASKS diff --git a/inferencex-e2e/infx/bench/eval/meta.py b/inferencex-e2e/infx/bench/eval/meta.py index 81e672930e..a410195b1a 100644 --- a/inferencex-e2e/infx/bench/eval/meta.py +++ b/inferencex-e2e/infx/bench/eval/meta.py @@ -19,10 +19,12 @@ def _first(values: Mapping[str, str], *names: str, default: str) -> str: + """The first non-empty value among ``names``, else ``default``.""" return next((values[name] for name in names if values.get(name)), default) def _is_true(value: str) -> bool: + """Whether ``value`` spells true, ignoring case.""" return value.lower() in _TRUE @@ -60,6 +62,7 @@ def build( values.update(_disaggregated(values)) def size(field: str, *names: str) -> int: + """The first set of ``names`` as an int, else 1, raising ``InputError`` if it is not a digit string.""" name = next((name for name in names if values.get(name)), None) if name is None: return 1 @@ -70,6 +73,7 @@ def size(field: str, *names: str) -> int: return int(values[name]) def dp(*names: str) -> bool: + """Whether the first set of ``names`` is true, defaulting to false.""" return _is_true(_first(values, *names, default="false")) framework, precision = values.get("FRAMEWORK", ""), values.get("PRECISION", "") @@ -120,6 +124,7 @@ def dp(*names: str) -> bool: def write(path: Path, document: Mapping[str, object]) -> None: + """Write ``document`` to ``path`` as indented JSON.""" path.write_text(json.dumps(document, indent=2) + "\n") diff --git a/inferencex-e2e/infx/bench/eval/stage.py b/inferencex-e2e/infx/bench/eval/stage.py index 9511298f62..a14d83f880 100644 --- a/inferencex-e2e/infx/bench/eval/stage.py +++ b/inferencex-e2e/infx/bench/eval/stage.py @@ -29,6 +29,7 @@ def copy(results_dir: Path, destination: Path, *, suffix: str = "") -> list[Path def _target(destination: Path, name: str, suffix: str) -> Path: + """Where ``name`` lands in ``destination``, with ``suffix`` before its extension and a clash counter.""" if not suffix: return destination / name extension = next(extension for extension in _EXTENSIONS if name.endswith(extension)) diff --git a/inferencex-e2e/infx/bench/eval/vendor.py b/inferencex-e2e/infx/bench/eval/vendor.py index d55ad8749f..f025333b3a 100644 --- a/inferencex-e2e/infx/bench/eval/vendor.py +++ b/inferencex-e2e/infx/bench/eval/vendor.py @@ -44,6 +44,7 @@ class StepError(Exception): """A setup step failed; ``returncode`` becomes the eval's exit code.""" def __init__(self, message: str, returncode: int) -> None: + """Keep ``message`` and the exit code ``returncode`` the eval should return.""" super().__init__(message) self.returncode = returncode @@ -112,6 +113,7 @@ def step( raise StepError(f"{self.spec.label} {stage} failed with exit code {rc}", rc) def adapter_argv(self, command: tuple[str, ...], *args: str) -> list[str]: + """The adapter command line with ``command`` and ``args``, plus model, output, and suite.""" suite = [self.provider.suite_flag, self.suite] if self.provider.suite_flag else [] return [ self.python, str(EVALS / self.spec.adapter), *command, *args, @@ -119,6 +121,7 @@ def adapter_argv(self, command: tuple[str, ...], *args: str) -> list[str]: ] # fmt: skip def published(self) -> bool: + """Whether every result glob of the suite matches a file in the results directory.""" results = self.ctx.results_dir return all(any(p.is_file() for p in results.glob(glob)) for glob in self.spec.results) @@ -192,6 +195,7 @@ def _provision(job: Job) -> str: def _provision_failed(job: Job, detail: str) -> StepError: + """Print ``detail`` and return the provisioning ``StepError`` with exit code 1.""" print(f"ERROR: {detail}", file=sys.stderr) return StepError(f"{job.spec.label} {_PROVISION} failed with exit code 1", 1) @@ -205,6 +209,7 @@ def _pip_target(job: Job, target: Path) -> list[str]: def _prepare_kimi(job: Job) -> tuple[list[str], dict[str, str]]: + """Install Kimi's runtime and verifier checkout, then return adapter args and ``PYTHONPATH``.""" runtime = job.scratch / "kimi-runtime" job.step("dependency installation", _pip_target(job, runtime)) checkout = job.scratch / "kimi-verifier" @@ -220,6 +225,7 @@ def _prepare_kimi(job: Job) -> tuple[list[str], dict[str, str]]: def _prepare_minimax(job: Job) -> tuple[list[str], dict[str, str]]: + """Fetch the pinned MiniMax source and dependencies, then return adapter args and no env.""" source = job.scratch / "minimax-source" dependencies = job.scratch / "minimax-deps" stage = "pinned runtime preparation" @@ -234,10 +240,12 @@ def _prepare_minimax(job: Job) -> tuple[list[str], dict[str, str]]: def _bfcl_project(job: Job) -> Path: + """The BFCL project directory inside the job's scratch space.""" return job.scratch / "bfcl-project" def _prepare_bfcl(job: Job) -> tuple[list[str], dict[str, str]]: + """Install the BFCL runtime, create its project directory, and return adapter args.""" job.step("dependency installation", [ job.python, str(EVALS / job.spec.adapter), "--install-runtime", str(job.scratch / "bfcl-wheel"), "--uv", uv.find(), diff --git a/inferencex-e2e/infx/bench/fixed_seq.py b/inferencex-e2e/infx/bench/fixed_seq.py index c856255c7c..43d1657780 100644 --- a/inferencex-e2e/infx/bench/fixed_seq.py +++ b/inferencex-e2e/infx/bench/fixed_seq.py @@ -32,7 +32,7 @@ class Point: def client_argv(point: Point) -> list[str]: - """An unthrottled burst that ignores EOS, after 2x-concurrency warmups.""" + """The client argv for ``point``, an unthrottled burst that ignores EOS after 2x-concurrency warmups.""" argv = [ PYTHON, "-m", "infx.bench_serving.benchmark_serving", "--model", point.model, @@ -62,6 +62,7 @@ def client_argv(point: Point) -> list[str]: def _run_client(point: Point) -> int: + """Run the benchmark client for ``point``, relaying signals, and return its status.""" with proc.RelaySignals() as relay: return relay.run(client_argv(point)) @@ -78,7 +79,7 @@ def served_model(base_url: str) -> str: def srt_single(args: argparse.Namespace) -> int: - """One single-node point; srt-slurm owns the server and power sampling.""" + """Benchmark one single-node point from the environment and return its status. srt-slurm owns the server and power.""" values = env.require( "MODEL", "CONC", "ISL", "OSL", "RANDOM_RANGE_RATIO", "RESULT_FILENAME", "RESULT_DIR", "SRT_FRONTEND_HOST", "SRT_FRONTEND_PORT", "RUN_EVAL", "EVAL_ONLY", "USE_CHAT_TEMPLATE", @@ -118,7 +119,7 @@ def srt_single(args: argparse.Namespace) -> int: def srt_sweep(args: argparse.Namespace) -> int: - """Every ``CONC_LIST`` point of an srt-slurm multi-node job; srt-slurm owns the servers.""" + """Benchmark every ``CONC_LIST`` point of an srt-slurm multi-node job and return the first failure, else 0.""" values = env.require( "ISL", "OSL", "RANDOM_RANGE_RATIO", "SRT_FRONTEND_HOST", "SRT_FRONTEND_PORT", "CONC_LIST", "PREFILL_NUM_WORKERS", "PREFILL_TP", "PREFILL_PP_SIZE", "PREFILL_PCP_SIZE", @@ -166,7 +167,7 @@ def srt_sweep(args: argparse.Namespace) -> int: def explicit_point(args: argparse.Namespace) -> int: - """One point described entirely by flags.""" + """Benchmark one point described entirely by ``args`` and return the client's status.""" return _run_client( Point( base_url=args.base_url, diff --git a/inferencex-e2e/infx/bench/proc.py b/inferencex-e2e/infx/bench/proc.py index e2605708d8..e5c20c7cdd 100644 --- a/inferencex-e2e/infx/bench/proc.py +++ b/inferencex-e2e/infx/bench/proc.py @@ -18,7 +18,6 @@ from typing_extensions import Self REPO_ROOT = Path(__file__).resolve().parents[2] -# TOKENS and TOKENIZER name token counts and tokenizer paths, never credentials. _SECRET_NAME = re.compile(r"TOKEN(?!S|IZER)|SECRET|API_?KEY|PASSWORD", re.IGNORECASE) @@ -28,7 +27,10 @@ def status(returncode: int) -> int: def echo(argv: Sequence[str | os.PathLike[str]], env: Mapping[str, str] | None = None) -> None: - """Print ``+ `` to stderr, masking the values of credential variables.""" + """Print ``+ `` to stderr, masking the values of credential variables. + + Names like ``TOKENS`` and ``TOKENIZER`` hold counts and paths, so they are not masked. + """ text = shlex.join(map(os.fspath, argv)) secrets = { value @@ -99,19 +101,23 @@ class _Signals: """Handle ``SIGNALS`` for the ``with`` block and remember the first one received.""" def __init__(self) -> None: + """Start with no signal received and no saved handlers.""" self.received: int | None = None self._previous: dict[int, Any] = {} def __enter__(self) -> Self: + """Install ``_handle`` for every signal in ``SIGNALS`` and return self.""" for signum in SIGNALS: self._previous[signum] = signal.signal(signum, self._handle) return self def __exit__(self, *_: object) -> None: + """Restore the handlers that were in place before the block.""" for signum, handler in self._previous.items(): signal.signal(signum, handler) def _handle(self, signum: int, _frame: FrameType | None) -> None: + """Record ``signum`` unless an earlier signal was already recorded.""" self.received = self.received or signum @@ -123,10 +129,12 @@ class RelaySignals(_Signals): """Run commands one at a time, relaying signals; after one, nothing new starts.""" def __init__(self) -> None: + """Start with no running child.""" super().__init__() self._child: subprocess.Popen[bytes] | None = None def _handle(self, signum: int, frame: FrameType | None) -> None: + """Record ``signum`` and forward it to the running child, if any.""" super()._handle(signum, frame) if self._child is not None: self._child.send_signal(signum) diff --git a/inferencex-e2e/infx/bench/server.py b/inferencex-e2e/infx/bench/server.py index 58d739d002..9d4e62ef07 100644 --- a/inferencex-e2e/infx/bench/server.py +++ b/inferencex-e2e/infx/bench/server.py @@ -45,7 +45,7 @@ def http_json(url: str) -> object | None: def process_alive(pid: int) -> bool: - """A zombie waiting to be reaped is not a live server.""" + """Whether ``pid`` is running, counting a zombie waiting to be reaped as dead.""" try: os.kill(pid, 0) except ProcessLookupError: @@ -68,6 +68,7 @@ def wait_ready( decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") def stream() -> None: + """Copy new bytes of ``log`` to stdout, restarting when the file is replaced or truncated.""" nonlocal offset, inode if log is None or not log.exists(): return @@ -88,7 +89,6 @@ def stream() -> None: if status is not None and status < 400: return if pid is not None and not process_alive(pid): - # Its last words may have landed during the probe. stream() raise NotReadyError(f"process {pid} died before {url} became ready") time.sleep(poll_s) diff --git a/inferencex-e2e/infx/bench_serving/backend_request_func.py b/inferencex-e2e/infx/bench_serving/backend_request_func.py index 5b74c317a9..c038b09bc2 100644 --- a/inferencex-e2e/infx/bench_serving/backend_request_func.py +++ b/inferencex-e2e/infx/bench_serving/backend_request_func.py @@ -33,13 +33,15 @@ class RequestFuncInput: @dataclass class RequestFuncOutput: + """One request's result. ``ttft`` is time to first token, ``itl`` the inter-token gaps.""" + generated_text: str = "" success: bool = False latency: float = 0.0 output_tokens: int = 0 - ttft: float = 0.0 # Time to first token - itl: list[float] = field(default_factory=list) # List of inter-token latencies - tpot: float = 0.0 # avg next-token latencies + ttft: float = 0.0 + itl: list[float] = field(default_factory=list) + tpot: float = 0.0 prompt_len: int = 0 error: str = "" @@ -48,6 +50,10 @@ async def async_request_tgi( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one TGI ``generate_stream`` request and return its timings and text. + + TGI rejects temperature 0.0, top_p 1.0, and ``ignore_eos``, so near values are sent and EOS is honored. + """ api_url = request_func_input.api_url assert api_url.endswith("generate_stream") # noqa: S101 @@ -56,10 +62,9 @@ async def async_request_tgi( "best_of": request_func_input.best_of, "max_new_tokens": request_func_input.output_len, "do_sample": True, - "temperature": 0.01, # TGI does not accept 0.0 temperature. - "top_p": 0.99, # TGI does not accept 1.0 top_p. + "temperature": 0.01, + "top_p": 0.99, "truncate": request_func_input.prompt_len, - # TGI does not accept ignore_eos flag. } payload = { "inputs": request_func_input.prompt, @@ -80,19 +85,16 @@ async def async_request_tgi( continue chunk_bytes = chunk_bytes.decode("utf-8") - # TGI pings may contain no data. if chunk_bytes.startswith(":"): continue chunk = chunk_bytes.removeprefix("data:") data = json.loads(chunk) timestamp = time.perf_counter() - # First token if ttft == 0.0: ttft = time.perf_counter() - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -118,6 +120,7 @@ async def async_request_trt_llm( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one TensorRT-LLM ``generate_stream`` request and return its timings and text.""" api_url = request_func_input.api_url assert api_url.endswith("generate_stream") # noqa: S101 @@ -152,12 +155,10 @@ async def async_request_trt_llm( data = json.loads(chunk) output.generated_text += data["text_output"] timestamp = time.perf_counter() - # First token if ttft == 0.0: ttft = timestamp - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -183,21 +184,22 @@ async def async_request_deepspeed_mii( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Send one non-streaming DeepSpeed-MII request and return its latency and text. + + MII rejects temperature 0.0 and cannot stream, so ``ttft`` stays 0 (see DeepSpeed-MII PR 311). + """ async with aiohttp.ClientSession(trust_env=True, timeout=AIOHTTP_TIMEOUT) as session: assert request_func_input.best_of == 1 # noqa: S101 payload = { "prompt": request_func_input.prompt, "max_tokens": request_func_input.output_len, - "temperature": 0.01, # deepspeed-mii does not accept 0.0 temp. + "temperature": 0.01, "top_p": 1.0, } output = RequestFuncOutput() output.prompt_len = request_func_input.prompt_len - # NOTE: DeepSpeed-MII doesn't support streaming as of Jan 28 2024, - # will use 0 as placeholder. - # See https://github.com/microsoft/DeepSpeed-MII/pull/311 output.ttft = 0 st = time.perf_counter() @@ -225,6 +227,10 @@ async def async_request_openai_completions( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one OpenAI completions request and return its timings, text, and token count. + + Skips SSE comment and event lines, and treats a choice-less chunk as the final usage report. + """ api_url = request_func_input.api_url assert api_url.endswith(("completions", "profile")), ( # noqa: S101 "OpenAI Completions API URL must end with 'completions' or 'profile'." @@ -265,25 +271,20 @@ async def async_request_openai_completions( continue chunk = chunk_bytes.decode("utf-8") - # SSE comments (keep-alive pings) and event-type lines carry no JSON. if chunk.startswith((":", "event:")): continue chunk = chunk.removeprefix("data: ") if chunk != "[DONE]": data = json.loads(chunk) - # The final usage-only chunk may contain no token. if choices := data.get("choices"): - # Special tokens may have empty text. text = choices[0].get("text") timestamp = time.perf_counter() - # First token if not first_chunk_received: first_chunk_received = True ttft = time.perf_counter() - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -323,6 +324,7 @@ async def async_request_openai_chat_completions( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one OpenAI chat completions request and return its timings, text, and token count.""" api_url = request_func_input.api_url assert api_url.endswith("chat/completions"), ( # noqa: S101 "OpenAI Chat Completions API URL must end with 'chat/completions'." @@ -370,7 +372,6 @@ async def async_request_openai_chat_completions( continue chunk = chunk_bytes.decode("utf-8") - # SSE comments (keep-alive pings) and event-type lines carry no JSON. if chunk.startswith((":", "event:")): continue chunk = chunk.removeprefix("data: ") @@ -380,12 +381,10 @@ async def async_request_openai_chat_completions( if choices := data.get("choices"): content = choices[0]["delta"].get("content") - # First token if ttft == 0.0: ttft = timestamp - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -420,6 +419,7 @@ async def async_request_openai_chat_completions( def get_model(pretrained_model_name_or_path: str) -> str: + """A ModelScope snapshot path when ``VLLM_USE_MODELSCOPE`` is true, else the name unchanged.""" if os.getenv("VLLM_USE_MODELSCOPE", "False").lower() == "true": from modelscope import snapshot_download @@ -521,6 +521,10 @@ def get_tokenizer( trust_remote_code: bool = False, **kwargs: Any, ) -> PreTrainedTokenizer | PreTrainedTokenizerFast: + """Load the tokenizer for a model path or hub id in ``tokenizer_mode`` with sglang's v5 fixes applied. + + Raises ``ValueError`` for slow mode with ``use_fast`` and ``ImportError`` for mistral mode without vllm. + """ if pretrained_model_name_or_path is not None and not os.path.exists( pretrained_model_name_or_path ): diff --git a/inferencex-e2e/infx/bench_serving/benchmark_outcome.py b/inferencex-e2e/infx/bench_serving/benchmark_outcome.py index a1b4ebfbfb..cfa9272708 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_outcome.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_outcome.py @@ -16,7 +16,10 @@ class BenchmarkOutcome(TypedDict): def benchmark_outcome(requested: int, completed: int) -> BenchmarkOutcome: - """Record the same five-percent gate before cleanup or artifact upload.""" + """Apply the five-percent failure gate to ``requested`` and ``completed`` and return the outcome. + + Raises ``ValueError`` unless both are ints with ``0 <= completed <= requested`` and ``requested > 0``. + """ if ( type(requested) is not int or type(completed) is not int diff --git a/inferencex-e2e/infx/bench_serving/benchmark_serving.py b/inferencex-e2e/infx/bench_serving/benchmark_serving.py index fec4874a9d..55ac1b8bc8 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_serving.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_serving.py @@ -49,9 +49,6 @@ try: from .backend_request_func import get_tokenizer except ImportError: - # vLLM moved get_tokenizer from vllm.transformers_utils.tokenizer to - # vllm.tokenizers; the old path was a deprecation alias in v0.20 and - # was removed by v0.23. Try the new location first, fall back to old. try: from vllm.tokenizers import get_tokenizer except ImportError: @@ -71,6 +68,8 @@ @dataclass class BenchmarkMetrics: + """Aggregate results of one run. ``e2el`` is client latency from request send to full response.""" + completed: int total_input: int total_output: int @@ -90,14 +89,12 @@ class BenchmarkMetrics: median_itl_ms: float std_itl_ms: float percentiles_itl_ms: list[tuple[float, float]] - # Client latency from request send to complete response. mean_e2el_ms: float median_e2el_ms: float std_e2el_ms: float percentiles_e2el_ms: list[tuple[float, float]] -# --- Multiprocessing helpers for sample_random_requests --- _worker_tokenizer = None @@ -110,9 +107,9 @@ def _load_tokenizer( LlamaTokenizer.all_special_tokens_extended (e.g. Qwen3.5 with newer transformers). Prefer backend_request_func.get_tokenizer on fallback so client tokenization stays aligned with the sglang server (#1381, #1428). + ``deepseek_v4`` always uses vLLM's loader, since HF AutoTokenizer may not know it. """ if tokenizer_mode == "deepseek_v4": - # HF AutoTokenizer may not recognize deepseek_v4; use vLLM's loader. try: from vllm.tokenizers import get_tokenizer as _vllm_get_tokenizer except ImportError: @@ -183,7 +180,7 @@ def _apply_chat_template(prompt: str, tokenizer: PreTrainedTokenizerBase, dsv4: def _process_prompt_chunk(chunk_args: tuple) -> list[tuple[str, int, int, None, int]]: - """Generate a chunk of random prompts in a worker process.""" + """Generate one chunk of random prompts in a worker, as ``(prompt, len, output_len, None, mismatch)`` tuples.""" ( indices, prefix_token_ids, @@ -244,6 +241,11 @@ def sample_random_requests( trust_remote_code: bool = False, num_workers: int = 0, ) -> list[tuple[str, int, int]]: + """Build ``num_prompts`` random-token prompts, each length between ``range_ratio`` times its target and the target. + + Returns ``(prompt, prompt_len, output_len, None)`` tuples. Raises ``ValueError`` for no prompts or + ``dsv4`` without a chat template. Padding uses a local RNG so arrival times ignore ``num_workers``. + """ if num_prompts <= 0: raise ValueError(f"--num-prompts must be positive, got {num_prompts}") vocab_size = tokenizer.vocab_size @@ -261,6 +263,7 @@ def sample_random_requests( input_len = input_len - chat_template_len def sample_uniform(seq_len: int) -> list[int]: + """Draw ``num_prompts`` lengths uniformly from ``range_ratio * seq_len`` to ``seq_len``.""" lower = int(seq_len * range_ratio) upper = seq_len return np.random.randint(lower, upper + 1, size=num_prompts).tolist() # noqa: NPY002 @@ -269,11 +272,10 @@ def sample_uniform(seq_len: int) -> list[int]: output_lens = sample_uniform(output_len) offsets = np.random.randint(0, vocab_size, size=num_prompts).tolist() # noqa: NPY002 - # Local padding draws keep downstream arrival times independent of num_workers. local_rng = np.random.RandomState( np.random.get_state()[1][ # noqa: NPY002 :4 - ].tolist() # derive seed from current state without advancing it + ].tolist() ) if num_workers <= 0: @@ -410,6 +412,11 @@ def calculate_metrics( selected_percentiles: list[float], goodput_config_dict: dict[str, float], ) -> tuple[BenchmarkMetrics, list[int]]: + """Summarize ``outputs`` over ``dur_s`` seconds into metrics, plus each request's output length. + + Retokenizes text when the server sent no usage count, which may overcount slightly. Goodput counts + a request whose TTFT, TPOT (0 for one token), and E2EL all meet ``goodput_config_dict``. + """ actual_output_lens: list[int] = [] total_input = 0 completed = 0 @@ -424,7 +431,6 @@ def calculate_metrics( output_len = outputs[i].output_tokens if not output_len: - # Chunks can bundle tokens; retokenization may slightly overcount. output_len = len( tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids ) @@ -435,7 +441,6 @@ def calculate_metrics( latency_minus_ttft = outputs[i].latency - outputs[i].ttft tpot = latency_minus_ttft / (output_len - 1) tpots.append(tpot) - # Note: if output_len <= 1, we regard tpot as 0 for goodput all_tpots.append(tpot) itls += outputs[i].itl ttfts.append(outputs[i].ttft) @@ -469,7 +474,6 @@ def calculate_metrics( "on the benchmark arguments.", stacklevel=2, ) - # Convert once; every numpy call below would otherwise rebuild an array from the list. ttfts, tpots, itls, e2els = (np.asarray(v or 0) for v in (ttfts, tpots, itls, e2els)) metrics = BenchmarkMetrics( completed=completed, @@ -479,8 +483,7 @@ def calculate_metrics( request_goodput=good_completed / dur_s, output_throughput=sum(actual_output_lens) / dur_s, total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s, - mean_ttft_ms=np.mean(ttfts) - * 1000, # ttfts is empty if streaming is not supported by backend + mean_ttft_ms=np.mean(ttfts) * 1000, std_ttft_ms=np.std(ttfts) * 1000, median_ttft_ms=np.median(ttfts) * 1000, percentiles_ttft_ms=[(p, np.percentile(ttfts, p) * 1000) for p in selected_percentiles], @@ -523,6 +526,10 @@ async def benchmark( max_concurrency: int | None, lora_modules: list[str] | None, ) -> dict[str, Any]: + """Warm up, then send ``input_requests`` to ``api_url`` at ``request_rate`` and return the result dict. + + Raises ``ValueError`` for an unknown backend or multi-modal input outside ``openai-chat``. + """ if backend in ASYNC_REQUEST_FUNCS: request_func = ASYNC_REQUEST_FUNCS[backend] else: @@ -531,7 +538,6 @@ async def benchmark( print("Starting initial single prompt test run...") test_prompt, test_prompt_len, test_output_len, test_mm_content = input_requests[0] if backend != "openai-chat" and test_mm_content is not None: - # multi-modal benchmark is only available on OpenAI Chat backend. raise ValueError("Multi-modal content is only supported on 'openai-chat' backend.") test_input = RequestFuncInput( model=model_id, @@ -552,6 +558,7 @@ async def benchmark( warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None async def warmup_limited_req_fn() -> RequestFuncOutput: + """Send the test request, under the warmup concurrency limit if one is set.""" if warmup_semaphore is None: return await request_func(request_func_input=test_input, pbar=warmup_pbar) async with warmup_semaphore: @@ -608,6 +615,7 @@ async def warmup_limited_req_fn() -> RequestFuncOutput: async def limited_request_func( request_func_input: RequestFuncInput, pbar: tqdm | None ) -> RequestFuncOutput: + """Send ``request_func_input``, under the concurrency limit if one is set.""" if semaphore is None: return await request_func(request_func_input=request_func_input, pbar=pbar) async with semaphore: @@ -712,6 +720,7 @@ def process_one_metric( metric_name: str, metric_header: str, ) -> None: + """Print the selected metric's mean, median, and percentiles and copy them into ``result``.""" if metric_attribute_name not in selected_percentile_metrics: return print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-")) @@ -752,6 +761,7 @@ def process_one_metric( def check_goodput_args(args: argparse.Namespace) -> dict[str, float]: + """Parse ``args.goodput`` into SLOs in ms, raising ``ValueError`` for unknown names or negative values.""" goodput_config_dict = {} valid_names = ["ttft", "tpot", "e2el"] if args.goodput: @@ -773,6 +783,7 @@ def check_goodput_args(args: argparse.Namespace) -> dict[str, float]: def parse_goodput(slo_pairs: list[str]) -> dict[str, float]: + """Parse ``KEY:VALUE`` strings into a dict, raising ``ArgumentTypeError`` on bad format.""" goodput_config_dict = {} try: for slo_pair in slo_pairs: @@ -791,6 +802,7 @@ def parse_goodput(slo_pairs: list[str]) -> dict[str, float]: def save_to_pytorch_benchmark_format( args: argparse.Namespace, results: dict[str, Any], file_name: str ) -> None: + """Write the latency metrics of ``results`` as PyTorch benchmark records beside ``file_name``.""" metrics = [ "median_ttft_ms", "mean_ttft_ms", @@ -820,6 +832,10 @@ def save_to_pytorch_benchmark_format( def main(args: argparse.Namespace) -> None: + """Run the benchmark from parsed ``args``, save results if asked, and exit nonzero when the run fails. + + Records the request outcome before saving so cleanup cannot turn a failed run into a pass. + """ print(args) if args.request_rate <= 0: raise ValueError(f"--request-rate must be positive, got {args.request_rate}") @@ -868,7 +884,6 @@ def main(args: argparse.Namespace) -> None: goodput_config_dict = check_goodput_args(args) - # Avoid GC processing "static" data - reduce pause times. gc.collect() gc.freeze() @@ -897,12 +912,9 @@ def main(args: argparse.Namespace) -> None: ) ) - # Preserve the request gate before writing results: cleanup or a surrounding - # launcher must not turn an error-containing benchmark into a passed point. try: outcome = benchmark_outcome(args.num_prompts, benchmark_result["completed"]) except ValueError as exc: - # Preserve invalid counts for diagnostics; the processor rejects them. outcome = { "status": "failed", "requested": args.num_prompts, @@ -939,7 +951,6 @@ def main(args: argparse.Namespace) -> None: result_json = {**result_json, **benchmark_result} if not args.save_detailed: - # Remove fields with too many data points for field in [ "ttfts", "itls", @@ -988,7 +999,6 @@ def main(args: argparse.Namespace) -> None: default=None, help="Server or API base url if not using http host and port.", ) - # Use 127.0.0.1 here instead of localhost to force the use of ipv4 parser.add_argument("--host", type=str, default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) parser.add_argument( diff --git a/inferencex-e2e/infx/bench_serving/benchmark_utils.py b/inferencex-e2e/infx/bench_serving/benchmark_utils.py index db746acafc..c4180e3f84 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_utils.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_utils.py @@ -8,9 +8,9 @@ def convert_to_pytorch_benchmark_format( args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any] ) -> list: - """ - Save the benchmark results in the format used by PyTorch OSS benchmark with - on metric per record + """Build one PyTorch OSS benchmark record per entry of ``metrics``, tagged with ``args`` and ``extra_info``. + + Returns an empty list unless ``SAVE_TO_PYTORCH_BENCHMARK_FORMAT`` is set. Format reference is at https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database """ records = [] diff --git a/inferencex-e2e/infx/bench_serving/encoding_dsv4.py b/inferencex-e2e/infx/bench_serving/encoding_dsv4.py index a83c9d504b..1d7bac4ddb 100644 --- a/inferencex-e2e/infx/bench_serving/encoding_dsv4.py +++ b/inferencex-e2e/infx/bench_serving/encoding_dsv4.py @@ -10,8 +10,6 @@ import re from typing import Any -# Special Tokens - bos_token: str = "<|begin▁of▁sentence|>" # noqa: RUF001, S105 eos_token: str = "<|end▁of▁sentence|>" # noqa: RUF001, S105 thinking_start_token: str = "" # noqa: S105 @@ -22,7 +20,6 @@ ASSISTANT_SP_TOKEN = "<|Assistant|>" # noqa: RUF001, S105 LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" # noqa: RUF001, S105 -# Task special tokens for internal classification tasks DS_TASK_SP_TOKENS = { "action": "<|action|>", # noqa: RUF001 "query": "<|query|>", # noqa: RUF001 @@ -33,8 +30,6 @@ } VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) -# Templates - system_msg_template: str = "{content}" user_msg_template: str = "{content}" latest_reminder_msg_template: str = "{content}" @@ -84,8 +79,6 @@ You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. """ -# Utility Functions - def to_json(value: Any) -> str: """Serialize a value to JSON string.""" @@ -159,6 +152,7 @@ def decode_dsml_to_arguments( """Convert param_name -> (value, is_string) pairs to OpenAI tool-call arguments.""" def _decode_value(key: str, value: str, string: str) -> str: + """Render one ``"key": value`` JSON member, quoting ``value`` when ``string`` is "true".""" if string == "true": value = to_json(value) return f"{to_json(key)}: {value}" @@ -193,9 +187,6 @@ def find_last_user_index(messages: list[dict[str, Any]]) -> int: return last_user_index -# Message Rendering - - def render_message( index: int, messages: list[dict[str, Any]], @@ -206,6 +197,8 @@ def render_message( """ Render a single message at the given index into its encoded string form. + An assistant message that follows a task message is a task output and renders without thinking. + Args: index: Index of the message to render. messages: Full list of messages in the conversation. @@ -323,7 +316,6 @@ def render_message( summary_content = content or "" rc = reasoning_content or "" - # Check if previous message has a task - if so, this is a task output (no thinking) prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None if thinking_mode == "thinking" and not prev_has_task: @@ -379,9 +371,6 @@ def render_message( return prompt -# Preprocessing - - def merge_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: """Merge OpenAI tool messages into user-message content_blocks. @@ -423,7 +412,6 @@ def merge_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: "content": msg.get("content", ""), "content_blocks": [text_block], } - # Preserve extra fields (task, wo_eos, mask, etc.) for key in ("task", "wo_eos", "mask"): if key in msg: new_msg[key] = msg[key] @@ -469,9 +457,6 @@ def sort_tool_results_by_call_order( return messages -# Main Encoding Function - - def encode_messages( messages: list[dict[str, Any]], thinking_mode: str, @@ -513,8 +498,6 @@ def encode_messages( if thinking_mode == "thinking" and effective_drop_thinking: full_messages = _drop_thinking_messages(full_messages) - # After dropping, recalculate how many messages to render - # (context may have shrunk too) num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) context_len = len(full_messages) - num_to_render else: @@ -555,14 +538,10 @@ def _drop_thinking_messages(messages: list[dict[str, Any]]) -> list[dict[str, An msg = copy.copy(msg) msg.pop("reasoning_content", None) result.append(msg) - # developer and other roles before last_user_idx are dropped return result -# Parsing (Decoding model output) - - def _read_until_stop(index: int, text: str, stop: list[str]) -> tuple[int, str, str | None]: """ Read text from index until one of the stop strings is found. From 378f46243bb8bb897cca5d51852db6fb8b9cfac6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:35:09 -0500 Subject: [PATCH 198/203] docs(klaud): replace comments with function docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give every function and method in infx/klaud a short docstring naming its inputs, result and notable errors. Delete the # comments, folding the ones that carried needed context into the nearest docstring. Code is unchanged. 中文:为 infx/klaud 中的每个函数和方法补充简短的文档字符串,说明输入、 返回值和关键异常。删除 # 注释,必要的上下文并入最近的文档字符串。代码未变。 --- inferencex-e2e/infx/klaud/__main__.py | 55 ++++++++--- inferencex-e2e/infx/klaud/api.py | 20 +++- inferencex-e2e/infx/klaud/claims.py | 18 ++++ inferencex-e2e/infx/klaud/github.py | 18 +++- inferencex-e2e/infx/klaud/lifecycle.py | 125 ++++++++++++++++++------ inferencex-e2e/infx/klaud/models.py | 18 +++- inferencex-e2e/infx/klaud/reporting.py | 104 ++++++++++++++++++-- inferencex-e2e/infx/klaud/validation.py | 34 +++++-- 8 files changed, 326 insertions(+), 66 deletions(-) diff --git a/inferencex-e2e/infx/klaud/__main__.py b/inferencex-e2e/infx/klaud/__main__.py index 362913c34e..f528e8d073 100644 --- a/inferencex-e2e/infx/klaud/__main__.py +++ b/inferencex-e2e/infx/klaud/__main__.py @@ -43,11 +43,11 @@ from infx.clusters import RunnerInventory PRODUCERS = "producers.json" -# An abort is lifted by removing its label, so it never delays the next attempt by itself. RETRY_NEXT_WAVE = frozenset({"capacity-deferred", "agent-not-started", "aborted"}) def observation_key(row: dict) -> tuple: + """Return the public identity tuple of feed ``row``, with its image normalized.""" image = row.get("image") if isinstance(image, str): image = normalized_image(image) @@ -69,6 +69,7 @@ def observation_key(row: dict) -> tuple: def target_clusters(entries: list[dict], inventory: RunnerInventory) -> frozenset[str]: + """Return the cluster IDs serving the runners of ``entries``, or none if one is unknown.""" try: return frozenset( inventory.cluster_for(node).id @@ -201,16 +202,21 @@ def choose( def weights_key(source: dict) -> tuple: + """Return the (model, precision, hardware) key that identifies the weights ``source`` needs.""" return (source.get("model"), source.get("precision"), source.get("hardware")) def recent_candidate_outcomes( repository: str, cooldown_hours: int ) -> tuple[dict[str, str | None], set[tuple]]: + """Return each candidate's latest outcome within ``cooldown_hours`` and missing-weights keys. + + Resumed sessions record outcomes in later waves, so history reaches MAX_SESSIONS waves back. + The newest session artifact counts. Unreadable history means no cooldown. + """ from .lifecycle import MAX_SESSIONS, WAVE_HOURS, resume_parent cutoff = datetime.now(UTC) - timedelta(hours=cooldown_hours) - # Resumed sessions of an older planner run record their outcomes in later waves. since = cutoff - timedelta(hours=MAX_SESSIONS * WAVE_HOURS) try: runs = github_items( @@ -228,7 +234,6 @@ def recent_candidate_outcomes( created = utc(run["created_at"]) if created >= since: dated.append((created, int(run["id"]))) - # A resume run's candidates come from its planner run's plan. plans[int(run["id"])] = resume_parent(run.get("display_title") or "") or int( run["id"] ) @@ -254,7 +259,6 @@ def recent_candidate_outcomes( or utc(artifact["created_at"]) < cutoff ): continue - # Each session of a candidate uploads its own artifact; the newest one counts. if match[1] not in latest or artifact["id"] > latest[match[1]][0]["id"]: latest[match[1]] = (artifact, run_id) except (AttributeError, KeyError, TypeError, ValueError): @@ -283,6 +287,7 @@ def recent_candidate_outcomes( def planned_candidates(repository: str, artifacts: list[dict]) -> list[dict]: + """Return the candidates in the klaud-plan artifact among ``artifacts``, or [] if unreadable.""" plan = next((a for a in artifacts if a.get("name") == "klaud-plan"), None) if not plan: return [] @@ -298,6 +303,10 @@ def planned_candidates(repository: str, artifacts: list[dict]) -> list[dict]: def recorded_outcome(repository: str, artifact: dict) -> str | None: + """Return the outcome a diagnostics ``artifact`` recorded, or None if it is unreadable. + + A skipped agent yields ``agent-not-started`` and a missing-weights failure ``weights-missing``. + """ with tempfile.TemporaryDirectory() as scratch: try: download_json(repository, artifact, Path(scratch)) @@ -319,6 +328,10 @@ def recorded_outcome(repository: str, artifact: dict) -> str | None: def order_by_cooldown(candidates: list[dict], recent: dict[str, str | None]) -> list[dict]: + """Return ``candidates`` never tried recently, then those whose recent outcome retries now. + + An abort is lifted by removing its label, so it never delays the next attempt by itself. + """ fresh = [candidate for candidate in candidates if candidate["id"] not in recent] deferred = [ candidate for candidate in candidates if recent.get(candidate["id"], "") in RETRY_NEXT_WAVE @@ -380,6 +393,7 @@ def abort_released(repository: str, candidate_id: str) -> bool: def active_pulls(pulls: list[dict], now: datetime, stale_days: int) -> list[dict]: + """Return ``pulls`` on Klaud branches or updated within ``stale_days`` before ``now``.""" cutoff = now - timedelta(days=stale_days) return [ pr @@ -392,6 +406,11 @@ def active_pulls(pulls: list[dict], now: datetime, stale_days: int) -> list[dict def plan( root: Path, directory: Path, review_batch_size: int, cooldown_hours: int, stale_pr_days: int ) -> None: + """Write ``candidates.json`` and ``open-prs.json`` under ``directory`` for overlap review. + + Unclaimed, unaborted feed candidates at ``root``'s HEAD are cooled down by + ``cooldown_hours`` and capped at ``review_batch_size``. PRs idle ``stale_pr_days`` drop out. + """ from . import claims policy = Policy() @@ -422,12 +441,10 @@ def plan( if recovery_file.exists(): blocked.update(json.loads(recovery_file.read_text())) candidates = [candidate for candidate in candidates if candidate["family"] not in blocked] - # klaud-abort keeps a candidate out of the pool until someone removes the label. aborted = aborted_candidates(repository) candidates = [candidate for candidate in candidates if candidate["id"] not in aborted] recent, missing_weights = recent_candidate_outcomes(repository, cooldown_hours) for candidate in candidates: - # Only candidates the cooldown would hold back need their abort history. held = candidate["id"] in recent and recent[candidate["id"]] not in RETRY_NEXT_WAVE if held and abort_released(repository, candidate["id"]): recent[candidate["id"]] = "aborted" @@ -501,7 +518,10 @@ def plan( def denied_bash_category(denial: dict) -> str: - """Retain only fixed categories, never command text, arguments or paths.""" + """Return a fixed category for the Bash command in permission ``denial``. + + Command text, arguments and paths are never retained. + """ command = denial.get("tool_input", {}) command = command.get("command") if isinstance(command, dict) else None if not isinstance(command, str): @@ -539,6 +559,7 @@ def denied_bash_category(denial: dict) -> str: def execution_diagnostics(path: Path) -> dict: + """Return numeric metrics, termination and denial categories from the log at ``path``.""" try: messages = json.loads(path.read_text()) if not isinstance(messages, list): @@ -611,13 +632,15 @@ def execution_diagnostics(path: Path) -> dict: def check_stop() -> dict: - """Require a verified terminal outcome, not merely an agent's final response.""" + """Return the Stop hook decision, blocking unless a verified terminal outcome exists. + + An aborted candidate never blocks, since the recover-current step cleans it up. + """ from .lifecycle import current_session try: session = current_session() pulls = session.pulls() - # The job's recover-current step cancels an aborted candidate's runs and closes its PR. if pulls and session.aborted(pulls[0]): return {} outcome = CandidateOutcome.model_validate_json( @@ -663,7 +686,6 @@ def resume_check() -> dict[str, bool]: if run is None or int(run) not in same: return no jobs = [job for job in candidate_jobs(session) if started_session(job)] - # This job's own session is already one of the started ones. if len(jobs) > MAX_SESSIONS or any( job["run_id"] in same and job["run_id"] != int(run) for job in jobs ): @@ -679,6 +701,10 @@ def resume_check() -> dict[str, bool]: def save_diagnostics( execution_file: Path, outcome_file: Path, action_outcome: str, output: Path ) -> bool: + """Write sanitized session diagnostics to ``output`` and return False if unverified. + + The outcome comes from the posted receipt, the evidence outcome, then ``outcome_file``. + """ diagnostics = { "action-outcome": action_outcome, **execution_diagnostics(execution_file), @@ -847,6 +873,11 @@ def regenerate_producers(directory: Path) -> None: def select( directory: Path, max_candidates: int, execution_file: Path | None = None, resumed: int = 0 ) -> None: + """Claim reviewed candidates that fit capacity and have a verified baseline. + + Takes up to ``max_candidates`` less the ``resumed`` sessions sharing the cap, and writes + their context, ownership.json and selection.json under ``directory``. + """ from . import claims, validation from .reporting import BaselinePreflight, Prose, missing_baseline_points, resolve_baseline @@ -855,7 +886,6 @@ def select( review, deferred = reviewed(contexts) decisions = {decision.candidate_id: decision for decision in review.decisions} selected = [] - # Sessions that recovery resumed in this wave run beside its new candidates and share the cap. slots = max(0, max_candidates - resumed) headroom: dict[str, int] = {} if slots and any(decision.decision == "proceed" for decision in review.decisions): @@ -1025,6 +1055,7 @@ def select( def main() -> int: + """Run the Klaud subcommand named on the command line and return its exit status.""" parser = argparse.ArgumentParser(prog="python -m infx.klaud", description=__doc__) parser.add_argument( "--root", @@ -1169,7 +1200,6 @@ def main() -> int: return 0 session = current_session() pulls = session.pulls() - # A resumed session continues with the baseline its PR already froze. record = baseline_for(session, pulls[0]) if pulls else None if record is None: context = json.loads( @@ -1318,7 +1348,6 @@ def main() -> int: if isinstance(error, VerificationError): reason = str(error) elif isinstance(error, ValidationError) and args.command == "report": - # Name the failing fields of the agent's own record without echoing its values. reason = "Invalid record: " + "; ".join( ".".join(str(part) for part in item["loc"]) + ": " + item["msg"] if item["loc"] diff --git a/inferencex-e2e/infx/klaud/api.py b/inferencex-e2e/infx/klaud/api.py index 707f6d18bf..2fae9aa9d5 100644 --- a/inferencex-e2e/infx/klaud/api.py +++ b/inferencex-e2e/infx/klaud/api.py @@ -53,14 +53,17 @@ def redirect_request( headers: Any, # noqa: ARG002 newurl: str, # noqa: ARG002 ) -> None: - return None + """Refuse every redirect by returning None.""" + return None # noqa: RET501 def reject_nonfinite(value: str) -> Never: # noqa: ARG001 + """Raise ReadError for a NaN or Infinity constant ``value`` in JSON.""" raise ReadError("invalid-json-number") def finite_float(value: str) -> float: + """Return JSON number text ``value`` as a float, raising ReadError unless finite.""" result = float(value) if not math.isfinite(result): raise ReadError("invalid-json-number") @@ -78,6 +81,11 @@ def fetch( opener: Callable[..., Any] | None = None, sleeper: Callable[[float], None] = time.sleep, ) -> Feed: + """GET one ``resource`` from ENDPOINTS and return it as a Feed. + + ``model``, ``date``, ``benchmark`` and ``token`` fill the query or auth header where the + resource takes them. Retries transient failures, then raises ReadError with a fixed code. + """ query = {} if resource == "benchmark-siblings": if benchmark is None: @@ -177,6 +185,7 @@ def fetch( def feed_issues(feed: Feed | None, now: datetime, policy: Policy) -> list[str]: + """Return issue codes for a missing, failed or stale ``feed`` at time ``now``.""" if feed is None: return ["feed-unavailable"] issues = ["feed-refresh-failed"] if feed.error else [] @@ -187,6 +196,7 @@ def feed_issues(feed: Feed | None, now: datetime, policy: Policy) -> list[str]: def catalog(images: Feed | None, now: datetime, policy: Policy) -> tuple[list[dict], list[str]]: + """Return one catalog item per public image row plus the sorted feed issue codes.""" issues = [f"images:{issue}" for issue in feed_issues(images, now, policy)] if images is None or not isinstance(images.payload, list): return [], [*issues, "images:invalid-or-missing-payload"] @@ -221,6 +231,7 @@ def catalog(images: Feed | None, now: datetime, policy: Policy) -> tuple[list[di def fetch_catalog(policy: Policy) -> tuple[list[dict], list[str]]: + """Fetch the public image feed and return its catalog and issues, or raise ReadError.""" try: images = fetch("images") except ReadError as error: @@ -229,6 +240,7 @@ def fetch_catalog(policy: Policy) -> tuple[list[dict], list[str]]: def fresh(value: Any, now: datetime, policy: Policy) -> bool: + """Return True if timestamp ``value`` is within the response age limit at ``now``.""" try: return ( -policy.clock_skew_seconds @@ -243,7 +255,8 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int """Return the idle nodes of each cluster that waiting Klaud jobs have not already requested. Clusters a waiting human job could run on, and clusters without headroom, are omitted. - Malformed or duplicate scheduler status yields no clusters. + A queued job without a node count needs one node, and jobs already holding a runner are + counted in allocatedNodes. Malformed or duplicate scheduler status yields no clusters. """ try: raw = feed.payload @@ -262,7 +275,6 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int human: set[str] = set() klaud: Counter[str] = Counter() for job in snapshot["queue"]: - # A job without a nodes label runs on one runner. nodes = 1 if job["nodeCount"] is None else job["nodeCount"] labels, runner = job["clusterIds"], job["runnerName"] if ( @@ -273,7 +285,6 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int or not isinstance(runner, str | None) ): return {} - # A job holding a runner is already counted in allocatedNodes. if runner is not None: continue if KLAUD_JOB.search(job["name"]): @@ -300,6 +311,7 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int def fetch_capacity(policy: Policy) -> dict[str, int]: + """Return per-cluster idle headroom from the private scheduler status feed.""" return cluster_headroom( fetch("scheduler", token=os.environ.get("KLAUD_DASHBOARD_API_KEY")), policy, diff --git a/inferencex-e2e/infx/klaud/claims.py b/inferencex-e2e/infx/klaud/claims.py index e1be6002eb..ca1b8dd4b9 100644 --- a/inferencex-e2e/infx/klaud/claims.py +++ b/inferencex-e2e/infx/klaud/claims.py @@ -18,6 +18,7 @@ def ref(repository: str, name: str) -> dict | None: + """Return the ref record for branch ``name`` in ``repository``, or None if absent.""" return next( ( row @@ -29,6 +30,10 @@ def ref(repository: str, name: str) -> dict | None: def record(repository: str, value: dict) -> dict: + """Return the JSON owner stored in the commit message of ref ``value``. + + Raises VerificationError when the message is not readable JSON. + """ commit = github.read(repository, "git/commits/" + value["object"]["sha"]) try: return json.loads(commit["message"]) @@ -39,6 +44,11 @@ def record(repository: str, value: dict) -> dict: def create( repository: str, name: str, payload: dict, base: str, previous: dict | None = None ) -> bool: + """Point branch ``name`` at a new commit whose message is ``payload`` as JSON. + + The commit sits on ``previous`` (fast-forward, no force) or on ``base`` for a new ref. + Return True if this caller won the ref, False if a concurrent writer did. + """ parent = previous["object"]["sha"] if previous else base commit = github.read(repository, "git/commits/" + parent) new = github.write( @@ -75,10 +85,12 @@ def create( def family_name(candidate: OwnedCandidate) -> str: + """Return the claim branch name derived from the candidate's family.""" return "klaud/claim-" + identity(candidate.family)[:16] def claim_family(repository: str, candidate: OwnedCandidate, parent: int) -> bool: + """Claim the candidate's family for run ``parent`` and return True if this call won.""" name = family_name(candidate) if ref(repository, name): return False @@ -87,6 +99,7 @@ def claim_family(repository: str, candidate: OwnedCandidate, parent: int) -> boo def release_family(repository: str, candidate: OwnedCandidate, parent: int) -> None: + """Delete the candidate's family claim if run ``parent`` owns exactly that candidate.""" name = family_name(candidate) value = ref(repository, name) if not value: @@ -98,6 +111,7 @@ def release_family(repository: str, candidate: OwnedCandidate, parent: int) -> N def family_owners(repository: str) -> list[Ownership]: + """Return every family claim's ownership, raising VerificationError on a malformed one.""" result = [] for value in github.items(repository, "git/matching-refs/heads/klaud/claim-"): owner = Ownership.model_validate(record(repository, value)) @@ -110,6 +124,10 @@ def family_owners(repository: str) -> list[Ownership]: def recovery_lease(repository: str, parent: int, candidate: OwnedCandidate) -> bool: + """Take the recovery lease for ``candidate`` of run ``parent`` and return True if held. + + A prior holder hands over only once its planner run or its reconcile job has completed. + """ name = f"klaud/recovery-{parent}-{candidate.id}" owner = { "run-id": int(os.environ["GITHUB_RUN_ID"]), diff --git a/inferencex-e2e/infx/klaud/github.py b/inferencex-e2e/infx/klaud/github.py index 0a8d55d8e2..aa7ea707ff 100644 --- a/inferencex-e2e/infx/klaud/github.py +++ b/inferencex-e2e/infx/klaud/github.py @@ -17,32 +17,41 @@ class VerificationError(ValueError): def read(repository: str, path: str, *, paginate: bool = False) -> list | dict: + """Return the decoded JSON from a GET of ``path`` in ``repository``, optionally paginated.""" return github.api(repository, path, paginate=paginate) def items(repository: str, path: str, key: str | None = None) -> list[dict]: + """Return every listed item at ``path``, unwrapped from ``key``, or raise VerificationError.""" try: return github.paginate(repository, path, item_key=key or "") except github.ListingError as error: raise VerificationError(str(error)) from error -# Every write bumps this, so listings cached before it are read again. writes = 0 def write(repository: str, path: str, method: str, payload: dict | None = None) -> dict: + """Send ``payload`` to ``path`` with ``method`` and return the JSON reply. + + Each call bumps the module ``writes`` counter so listings cached before it are read again. + """ global writes # noqa: PLW0603 writes += 1 return github.api(repository, path, method=method, data=payload) def artifacts(repository: str, run_id: int) -> list[dict]: + """Return the artifact records of workflow run ``run_id``.""" return items(repository, f"actions/runs/{run_id}/artifacts?per_page=100", "artifacts") def file_at(repository: str, head: str, path: str) -> bytes: - """Read data at an immutable revision; callers never execute downloaded code.""" + """Return the bytes of ``path`` at commit ``head``, raising VerificationError if unsafe. + + ``head`` must be a full SHA and the file at most 4 MiB. Callers never execute the result. + """ import re if not re.fullmatch(r"[0-9a-f]{40}", head): @@ -59,7 +68,10 @@ def file_at(repository: str, head: str, path: str) -> bytes: def download_json(repository: str, artifact: dict, destination: Path) -> None: - """Read bounded JSON only; never extract archive paths or execute artifact code.""" + """Write the JSON members of ``artifact`` under ``destination``, raising VerificationError. + + Archives over 256 MiB, unsafe paths and duplicates are rejected. Artifact code never runs. + """ limit = 256 * 1024 * 1024 if artifact["expired"] or not 0 < artifact["size_in_bytes"] <= limit: raise VerificationError("Artifact unavailable or too large") diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index 75df9c5c62..0b740900e1 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -1,4 +1,8 @@ -"""Finish owned sessions; resume or reconcile interrupted ones in the next autosweep.""" +"""Finish owned sessions; resume or reconcile interrupted ones in the next autosweep. + +A candidate gets at most ``MAX_SESSIONS`` sessions, its planner run's own plus three resume +runs, and planner waves start every ``WAVE_HOURS`` as scheduled in klaud-plan.yml. +""" from __future__ import annotations @@ -30,22 +34,14 @@ "evals-only", "agentx-fast", } -# Leaves the scheduler time to rerun what it preempted before recovery reruns it. RERUN_GRACE_SECONDS = 30 * 60 -# Recovery stops waiting for a preempted final sweep that stayed cancelled this long. RERUN_GIVE_UP_SECONDS = 24 * 60 * 60 MAX_SWEEP_ATTEMPTS = 10 -# Sessions per candidate: its planner run's session plus three resume runs. MAX_SESSIONS = 4 -# Planner waves start this often (klaud-plan.yml's schedule). WAVE_HOURS = 6 -# Recovery stops resuming, or keeping unreconciled sessions, this long after the last ended. RESUME_GIVE_UP_SECONDS = 24 * 60 * 60 -# The klaud-candidate.yml step that lets a resume run start only a session recovery granted. GATE_STEP = "Check whether this session should run" -# The klaud-candidate.yml step that runs the session's agent. AGENT_STEP = "Let Klaud Cold own the candidate PR" -# GitHub concludes a job stopped at its timeout-minutes as cancelled, with this annotation. TIMEOUT_ANNOTATION = "exceeded the maximum execution time" @@ -72,11 +68,13 @@ class ResumeSession(PendingCleanup): """Recovery may resume this unfinished session; keep its PR, branch, claim and lease.""" def __init__(self, pull: dict) -> None: + """Carry the resumable ``pull`` with a fixed public message.""" super().__init__("Unfinished session approved to resume in a new run") self.pull = pull def terminal(run: dict) -> bool: + """Return True if workflow ``run`` has completed with a conclusion.""" return run["status"] == "completed" and bool(run.get("conclusion")) @@ -89,6 +87,7 @@ def __init__( *, recovering: bool = False, ) -> None: + """Track ``candidate`` of planner run ``parent`` in ``repository``, as recovery if set.""" self.repository, self.parent, self.candidate = repository, parent, candidate self.recovering = recovering self.branch = f"klaud/auto-{candidate.id}" @@ -105,6 +104,10 @@ def listing(self, path: str, key: str | None = None) -> list[dict]: return cached[1] def validation(self, run: dict) -> tuple[dict, list[dict], list[dict]]: + """Return the sweep evidence of final sweep ``run``, cached per run attempt. + + Raises VerificationError if the sweep or its baseline coverage fails verification. + """ key = (run["id"], run["head_sha"], run["run_attempt"]) if key not in self.validations: from .reporting import baseline_for, check_baseline_coverage @@ -129,6 +132,11 @@ def pull_window_end(self) -> datetime | None: return self.first_attempt_end def pulls(self) -> list[dict]: + """Return this candidate's PR as a list of at most one, raising VerificationError. + + A PR opened outside the planner's first attempt is ignored. More than one, or one not + owned by the bot from this repository, is an error. + """ query = urlencode( { "state": "all", @@ -152,10 +160,12 @@ def pulls(self) -> list[dict]: return pulls def resume_runs(self) -> list[dict]: - """Recovery's resume runs of this candidate's session, oldest first.""" + """Recovery's resume runs of this candidate's session, oldest first. + + A session resumes within RESUME_GIVE_UP_SECONDS of the last ending and lasts at most a + GitHub job's 6 hours, which bounds when every resume run starts. + """ prefix = resume_title(self.parent["id"], self.candidate.id, 0).removesuffix("0") - # A session resumes within RESUME_GIVE_UP_SECONDS of the last one ending, and each - # session lasts at most a GitHub job's 6 hours, so every resume run starts by then. until = utc(self.parent["created_at"]) + MAX_SESSIONS * timedelta( seconds=RESUME_GIVE_UP_SECONDS, hours=6 ) @@ -191,6 +201,10 @@ def last_session_end(self) -> str | None: return max((run["updated_at"] for run in runs), key=utc) def runs(self) -> list[dict]: + """Return the candidate's targeted e2e runs followed by its PR sweep runs. + + In recovery, raises VerificationError if a maintainer started a sweep after the session. + """ ended = self.last_session_end() end = ended or "*" query = {"per_page": 100, "created": f"{self.parent['created_at']}..{end}"} @@ -239,9 +253,11 @@ def runs(self) -> list[dict]: return targeted + sweeps def aborted(self, pull: dict) -> bool: + """Return True if ``pull`` carries the abort label.""" return any(label["name"] == ABORT_LABEL for label in pull["labels"]) def refresh(self, pull: dict) -> dict: + """Return fresh ``pull``, raising VerificationError if it moved, merged or changed hands.""" self.check_parent() current = github.read(self.repository, f"pulls/{pull['number']}") if ( @@ -253,6 +269,7 @@ def refresh(self, pull: dict) -> dict: return current def check_parent(self) -> None: + """Drop cached listings and, in recovery, raise VerificationError if the session resumed.""" self.listings.clear() if self.recovering and ( not terminal(github.read(self.repository, f"actions/runs/{self.parent['id']}")) @@ -261,6 +278,7 @@ def check_parent(self) -> None: raise VerificationError("Session resumed; leave the active session intact") def report(self, pull: dict) -> dict | None: + """Return the outcome record of this session's latest bot report on ``pull``, or None.""" comments = self.listing(f"issues/{pull['number']}/comments?per_page=100") matches = [ comment @@ -273,6 +291,10 @@ def report(self, pull: dict) -> dict | None: return json.loads(body[len(self.marker) :].split("\n-->", 1)[0]) def pending(self, pull: dict) -> CandidateOutcome | None: + """Return the outcome of this session's pending cleanup request on ``pull``, or None. + + Raises VerificationError if the PR head moved since or the outcome claims validated. + """ marker = self.marker.replace("klaud-outcome:", "klaud-cleanup:") comments = self.listing(f"issues/{pull['number']}/comments?per_page=100") matches = [ @@ -290,6 +312,7 @@ def pending(self, pull: dict) -> CandidateOutcome | None: return outcome def branch_exists(self) -> bool: + """Return True if the candidate branch exists.""" refs = self.listing("git/matching-refs/heads/" + self.branch) return any(ref["ref"] == "refs/heads/" + self.branch for ref in refs) @@ -311,6 +334,11 @@ def verify( require_ready: bool = True, require_branch_deleted: bool = True, ) -> None: + """Raise unless GitHub state matches ``outcome`` for this session, else return None. + + Raises PendingCleanup while owned jobs run and VerificationError on any mismatch. The + flags relax the report, ready-for-review and deleted-branch checks. + """ pulls = self.pulls() pull = pulls[0] if pulls else None if pull and pull["merged_at"]: @@ -325,7 +353,6 @@ def verify( if outcome.outcome == "validated": if not pull or pull["state"] != "open" or (require_ready and pull["draft"]): raise VerificationError("Validated PR must remain ready for review") - # Klaud runs every final sweep fail-fast; any other sweep label is invalid. sweep_labels = {label["name"] for label in pull["labels"]} & SWEEP_LABELS if sweep_labels != {"full-sweep-fail-fast"}: raise VerificationError("Validated PR has incompatible sweep labels") @@ -374,6 +401,11 @@ def verify( raise VerificationError("Missing verified completion report") def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: + """Apply ``outcome`` to this session and return it with the real PR and run IDs. + + Validated PRs are published and marked ready. Others have runs cancelled, the PR closed + and the branch deleted. Then the report is posted and the family claim released. + """ pulls = self.pulls() pull = pulls[0] if pulls else None runs = self.runs() @@ -435,7 +467,6 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: try: github.write(self.repository, f"actions/runs/{run['id']}/cancel", "POST") except (OSError, subprocess.SubprocessError): - # GitHub refuses to cancel a run that ended after it was listed. if not terminal(github.read(self.repository, f"actions/runs/{run['id']}")): raise if any(not terminal(run) for run in self.runs()): @@ -541,6 +572,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: def subprocess_ready(repository: str, number: int, *, undo: bool = True) -> None: + """Mark PR ``number`` ready for review, or back to draft if ``undo``, counting a write.""" github.writes += 1 subprocess.run( ["gh", "pr", "ready", str(number), "--repo", repository] + (["--undo"] if undo else []), @@ -551,17 +583,21 @@ def subprocess_ready(repository: str, number: int, *, undo: bool = True) -> None def current_session() -> Session: + """Return the Session for the candidate in ``$KLAUD_EVIDENCE/candidate.json``. + + A resume run passes ``KLAUD_PARENT_RUN``, the planner run that owns the candidate. + """ context = json.loads((Path(os.environ["KLAUD_EVIDENCE"]) / "candidate.json").read_text()) candidate = OwnedCandidate.model_validate( {key: context[key] for key in ("id", "family", "base")} ) repository = os.environ["GITHUB_REPOSITORY"] - # A resume run continues the session of the planner run that owns the candidate. parent = os.environ.get("KLAUD_PARENT_RUN") or os.environ["GITHUB_RUN_ID"] return Session(repository, github.read(repository, f"actions/runs/{int(parent)}"), candidate) def hit_timeout(repository: str, job: dict) -> bool: + """Return True if ``job`` ran and GitHub annotated it as stopped at its timeout-minutes.""" return bool(job.get("runner_name")) and any( TIMEOUT_ANNOTATION in (annotation.get("message") or "") for annotation in github.items( @@ -571,7 +607,10 @@ def hit_timeout(repository: str, job: dict) -> bool: def preempted(repository: str, run: dict) -> bool: - """Cancelled by someone else: no job of its current attempt failed or hit its timeout.""" + """Cancelled by someone else: no job of its current attempt failed or hit its timeout. + + Failures are ruled out first because a fail-fast sweep cancels every other job. + """ if run["conclusion"] != "cancelled": return False jobs = github.items( @@ -579,14 +618,17 @@ def preempted(repository: str, run: dict) -> bool: f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100", "jobs", ) - # A fail-fast sweep cancels every other job, so rule out failures before reading annotations. return not any(job["conclusion"] in ("failure", "timed_out") for job in jobs) and not any( job["conclusion"] == "cancelled" and hit_timeout(repository, job) for job in jobs ) def await_rerun(session: Session, pull: dict, run: dict, now: datetime) -> None: - """Keep a preempted final sweep's candidate alive until the sweep runs again.""" + """Raise AwaitingRerun to keep ``session`` while preempted final sweep ``run`` can rerun. + + Waits RERUN_GRACE_SECONDS for the scheduler's own rerun, then requests one. Returns if + ``run`` was not preempted or is out of attempts. + """ try: if not preempted(session.repository, run): return @@ -664,30 +706,34 @@ def sessions_started(session: Session) -> int: def resumes(session: Session, pull: dict | None, now: datetime) -> bool: - """Whether a new resume run would continue this unfinished session.""" + """Return True if a new resume run would continue this unfinished session at ``now``. + + The last started job must have ended, run the gate step, and failed, timed out or been + cancelled within RESUME_GIVE_UP_SECONDS. Only klaud-abort stops a candidate. + """ if session.last_session_end() is None or not resumable(session, pull): return False started = [job for job in candidate_jobs(session) if started_session(job)] return ( 0 < len(started) < MAX_SESSIONS - # Runs that failed before their agent started never extend this. and bool(started[-1].get("completed_at")) and now - utc(started[-1]["completed_at"]) < timedelta(seconds=RESUME_GIVE_UP_SECONDS) - # A planner run from before resumption has no gate. and any(step["name"] == GATE_STEP for step in started[-1]["steps"]) - # A session that runs out fails its job, and a cancelled session was preempted like any - # other Klaud work; only klaud-abort stops a candidate. and started[-1]["conclusion"] in ("failure", "timed_out", "cancelled") ) def resume(session: Session, pull: dict | None, now: datetime) -> None: - """Approve resuming an unfinished session; recovery grants approved ones after its pass.""" + """Raise ResumeSession with ``pull`` when recovery may resume this session, else return. + + Recovery grants the approved sessions after its pass. + """ if session.recovering and resumes(session, pull, now): raise ResumeSession(pull) def resume_marker(parent: int, candidate: str, number: int) -> str: + """Return the comment marker that records the grant of session ``number``.""" return f"" @@ -703,7 +749,10 @@ def granted(session: Session, pull: dict, number: int) -> bool: def grant(session: Session, pull: dict) -> int: - """Grant an unfinished session its next session, then dispatch the resume run that runs it.""" + """Grant an unfinished session its next session, dispatch its resume run, return its number. + + If the dispatch fails the grant comment is deleted, so no uncounted session can start. + """ repository, parent = session.repository, session.parent["id"] session.refresh(pull) number = sessions_started(session) + 1 @@ -740,7 +789,6 @@ def grant(session: Session, pull: dict) -> int: }, ) except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): - # A grant without its resume run would let a later run start an uncounted session. if posted is not None: with contextlib.suppress( OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError @@ -767,6 +815,7 @@ def retained(session: Session, now: datetime) -> bool: def summarize(repository: str, candidate: OwnedCandidate, parent: int, status: str) -> None: + """Append ``status`` for ``candidate`` of run ``parent`` to the job summary, if one exists.""" if summary := os.environ.get("GITHUB_STEP_SUMMARY"): with open(summary, "a") as output: output.write( @@ -776,6 +825,11 @@ def summarize(repository: str, candidate: OwnedCandidate, parent: int, status: s def reconcile(session: Session, now: datetime) -> bool: + """Settle interrupted ``session`` at ``now`` and return True if anything changed. + + Raises PendingCleanup subclasses while work runs, a recent preempted sweep may rerun, + or the session may resume. Returns False for a merged PR or no owned work. + """ pulls = session.pulls() pull = pulls[0] if pulls else None if pull and pull["merged_at"]: @@ -855,6 +909,11 @@ def reconcile(session: Session, now: datetime) -> bool: def recover(max_resumes: int) -> None: + """Reconcile every owned candidate, then resume up to ``max_resumes`` approved sessions. + + Writes still-blocked families to ``$RUNNER_TEMP/klaud-recovery.json`` and the resume count + to ``$GITHUB_OUTPUT``. Unreadable state counts as live work, and oldest planners go first. + """ repository = os.environ["GITHUB_REPOSITORY"] now = datetime.now(UTC) inventory = github.items( @@ -894,7 +953,6 @@ def recover(max_resumes: int) -> None: for candidate in owner.candidates: results[(owner.run_id, candidate.id)] = False blocked.add(candidate.family) - # Unreadable state counts as live work, as in retained(). live.add(candidate.id) summarize( repository, @@ -964,7 +1022,6 @@ def recover(max_resumes: int) -> None: if status: summarize(repository, candidate, parent["id"], status) resumed = 0 - # Oldest planner runs first; resumed sessions share this wave's candidate cap. approved.sort(key=lambda pair: (pair[0].parent["id"], pair[0].candidate.id)) for index, (session, pull) in enumerate(approved): run_id = session.parent["id"] @@ -1011,6 +1068,11 @@ def recover(max_resumes: int) -> None: def owned_activity(repository: str) -> tuple[set[str], dict[str, list[int]], set[int]]: + """Return active Klaud work in ``repository``. + + That is live candidate IDs, active targeted run IDs per candidate, and planner runs whose + reconcile job is unfinished. + """ live: set[str] = set() targeted: dict[str, list[int]] = {} planners: set[int] = set() @@ -1041,12 +1103,18 @@ def owned_activity(repository: str) -> tuple[set[str], dict[str, list[int]], set def bot_committed(commits: list[dict]) -> bool: + """Return True if ``commits`` is non-empty and the Actions bot committed all of them.""" return bool(commits) and all( (commit.get("committer") or {}).get("login") == COMMITTER for commit in commits ) def sweep_abandoned(repository: str, live: set[str]) -> set[str]: + """Clean up Klaud work no live session owns and return the widened ``live`` candidate IDs. + + Closes abandoned draft PRs and cancels orphan targeted runs. Deletes stale claims, leases + and branches. A claim whose planner run cannot be read counts as live. + """ sessions, targeted, planners = owned_activity(repository) live = live | sessions claimed = [] @@ -1057,7 +1125,6 @@ def sweep_abandoned(repository: str, live: set[str]) -> set[str]: try: parent = github.read(repository, f"actions/runs/{owner.run_id}") except (OSError, ValueError, subprocess.SubprocessError): - # A claim whose planner run cannot be read may still own live work. live.add(candidate.id) continue claimed.append((owner, candidate, parent)) diff --git a/inferencex-e2e/infx/klaud/models.py b/inferencex-e2e/infx/klaud/models.py index 05b0af3a74..c952defae8 100644 --- a/inferencex-e2e/infx/klaud/models.py +++ b/inferencex-e2e/infx/klaud/models.py @@ -1,4 +1,7 @@ -"""Versioned local contracts, deliberately separate from external API schemas.""" +"""Versioned local contracts, deliberately separate from external API schemas. + +``ABORT_LABEL`` is the maintainer label that stops a candidate now and blocks retries while set. +""" from __future__ import annotations @@ -9,11 +12,11 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -# A maintainer label that stops a candidate now and keeps the planner from retrying it while set. ABORT_LABEL = "klaud-abort" def utc(value: str | datetime) -> datetime: + """Return an ISO timestamp or aware datetime as a UTC datetime, raising ValueError if naive.""" result = ( datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value ) @@ -23,17 +26,22 @@ def utc(value: str | datetime) -> datetime: def stamp(value: datetime) -> str: + """Return an aware datetime as an ISO UTC string ending in ``Z``.""" return utc(value).isoformat().replace("+00:00", "Z") def identity(value: Any) -> str: + """Return the SHA-256 hex digest of ``value`` as canonical JSON.""" return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() ).hexdigest() def normalized_image(value: str) -> str: - """Normalize registry paths without changing tags, digests or repositories.""" + """Return image ``value`` with a ``registry#path`` separator turned into ``/``. + + Tags, digests and repositories are left unchanged. + """ registry, separator, remainder = value.partition("#") if separator and remainder and ("." in registry or ":" in registry): return registry + "/" + remainder @@ -66,6 +74,7 @@ class CandidateReview(Contract): @model_validator(mode="after") def consistent_decision(self) -> CandidateReview: + """Return the review, raising ValueError if the decision contradicts its fields.""" if self.decision == "proceed" and ( not self.family or self.pull_requests or not self.baseline_model ): @@ -125,6 +134,7 @@ class CandidateOutcome(Contract): @model_validator(mode="after") def consistent_outcome(self) -> CandidateOutcome: + """Return the outcome, raising ValueError if phase, PR, runs or reason code disagree.""" if self.outcome != "unexpected-error" and self.phase == "unknown": raise ValueError("Completed outcomes require a known phase") if self.outcome in ("validated", "aborted") and self.pull_request is None: @@ -154,6 +164,7 @@ class Feed(Contract): @field_validator("retrieved_at") @classmethod def valid_timestamp(cls, value: str) -> str: + """Return ``value`` unchanged, raising ValueError unless it is an aware ISO timestamp.""" utc(value) return value @@ -175,6 +186,7 @@ class PublicRow(BaseModel): @field_validator("date") @classmethod def valid_date(cls, value: str) -> str: + """Return ``value`` unchanged, raising ValueError unless it is a YYYY-MM-DD date.""" from datetime import date if date.fromisoformat(value).isoformat() != value: diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index 95b8257214..b7c6d04ec4 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -44,6 +44,10 @@ def public_prose(value: str) -> str: + """Return ``value`` unchanged, raising ValueError on mentions, HTML, newlines or pipes. + + Backquoted image digests are exempt from the check. + """ prose = re.sub(r"`[A-Za-z0-9_./:+-]+@sha256:[0-9a-f]{64}`", "", value) if any(char in prose for char in ("@", "<", ">", "\n", "|")): raise ValueError("Use one sentence of public prose without mentions, HTML or tables") @@ -107,6 +111,7 @@ class Point(Contract): @field_validator("dataset", mode="before") @classmethod def canonical_dataset(cls, value: Any) -> Any: + """Return a string ``value`` as its dashboard dataset slug, anything else unchanged.""" return dataset_slug(value) if isinstance(value, str) else value @@ -159,6 +164,7 @@ class Baseline(Contract): @field_validator("sources") @classmethod def public_sources(cls, urls: list[str]) -> list[str]: + """Return ``urls``, raising ValueError unless each is a public dashboard API URL.""" from urllib.parse import urlsplit for url in urls: @@ -174,6 +180,7 @@ def public_sources(cls, urls: list[str]) -> list[str]: @model_validator(mode="after") def distinct(self) -> Self: + """Return the baseline, raising ValueError on duplicates or invalid retirements.""" unique_points(self.points) unique_evals(self.evals) retired = [key for retirement in self.retirements for key in retirement.points] @@ -189,6 +196,7 @@ def distinct(self) -> Self: @model_serializer(mode="wrap") def compatible(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + """Return the serialized baseline, omitting ``retirements`` when it is empty.""" data = handler(self) if not self.retirements: data.pop("retirements", None) @@ -227,6 +235,7 @@ class Attempt(Contract): @model_validator(mode="after") def consistent(self) -> Self: + """Return the attempt, raising ValueError if coverage, numbering or provenance disagree.""" unique_points(self.points) unique_evals(self.evals) if ( @@ -251,11 +260,13 @@ def consistent(self) -> Self: def unique_points(points: list[Point]) -> None: + """Raise ValueError if two ``points`` share a key.""" if len({point.key for point in points}) != len(points): raise ValueError("Duplicate comparison point") def unique_evals(rows: list[Evaluation]) -> None: + """Raise ValueError if two eval ``rows`` share key, suite and metric.""" if len({(row.key, row.suite, row.metric) for row in rows}) != len(rows): raise ValueError("Duplicate comparison eval") @@ -268,6 +279,7 @@ def day(value: str) -> date: def point_key(entry: dict) -> str: + """Return the identity of matrix ``entry`` without image, name, fingerprint or queue fields.""" return identity( { key: value @@ -285,6 +297,7 @@ def point_key(entry: dict) -> str: def point_label(entry: dict) -> str: + """Return a label for matrix ``entry`` with its workload, concurrency, shape and key prefix.""" if entry.get("prefill") is not None: shape = "/".join( f"{role[0].upper()}TP{entry[role]['tp']}x{entry[role]['num-worker']}" @@ -303,7 +316,10 @@ def label_workload(label: str) -> tuple[int, int] | None: def values(row: dict) -> Values: - """Collector/API metrics are seconds; only this presentation layer converts ms.""" + """Return the throughput, latency in ms and error count of result ``row``. + + Collector and API metrics are seconds, and only this presentation layer converts to ms. + """ metrics = row.get("metrics", row) if "request_metrics" in metrics: request = metrics["request_metrics"] @@ -327,21 +343,27 @@ def values(row: dict) -> Values: def number(value: float | None) -> str: + """Return ``value`` with two decimals and separators, trailing zeros trimmed, or N/A.""" return "N/A" if value is None else f"{value:,.2f}".rstrip("0").rstrip(".") def delta(old: float | None, new: float | None, comparable: bool = True) -> str: + """Return the signed percent change from ``old`` to ``new``, or N/A if not comparable.""" if not comparable or old is None or new is None or old == 0: return "N/A" return f"{(new / old - 1) * 100:+.1f}%".replace(".0%", "%") def translated(english: str, chinese: str) -> str: + """Return ``english`` followed by ``chinese`` in a collapsed details block.""" return english + "\n\n
\n中文\n\n" + chinese + "\n\n
" def short_label(label: str) -> str: + """Return ``label`` with its first ISL/OSL pair written in k where it divides by 1024.""" + def shape(match: re.Match) -> str: + """Return the matched ISL/OSL pair with each multiple of 1024 written in k.""" return "/".join( f"{int(n) // 1024}k" if int(n) > 0 and int(n) % 1024 == 0 else n for n in match.groups() ) @@ -350,7 +372,10 @@ def shape(match: re.Match) -> str: def point_layout(points: list[Point]) -> tuple[str, list[str], str]: - """Factor out only shared settings; retain full labels for ambiguous concurrencies.""" + """Return the first column title, row labels and shared caption for ``points``. + + Only settings shared by every point are factored out, so ambiguous concurrencies keep labels. + """ labels = [short_label(p.label) for p in points] shapes = [ re.fullmatch(r"(.+) c\d+ (.+)", re.sub(r" [0-9a-f]{6}$", "", label)) for label in labels @@ -380,6 +405,7 @@ def point_layout(points: list[Point]) -> tuple[str, list[str], str]: def table(header: list[str], rows: list[list[str]]) -> str: + """Return a Markdown table of ``header`` and ``rows``, or an empty string without rows.""" if not rows: return "" alignment = [ @@ -396,6 +422,7 @@ def table(header: list[str], rows: list[list[str]]) -> str: def note_lines(issues: dict[str, list[str]], total: int) -> str: + """Return one Markdown note per reason in ``issues``, naming its labels or all rows.""" return " \n".join( f"**Note:** {'All rows' if len(labels) == total else ', '.join(labels)}: {reason}." for reason, labels in issues.items() @@ -403,6 +430,7 @@ def note_lines(issues: dict[str, list[str]], total: int) -> str: def measured(new: float | None, old: float | None, comparable: bool) -> str: + """Return ``new`` with its change from ``old``, or N/A if ``new`` is missing.""" return "N/A" if new is None else f"{number(new)} ({delta(old, new, comparable)})" @@ -429,6 +457,10 @@ def mismatch(previous: Point | None, point: Point) -> str | None: def point_table( points: list[Point], baseline: Baseline | None = None, *, context: bool = True ) -> str: + """Return the Markdown comparison of ``points`` against ``baseline`` with its notes. + + ``context`` adds the shared settings caption. + """ old = {point.key: point for point in baseline.points} if baseline else {} heading, labels, settings = point_layout(points) rows, issues = [], {} @@ -479,6 +511,7 @@ def point_table( def eval_table(rows: list[Evaluation], baseline: Baseline | None, *, compare: bool = True) -> str: + """Return the Markdown table of eval ``rows`` with notes, versus ``baseline`` if ``compare``.""" previous = {(row.key, row.suite, row.metric): row for row in baseline.evals} if baseline else {} values, issues = [], {} for row in rows: @@ -539,6 +572,10 @@ def retirement_reasons(baseline: Baseline) -> dict[str, str]: def baseline_table(baseline: Baseline, *, context: bool = True) -> str: + """Return the Markdown table of the frozen ``baseline`` points with notes. + + ``context`` adds the shared settings caption. + """ points = baseline.points retired = retirement_reasons(baseline) heading, labels, settings = point_layout(points) @@ -585,6 +622,7 @@ def baseline_table(baseline: Baseline, *, context: bool = True) -> str: def render_body(baseline: Baseline) -> str: + """Return the bilingual PR body section that presents ``baseline`` and the abort label.""" _, _, settings = point_layout(baseline.points) sources = ", ".join(f"[API {i + 1}]({url})" for i, url in enumerate(baseline.sources)) or "N/A" meta = " · ".join(part for part in (settings, f"Sources: {sources}") if part) @@ -610,6 +648,7 @@ def render_body(baseline: Baseline) -> str: def render_attempt(record: Attempt, baseline: Baseline | None, repository: str) -> str: + """Return the bilingual comment for attempt ``record`` in ``repository`` versus ``baseline``.""" titles = { "initial": ("Initial attempt", "初次尝试"), "repair": (f"Repair {record.number}/5", f"修复 {record.number}/5"), @@ -664,10 +703,15 @@ def render_attempt(record: Attempt, baseline: Baseline | None, repository: str) def marker(session: Session, name: str) -> str: + """Return the comment marker of report ``name`` in ``session``.""" return f"", 1)[0]) if "record" in value: record = value["record"] @@ -699,6 +747,7 @@ def decode[Record: Contract]( def latest_next(session: Session, pull: dict) -> Prose | None: + """Return the next step of the session's latest attempt report on ``pull``, or None.""" attempts = [ c for c in github.items(session.repository, f"issues/{pull['number']}/comments?per_page=100") @@ -717,6 +766,7 @@ def latest_next(session: Session, pull: dict) -> Prose | None: def baseline_for(session: Session, pull: dict) -> Baseline | None: + """Return the baseline frozen in ``pull``'s body or baseline comment, or None.""" current = session.refresh(pull) embedded = baseline_from_body(current.get("body") or "") if embedded: @@ -726,12 +776,17 @@ def baseline_for(session: Session, pull: dict) -> Baseline | None: def baseline_marker(record: Baseline) -> str: + """Return a hidden HTML comment embedding ``record`` as compressed base64 JSON.""" packed = zlib.compress(record.model_dump_json(by_alias=True).encode(), level=9) encoded = base64.urlsafe_b64encode(packed).decode() return f"" def baseline_from_body(body: str) -> Baseline | None: + """Return the baseline embedded in PR ``body``, or None. + + Raises VerificationError if the record is ambiguous or invalid. + """ prefix = "" @@ -772,7 +832,10 @@ def initialize_body(session: Session, pull: dict, record: Baseline) -> None: def check_reported_datasets(record: Attempt) -> None: - """A hand-written attempt must name each passed AgentX point's corpus, or deltas stay N/A.""" + """Raise VerificationError if a passed AgentX point of ``record`` lacks its dataset. + + Without the corpus name its deltas would stay N/A. + """ if any( point.scenario == "agentic-coding" and point.result == "passed" and not point.dataset for point in record.points @@ -783,6 +846,11 @@ def check_reported_datasets(record: Attempt) -> None: def publish(session: Session, record: Baseline | Attempt) -> None: + """Publish ``record`` to the candidate PR, raising VerificationError if it mismatches. + + A baseline is frozen in the PR body. An attempt is upserted as a comment, split into + numbered chunk comments past 20 points or evals. + """ pulls = session.pulls() if not pulls: raise VerificationError("Candidate PR not found") @@ -846,6 +914,10 @@ def publish(session: Session, record: Baseline | Attempt) -> None: def upsert(session: Session, pull: dict, name: str, data: str, text: str) -> None: + """Create or update report comment ``name`` on ``pull`` with hidden ``data`` and ``text``. + + Raises VerificationError if the body exceeds GitHub's comment size. + """ previous = stored(session, pull, name) body = marker(session, name) + data + "\n-->\n" + text if len(body) > 60000: @@ -860,7 +932,11 @@ def upsert(session: Session, pull: dict, name: str, data: str, text: str) -> Non def publish_final( session: Session, run: dict, evidence: tuple[dict, list[dict], list[dict]] ) -> None: - """Normal finish and recovery publish the same artifact-derived final report.""" + """Publish the final report of passed sweep ``run`` built from its verified ``evidence``. + + Normal finish and recovery publish the same artifact-derived report. Mixed images raise + VerificationError. + """ from infx.results.eval_artifacts import eval_key from .validation import benchmark_entries, expected_evals @@ -991,6 +1067,7 @@ def _matches_public_point(row: dict, point: dict) -> bool: def matrix_points(matrix: dict) -> list[dict]: + """Return one copy of each benchmark entry in ``matrix`` per concurrency, as ``conc``.""" from .validation import benchmark_entries return [ @@ -1010,7 +1087,9 @@ def missing_baseline_points(matrix: dict, baseline: Baseline) -> list[Point]: def check_baseline_coverage( repository: str, candidate: OwnedCandidate, matrix: dict, baseline: Baseline | None ) -> None: - """Current-family completeness cannot replace the frozen original point roster. + """Raise VerificationError unless ``matrix`` keeps the frozen ``baseline`` point roster. + + Current-family completeness cannot replace that roster. The agent can rewrite the recorded baseline, so its retirements exempt points only when the planner's rule, re-run on the family and MODELS.md at the candidate base (never the PR @@ -1126,6 +1205,7 @@ def markdown_table(markdown: str, header: tuple[str, ...]) -> list[dict[str, str """ def cells(line: str) -> list[str] | None: + """Return the trimmed cells of table row ``line``, or None if it is not one.""" line = line.strip() if len(line) < 2 or line[0] != "|" or line[-1] != "|": return None @@ -1146,6 +1226,7 @@ def cells(line: str) -> list[str] | None: def calendar_date(value: str) -> bool: + """Return True if ``value`` parses as an ISO date.""" try: datetime.fromisoformat(value) except ValueError: @@ -1173,6 +1254,7 @@ def deprecated_workloads( return {} def lists(cell: str, scenario: str) -> bool: + """Return True if ``cell`` names ``scenario`` as a whole word.""" return re.search(rf"(? dict: - """The public eval-row fields that identify a default eval of ``entry`` at ``conc``.""" + """The public eval-row fields that identify a default eval of ``entry`` at ``conc``. + + Eval rows carry no workload, offload or image, and eval ingest encodes an aggregate + multi-node deployment as disaggregated without decode workers. + """ point = public_point({**entry, "conc": conc}) omitted = ( EVAL_IDENTITY_OMITS @@ -1370,6 +1454,10 @@ def resolve_baseline( current = {key: public_point(entry) for key, entry in entries.items()} def current_key(entry: dict) -> str | None: + """Return the current-family key of producer ``entry``, or None if it has none. + + Raises VerificationError if several current points match. + """ key = point_key(entry) if key in current: return key diff --git a/inferencex-e2e/infx/klaud/validation.py b/inferencex-e2e/infx/klaud/validation.py index 99fd284577..b5deafd5d5 100644 --- a/inferencex-e2e/infx/klaud/validation.py +++ b/inferencex-e2e/infx/klaud/validation.py @@ -19,15 +19,19 @@ if TYPE_CHECKING: from infx.matrix.revision import Revision -# Klaud autosweeps cover only AgentX; fixed-seq-len points are out of scope. SCENARIO = "agentic-coding" def in_scope(entry: dict) -> bool: + """Return True if matrix ``entry`` is an AgentX point. + + Klaud autosweeps cover only AgentX, so fixed-seq-len points are out of scope. + """ return entry.get("scenario-type") == SCENARIO def benchmark_entries(matrix: dict) -> list[dict]: + """Return every single-node and multi-node benchmark entry of ``matrix``.""" return [ entry for group in ("single_node", "multi_node") @@ -37,6 +41,7 @@ def benchmark_entries(matrix: dict) -> list[dict]: def benchmark_points(entries: list[dict]) -> set[tuple]: + """Return the (recipe fingerprint, concurrency, image) point of each entry concurrency.""" return { (entry["recipe-fingerprint"], int(conc), entry["image"]) for entry in entries @@ -51,7 +56,7 @@ def project_prefix(repository: str, head: str) -> str: def canonical_matrix(repository: str, head: str, family: str) -> dict: - """Generate the family's AgentX points from exact-head YAML using trusted local code. + """Return the family's AgentX matrix generated from exact-head YAML by trusted local code. Candidate source is data only. A future generator-policy change can require manual inspection of an older run; it cannot silently waive points or required evals. @@ -116,6 +121,7 @@ class ProducerRegenerationError(VerificationError): """ def __init__(self, stderr: str) -> None: + """Store the tool's ``stderr`` and carry the fixed public reason as the message.""" super().__init__(PRODUCER_UNAVAILABLE) self.stderr = stderr @@ -163,6 +169,10 @@ def producer_matrix(repository: str, head: str, family: str) -> dict: def throughput_rows(producer: Revision, key: str) -> list[dict]: + """Return the producer's throughput rows for config ``key`` without eval-only rows. + + Producers too old for ``--no-evals`` have their eval rows stripped here instead. + """ try: return producer.generate([key], ["--no-evals"]) except subprocess.CalledProcessError as error: @@ -179,6 +189,10 @@ def throughput_rows(producer: Revision, key: str) -> list[dict]: def check_matrix(matrix: dict, canonical: dict, head: str, family: str) -> None: + """Raise VerificationError unless ``matrix`` matches ``canonical`` for ``head`` and ``family``. + + Checks the selected family, per-point settings, duplicate points, coverage and default evals. + """ from pydantic import TypeAdapter from infx.matrix.validation import ( @@ -227,6 +241,7 @@ def check_matrix(matrix: dict, canonical: dict, head: str, family: str) -> None: def expected_evals(matrix: dict) -> set[tuple]: + """Return the artifact identities of every default eval in ``matrix``.""" return {key for key, _, _ in eval_identities(matrix)} @@ -271,6 +286,10 @@ def eval_identities(matrix: dict) -> list[tuple[tuple, dict, Any]]: def check_coverage( directory: Path, manifest: dict, run: dict, family: str, canonical: dict ) -> None: + """Raise VerificationError unless the artifacts in ``directory`` cover the sweep. + + ``manifest`` must match ``run``, and its matrix must match ``canonical`` for ``family``. + """ if ( manifest["head"] != run["head_sha"] or manifest["run-id"] != run["id"] @@ -302,11 +321,10 @@ def check_coverage( def select_artifacts(inventory: list[dict], run: dict, attempt: dict) -> list[dict]: - """Newest same-name artifact within this run; never overlay two archives. + """Return the newest ``inventory`` artifact of each wanted name in ``run``, never overlaid. - Retained point artifacts from successful jobs in an earlier attempt are allowed. - The final manifest must be current. An earlier aggregate is retained only when - none of its point producers reran. Coverage/raw agreement still must pass. + Points from before ``attempt`` may be retained, but the manifest must be current and an + older aggregate only stands if none of its producers reran. Raises VerificationError. """ selected: dict[str, dict] = {} for artifact in inventory: @@ -355,6 +373,10 @@ def select_artifacts(inventory: list[dict], run: dict, attempt: dict) -> list[di def verify_sweep(repository: str, run: dict, family: str) -> tuple[dict, list[dict], list[dict]]: + """Verify the passed final sweep ``run`` for ``family`` and return its data. + + Return the canonical matrix, benchmark rows and eval rows, or raise VerificationError. + """ if run["status"] != "completed" or run["conclusion"] != "success": raise VerificationError("Final sweep has not passed") attempt = github.read(repository, f"actions/runs/{run['id']}/attempts/{run['run_attempt']}") From e8b986ae0692b3323933f6c14c62407b1a69681f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:44:21 -0500 Subject: [PATCH 199/203] chore(lint): drop ruff suppressions the code can satisfy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use bare return in redirect handlers, contextlib.suppress for silent fallbacks, and top-level imports in merge_with_reuse. 中文:重定向处理器改用裸 return,静默回退改用 contextlib.suppress,merge_with_reuse 的导入移到文件顶部。 --- .../infx/evals/minimax_m3_full_eval.py | 2 +- inferencex-e2e/infx/klaud/api.py | 2 +- .../results/agentic/collect_sweep_results.py | 9 +++----- .../infx/results/collect_eval_results.py | 5 ++-- .../infx/workflows/merge_with_reuse.py | 23 ++++++++++--------- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py index 2be094dc68..fcb04545c9 100755 --- a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py @@ -59,7 +59,7 @@ class FullSuiteError(RuntimeError): class _RejectRedirects(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args: Any, **kwargs: Any) -> None: # noqa: ARG002 """Refuse every redirect by returning None.""" - return None # noqa: RET501 + return _NO_REDIRECT_OPENER = urllib.request.build_opener(_RejectRedirects()) diff --git a/inferencex-e2e/infx/klaud/api.py b/inferencex-e2e/infx/klaud/api.py index 2fae9aa9d5..b6023b9836 100644 --- a/inferencex-e2e/infx/klaud/api.py +++ b/inferencex-e2e/infx/klaud/api.py @@ -54,7 +54,7 @@ def redirect_request( newurl: str, # noqa: ARG002 ) -> None: """Refuse every redirect by returning None.""" - return None # noqa: RET501 + return def reject_nonfinite(value: str) -> Never: # noqa: ARG001 diff --git a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py index 1713a9763a..9eea0543db 100644 --- a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py +++ b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py @@ -14,6 +14,7 @@ python collect_sweep_results.py """ +import contextlib import json import sys from pathlib import Path @@ -150,12 +151,8 @@ def load_experiment(exp_dir: Path) -> dict | None: metadata_file = exp_dir / "benchmark_metadata.json" total_time_sec = None if metadata_file.exists(): - try: - with open(metadata_file) as f: - metadata = json.load(f) - total_time_sec = metadata.get("benchmark_runtime_sec") - except Exception: # noqa: BLE001, S110 - pass + with contextlib.suppress(Exception), open(metadata_file) as f: + total_time_sec = json.load(f).get("benchmark_runtime_sec") if not total_time_sec or total_time_sec <= 0: first_start_ms = df["start_time_ms"].min() diff --git a/inferencex-e2e/infx/results/collect_eval_results.py b/inferencex-e2e/infx/results/collect_eval_results.py index 870889727b..fb20a1d8cf 100644 --- a/inferencex-e2e/infx/results/collect_eval_results.py +++ b/inferencex-e2e/infx/results/collect_eval_results.py @@ -1,3 +1,4 @@ +import contextlib import json import sys from pathlib import Path @@ -59,14 +60,12 @@ def find_eval_sets(root: Path) -> list[Path]: extracted flat into root (no subdirectory), so check root itself too. """ out: list[Path] = [] - try: + with contextlib.suppress(Exception): if (root / "meta_env.json").exists(): out.append(root) out.extend( d for d in sorted(root.iterdir()) if d.is_dir() and (d / "meta_env.json").exists() ) - except Exception: # noqa: BLE001, S110 - pass return out diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index f541a9d64d..dc23c7c0ec 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -34,6 +34,18 @@ from infx.config import git_repository_root, project_root +from .prepare_perf_changelog_merge import ( + canonicalize_appended_links, + resolve_conflict_bytes, +) +from .sweep_runs import ( + artifact_names, + completed_pr_runs, + has_reusable_result_artifacts, + pr_commit_shas, +) +from .validate_perf_changelog import read_git_file + try: import git as gitpython import requests @@ -70,17 +82,6 @@ class RateLimitExceededException(GithubException): # type: ignore[no-redef] from github.PullRequest import PullRequest from github.Repository import Repository -from .prepare_perf_changelog_merge import ( # noqa: E402 - canonicalize_appended_links, - resolve_conflict_bytes, -) -from .sweep_runs import ( # noqa: E402 - artifact_names, - completed_pr_runs, - has_reusable_result_artifacts, - pr_commit_shas, -) -from .validate_perf_changelog import read_git_file # noqa: E402 DEFAULT_REPO = "SemiAnalysisAI/InferenceX" DEFAULT_CHECK_TIMEOUT = 900 From cad5f1749d17d3afdcbe0976d470987274afaa80 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:46:31 -0500 Subject: [PATCH 200/203] chore(lint): drop ruff suppressions the code can satisfy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop unused parameters from private helpers and their callers, and mark the stub GithubException.headers as ClassVar. 中文:删除私有辅助函数及其调用处未使用的参数,并将桩 GithubException.headers 标注为 ClassVar。 --- inferencex-e2e/infx/bench_serving/benchmark_serving.py | 2 -- inferencex-e2e/infx/datasets/proxy_to_weka.py | 4 ++-- inferencex-e2e/infx/evals/bfcl_adapter.py | 8 +++----- .../infx/results/agentic/collect_sweep_results.py | 4 ++-- inferencex-e2e/infx/results/agentic/request_metrics.py | 3 +-- .../infx/tests/bench_serving/test_benchmark_serving.py | 2 +- inferencex-e2e/infx/workflows/merge_with_reuse.py | 4 ++-- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/inferencex-e2e/infx/bench_serving/benchmark_serving.py b/inferencex-e2e/infx/bench_serving/benchmark_serving.py index 55ac1b8bc8..8239e70b44 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_serving.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_serving.py @@ -408,7 +408,6 @@ def calculate_metrics( outputs: list[RequestFuncOutput], dur_s: float, tokenizer: PreTrainedTokenizerBase, - selected_percentile_metrics: list[str], # noqa: ARG001 selected_percentiles: list[float], goodput_config_dict: dict[str, float], ) -> tuple[BenchmarkMetrics, list[int]]: @@ -678,7 +677,6 @@ async def limited_request_func( outputs=outputs, dur_s=benchmark_duration, tokenizer=tokenizer, - selected_percentile_metrics=selected_percentile_metrics, selected_percentiles=selected_percentiles, goodput_config_dict=goodput_config_dict, ) diff --git a/inferencex-e2e/infx/datasets/proxy_to_weka.py b/inferencex-e2e/infx/datasets/proxy_to_weka.py index 70f1e3492c..b713e2874d 100644 --- a/inferencex-e2e/infx/datasets/proxy_to_weka.py +++ b/inferencex-e2e/infx/datasets/proxy_to_weka.py @@ -135,7 +135,7 @@ def remap_hash(h: str, m: dict[str, int]) -> int: return m[h] -def infer_block_size(rows: list[dict]) -> int: # noqa: ARG001 +def infer_block_size() -> int: """Return the proxy's 64-token block size. Partial trailing blocks make hash_token_count / len(hash_ids) unreliable. @@ -331,7 +331,7 @@ def session_to_weka(session_id: str, rows: list[dict]) -> dict: think_times = compute_think_times(rows) hash_map: dict[str, int] = {} - block_size = infer_block_size(rows) + block_size = infer_block_size() out_requests: list[dict] = [] instance_count: dict[str, int] = {} diff --git a/inferencex-e2e/infx/evals/bfcl_adapter.py b/inferencex-e2e/infx/evals/bfcl_adapter.py index b11f290016..8646526a60 100644 --- a/inferencex-e2e/infx/evals/bfcl_adapter.py +++ b/inferencex-e2e/infx/evals/bfcl_adapter.py @@ -243,7 +243,6 @@ def _positive_int(value: str) -> int: def _source_details( - suite: SuiteSpec, # noqa: ARG001 case_ids_by_category: Mapping[str, tuple[str, ...]], ) -> dict[str, Any]: """Return the pinned upstream source metadata plus the case IDs selected per category.""" @@ -287,13 +286,12 @@ def _expected_category_details( def _diagnostics( - suite: SuiteSpec, case_ids_by_category: Mapping[str, tuple[str, ...]], scores: Sequence[CategoryScore] | None = None, ) -> dict[str, Any]: """Return source metadata and per-category diagnostics, with placeholders if scores is None.""" return { - "source": _source_details(suite, case_ids_by_category), + "source": _source_details(case_ids_by_category), "categories": ( [score.as_dict() for score in scores] if scores is not None @@ -334,7 +332,7 @@ def _native_report( "total_count": total_count, "expected_count": suite.expected_sample_count, }, - "bfcl": _diagnostics(suite, case_ids_by_category, scores), + "bfcl": _diagnostics(case_ids_by_category, scores), } if integration_error is not None: report["integration_error"] = _error_dict(integration_error) @@ -411,7 +409,7 @@ def _compatibility_result( "results": results, "configs": configs, "n-samples": task_samples, - "bfcl": _diagnostics(suite, case_ids_by_category, scores), + "bfcl": _diagnostics(case_ids_by_category, scores), } if integration_error is not None: result["integration_error"] = _error_dict(integration_error) diff --git a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py index 9eea0543db..82c20e6d72 100644 --- a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py +++ b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py @@ -22,7 +22,7 @@ import pandas as pd -def _load_custom_client_csv(client_csv: Path, exp_dir: Path) -> pd.DataFrame | None: # noqa: ARG001 +def _load_custom_client_csv(client_csv: Path) -> pd.DataFrame | None: """Load per-request metrics from custom benchmark client CSV.""" df = pd.read_csv(client_csv) if len(df) == 0: @@ -144,7 +144,7 @@ def load_experiment(exp_dir: Path) -> dict | None: return result result.update(aiperf_metrics) elif client_csv.exists(): - df = _load_custom_client_csv(client_csv, exp_dir) + df = _load_custom_client_csv(client_csv) if df is None or len(df) == 0: return result diff --git a/inferencex-e2e/infx/results/agentic/request_metrics.py b/inferencex-e2e/infx/results/agentic/request_metrics.py index 0586e4184a..7d032b4e7a 100644 --- a/inferencex-e2e/infx/results/agentic/request_metrics.py +++ b/inferencex-e2e/infx/results/agentic/request_metrics.py @@ -282,7 +282,6 @@ def _aiperf_percent_metric_as_rate( def compute_cache_stats( - records: list[dict[str, Any]], # noqa: ARG001 aggregate: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any]]: """Return flat and nested theoretical prefix cache hit rate from the aiperf ``aggregate``.""" @@ -311,7 +310,7 @@ def compute_request_metrics( qps_flat, qps_nested = compute_qps_stats(records) latency_flat, latency_nested = compute_latency_stats(records) workload_flat, workload_nested = compute_workload_stats(records, traces) - cache_flat, cache_nested = compute_cache_stats(records, aggregate) + cache_flat, cache_nested = compute_cache_stats(aggregate) throughput_flat, throughput_nested = compute_throughput_stats(records) for part in (qps_flat, latency_flat, workload_flat, cache_flat, throughput_flat): diff --git a/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py b/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py index 203b88236f..2eacb4ed17 100644 --- a/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py +++ b/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py @@ -142,7 +142,7 @@ def test_metrics_retokenize_a_success_without_reported_usage(): requests = [('p', 5, 4, None), ('p', 5, 7, None)] metrics, output_lens = client.calculate_metrics( - requests, outputs, 2.0, WordTokenizer(), ['tpot'], [50], {}, + requests, outputs, 2.0, WordTokenizer(), [50], {}, ) assert output_lens == [4, 7] diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index dc23c7c0ec..aaa22cf4b2 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -30,7 +30,7 @@ import sys import time from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from infx.config import git_repository_root, project_root @@ -70,7 +70,7 @@ class GithubException(Exception): # type: ignore[no-redef] # noqa: N818 status = 0 data: object = None - headers: dict = {} # type: ignore[type-arg] # noqa: RUF012 + headers: ClassVar[dict] = {} # type: ignore[type-arg] class RateLimitExceededException(GithubException): # type: ignore[no-redef] """Stub when PyGithub is not installed.""" From f0796a5b6bda04b02b47e180edf6e69710d87e59 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:42:14 -0500 Subject: [PATCH 201/203] refactor(yaml): cache PyYAML parses instead of routing through libyaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The libyaml path needed guards and adversarial testing to match PyYAML on every input, which is a lot of code for a parser speedup. yamlio now parses with PyYAML itself and caches each text up to 128 KiB per loader, handing out deep copies. full-sweep takes 2.9 s instead of 1.9 s with libyaml (18 s uncached), plan 1.5 s instead of 0.45 s, and outputs are byte-identical. 中文:libyaml 路径需要大量守卫和对抗测试才能在所有输入上与 PyYAML 一致,为了解析加速 付出的代码量过大。yamlio 现在直接使用 PyYAML 解析,并按加载器缓存每个不超过 128 KiB 的文本,返回深拷贝。full-sweep 耗时 2.9 秒(libyaml 为 1.9 秒,无缓存为 18 秒),plan 耗时 1.5 秒(libyaml 为 0.45 秒),输出逐字节一致。 --- inferencex-e2e/infx/tests/test_yamlio.py | 174 ++--------------- inferencex-e2e/infx/yamlio.py | 227 ++--------------------- 2 files changed, 27 insertions(+), 374 deletions(-) diff --git a/inferencex-e2e/infx/tests/test_yamlio.py b/inferencex-e2e/infx/tests/test_yamlio.py index 64a336ddd4..5cc685a80f 100644 --- a/inferencex-e2e/infx/tests/test_yamlio.py +++ b/inferencex-e2e/infx/tests/test_yamlio.py @@ -1,50 +1,9 @@ -import io -import sys - import pytest import yaml from infx import yamlio -@pytest.mark.parametrize( - ("text", "message"), - [ - ("a:\n b\tc: 1\n", "that cannot start any token"), - ("a: {b: c?d}\n", "expected ',' or '}', but got '?'"), - ("a: 1\n\ufeff\nb: 2\n", "could not find expected ':'"), - ("a: |#\n x\n", "expected chomping or indentation indicators"), - ("[!!str, a]\n", "could not determine a constructor for the tag"), - ("[?]]\n", "expected '', but found ']'"), - ], - ids=[ - "tab-in-plain-scalar", - "question-mark-in-flow-scalar", - "inner-byte-order-mark", - "comment-on-block-scalar-header", - "tag-before-flow-comma", - "token-after-empty-flow-key", - ], -) -def test_documents_only_libyaml_accepts_raise_pyyamls_error(text, message): - with pytest.raises(yaml.YAMLError, match=message): - yamlio.safe_load(text) - - -def test_a_tagged_empty_flow_key_is_null_as_in_pyyaml(): - assert yamlio.safe_load("{a: 1, ! }\n") == {"a": 1, None: None} - - -@pytest.mark.parametrize( - "text", - ["[" * 2000 + "]" * 2000, "#\r" + "- " * 600 + "x\r"], - ids=["flow", "block-after-carriage-returns"], -) -def test_nesting_past_pyyamls_recursion_limit_raises_its_recursion_error(text): - with pytest.raises(RecursionError): - yamlio.safe_load(text) - - def test_stream_errors_name_the_file_with_pyyamls_wording(tmp_path): path = tmp_path / "bad.yaml" path.write_text("a: [1\n") @@ -58,133 +17,23 @@ def test_stream_errors_name_the_file_with_pyyamls_wording(tmp_path): ) -def test_a_file_that_fails_to_decode_late_raises_pyyamls_earlier_syntax_error(tmp_path): - path = tmp_path / "bad.yaml" - path.write_bytes(b"a: b: c\n" + b"# padding\n" * 2000 + b"x: \xff\n") - - with path.open(encoding="utf-8") as stream, pytest.raises(yaml.YAMLError) as error: - yamlio.safe_load(stream) - - assert "mapping values are not allowed here" in str(error.value) - - -def test_a_file_iterated_before_loading_loads_its_remaining_lines(tmp_path): - path = tmp_path / "x.yaml" - path.write_text("skip: 1\na: 1\n") - - with path.open() as stream: - next(stream) - assert yamlio.safe_load(stream) == {"a": 1} - - -def test_a_closed_file_raises_reads_closed_file_error(tmp_path): - path = tmp_path / "x.yaml" - path.write_text("a: 1\n") - with path.open() as stream: - pass - - with pytest.raises(ValueError, match=r"^I/O operation on closed file\.$"): - yamlio.safe_load(stream) - - -def test_text_streams_that_need_a_read_size_load(): - class Chunks(io.TextIOBase): - def __init__(self) -> None: - self.left = "a: 1\n" - - def read(self, size: int) -> str: - chunk, self.left = self.left[:size], self.left[size:] - return chunk - - assert yamlio.safe_load(Chunks()) == {"a": 1} - - -def test_str_subclasses_load_like_str(): - class Text(str): - __slots__ = () +def test_a_text_that_fails_to_parse_raises_on_every_load(): + for _ in range(2): + with pytest.raises(yaml.YAMLError, match="expected ',' or ']'"): + yamlio.safe_load("a: [1\n") - assert yamlio.safe_load(Text("a: 1\n")) == {"a": 1} - - -def test_a_metaclass_table_property_is_ignored_as_pyyaml_ignores_it(): - class Meta(type): - @property - def yaml_constructors(cls): - return { - **yaml.SafeLoader.yaml_constructors, - "tag:yaml.org,2002:int": lambda _, node: f"META:{node.value}", - } - - class Loader(yaml.SafeLoader, metaclass=Meta): - pass - - assert yamlio.load("n: 1\n", Loader) == {"n": 1} - - -def test_a_replaced_constructor_that_refuses_comparison_applies(): - class Fussy: - __hash__ = object.__hash__ - - def __init__(self, label): - self.label = label - - def __eq__(self, other): - raise TypeError("ambiguous comparison") - - def __call__(self, _, node): - return f"{self.label}:{node.value}" - - class Loader(yaml.SafeLoader): - pass - - Loader.add_constructor("tag:yaml.org,2002:int", Fussy("OLD")) - first = yamlio.load("n: 1\n", Loader) - Loader.add_constructor("tag:yaml.org,2002:int", Fussy("NEW")) - - assert (first, yamlio.load("n: 1\n", Loader)) == ({"n": "OLD:1"}, {"n": "NEW:1"}) - - -def test_methods_patched_onto_a_loader_after_a_load_apply_to_the_same_text(): - class Loader(yaml.SafeLoader): - pass - - def construct_mapping(self, node, deep=False): - return {**yaml.SafeLoader.construct_mapping(self, node, deep), "patched": True} - - first = yamlio.load("a: 1\n", Loader) - Loader.construct_mapping = construct_mapping - - assert (first, yamlio.load("a: 1\n", Loader)) == ({"a": 1}, {"a": 1, "patched": True}) - - -def test_constructors_added_after_a_load_apply_to_the_same_text(): - class Loader(yaml.SafeLoader): - pass - - first = yamlio.load("n: 12\n", Loader) - Loader.add_constructor("tag:yaml.org,2002:int", lambda _, node: f"INT {node.value}") - - assert (first, yamlio.load("n: 12\n", Loader)) == ({"n": 12}, {"n": "INT 12"}) +def test_custom_loader_constructors_apply(): + with pytest.raises(yaml.YAMLError, match="found duplicate key 'a'"): + yamlio.load("x: 1\ny: {a: 1, a: 2}\n", yamlio.UniqueKeyLoader) -def test_the_int_digit_limit_applies_to_a_text_loaded_under_a_higher_limit(): - text = "n: " + "1" * 4400 + "\n" - limit = sys.get_int_max_str_digits() - sys.set_int_max_str_digits(5000) - try: - yamlio.safe_load(text) - finally: - sys.set_int_max_str_digits(4300) - try: - with pytest.raises(ValueError, match="Exceeds the limit"): - yamlio.safe_load(text) - finally: - sys.set_int_max_str_digits(limit) +def test_one_text_loads_through_each_loader_separately(): + text = "a: 1\na: 2\n" -def test_custom_loader_constructors_apply(): + assert yamlio.safe_load(text) == {"a": 2} with pytest.raises(yaml.YAMLError, match="found duplicate key 'a'"): - yamlio.load("x: 1\ny: {a: 1, a: 2}\n", yamlio.UniqueKeyLoader) + yamlio.load(text, yamlio.UniqueKeyLoader) def test_safe_unique_key_loader_merges_and_rejects_repeated_own_keys(): @@ -236,6 +85,7 @@ def test_repeated_loads_keep_anchor_sharing(): def test_a_deep_alias_chain_loads_as_pyyaml_builds_it(): chain = "".join(f"k{i}: &a{i} [{f'*a{i - 1}' if i else 'x'}]\n" for i in range(1200)) text = "first: null\n" + chain + "first: *a1199\n" + yamlio.safe_load(text) loaded = yamlio.safe_load(text) diff --git a/inferencex-e2e/infx/yamlio.py b/inferencex-e2e/infx/yamlio.py index 10d8750f06..bd8997aae8 100644 --- a/inferencex-e2e/infx/yamlio.py +++ b/inferencex-e2e/infx/yamlio.py @@ -1,12 +1,8 @@ -"""PyYAML loading through libyaml, with PyYAML's own results and errors.""" +"""PyYAML loading that parses each repeated text once per process.""" from __future__ import annotations import functools -import io -import os -import re -import sys from collections.abc import Iterator from typing import IO, Any @@ -16,228 +12,35 @@ from infx import fastcopy -_C_SAFE_LOADER = ( - None if os.environ.get("INFX_PURE_YAML") == "1" else getattr(yaml, "CSafeLoader", None) -) _MERGE_TAG = "tag:yaml.org,2002:merge" _VALUE_TAG = "tag:yaml.org,2002:value" -_TABLES = ( - "yaml_constructors", - "yaml_multi_constructors", - "yaml_implicit_resolvers", - "yaml_path_resolvers", -) -_CLASS_ATTRIBUTES = { - *_TABLES, - "__module__", - "__qualname__", - "__doc__", - "__firstlineno__", - "__static_attributes__", -} -_DIVERGENT = re.compile(r"(?m)^%|[\t\ufeff\r\x85\u2028\u2029]|[|>][-+0-9]*#") -_MAX_TOKEN_DEPTH = 50 -_OPENING_TOKENS = ( - yaml.BlockMappingStartToken, - yaml.BlockSequenceStartToken, - yaml.FlowMappingStartToken, - yaml.FlowSequenceStartToken, -) -_CLOSING_TOKENS = (yaml.BlockEndToken, yaml.FlowMappingEndToken, yaml.FlowSequenceEndToken) -_FLOW_TOKENS = { - yaml.FlowMappingStartToken: 1, - yaml.FlowSequenceStartToken: 1, - yaml.FlowMappingEndToken: -1, - yaml.FlowSequenceEndToken: -1, -} _CACHED_TEXT_CHARACTERS = 128 * 1024 -_PYYAML = object() - - -_TWINS: dict[type, tuple[type, tuple[type, ...], tuple[dict[str, Any], ...], tuple[Any, ...]]] = {} -_AFTER_EMPTY_FLOW_KEY = (yaml.ValueToken, yaml.FlowEntryToken, yaml.FlowSequenceEndToken) - - -def _class_state(classes: tuple[type, ...]) -> tuple[dict[str, Any], ...]: - """Return a snapshot of each class's own attributes in ``classes``, lookup tables excluded.""" - return tuple( - {key: value for key, value in dict(vars(cls)).items() if key not in _TABLES} - for cls in classes - ) - - -_PYYAML_CLASSES = tuple( - dict.fromkeys( - cls - for loader in (yaml.SafeLoader, _C_SAFE_LOADER) - if loader is not None - for cls in loader.__mro__ - if cls is not object - ) -) -_PYYAML_STATE = _class_state(_PYYAML_CLASSES) - - -def _table_state(loader: type) -> tuple[dict[Any, Any], ...]: - """Return copies of the constructor and resolver tables of ``loader``.""" - return tuple( - {key: list(value) if isinstance(value, list) else value for key, value in table.items()} - for table in (dict(getattr(loader, name)) for name in _TABLES) - ) - - -def _fast_loader(loader: type) -> type | None: - """libyaml twin of a SafeLoader subclass that only customizes its lookup tables. - - The twin parses with a copy of the tables and is rebuilt, with the parse cache cleared, - once ``add_constructor`` and friends change them or an attribute of any class involved - changes. Patched PyYAML classes, metaclasses and attributes that cannot be compared leave - every load to PyYAML. - """ - if _C_SAFE_LOADER is None or type(loader) is not type: - return None - if not issubclass(loader, yaml.SafeLoader): - return None - known = _TWINS.get(loader) - mro = loader.__mro__ - try: - if known is not None: - twin, classes, attributes, tables = known - if all( - saved == vars(cls) for saved, cls in zip(attributes, classes, strict=True) - ) and all( - saved == getattr(loader, name) for saved, name in zip(tables, _TABLES, strict=True) - ): - return twin - if ( - any(set(vars(cls)) - _CLASS_ATTRIBUTES for cls in mro[: mro.index(yaml.SafeLoader)]) - or _class_state(_PYYAML_CLASSES) != _PYYAML_STATE - ): - return None - classes = tuple(dict.fromkeys((*mro[:-1], *_PYYAML_CLASSES))) - attributes = tuple(dict(vars(cls)) for cls in classes) - tables = _table_state(loader) - except Exception: # noqa: BLE001 - return None - if known is not None: - _cached_libyaml_load.cache_clear() - twin = type( - f"Fast{loader.__name__}", (_C_SAFE_LOADER,), dict(zip(_TABLES, tables, strict=True)) - ) - _TWINS[loader] = (twin, classes, attributes, tables) - return twin - - -def _may_nest_deeply(text: str) -> bool: - """Whether the token depth can exceed the limit: flow nesting is at most the number of - openers, and block nesting at most one more than the longest run of spaces and block - indicators that starts a line.""" - room = _MAX_TOKEN_DEPTH - 1 - text.count("[") - text.count("{") - return room < 0 or re.search(rf"(?m)^[ ?:-]{{{room + 1},}}", text) is not None - - -def _needs_pyyaml(text: str, fast: type) -> bool: - """Whether the document has a tag, which libyaml and PyYAML scan differently, nesting too - deep for libyaml's composer, a plain scalar inside a flow collection holding ``?``, which - ends it in PyYAML, or a token after an empty explicit flow key, which libyaml skips. - - Past 50 levels PyYAML nears its RecursionError while libyaml's C composer may overflow.""" - if "?" not in text and "!" not in text and not _may_nest_deeply(text): - return False - depth = flow = 0 - previous: type | None = None - for token in yaml.scan(text, Loader=fast): - kind = type(token) - if kind in _OPENING_TOKENS: - depth += 1 - if depth > _MAX_TOKEN_DEPTH: - return True - elif kind in _CLOSING_TOKENS: - depth -= 1 - if ( - kind is yaml.TagToken - or (flow and kind is yaml.ScalarToken and token.plain and "?" in token.value) - or (flow and previous is yaml.KeyToken and kind in _AFTER_EMPTY_FLOW_KEY) - ): - return True - flow += _FLOW_TOKENS.get(kind, 0) - previous = kind - return False - - -def _libyaml_load(text: str, fast: type) -> Any: - """libyaml's result for ``text``, or ``_PYYAML`` where PyYAML has to decide. - - Directives, tabs, byte order marks, a comment glued to a block scalar header, characters - PyYAML's reader refuses, and line breaks other than newline all go to PyYAML.""" - if _DIVERGENT.search(text) or yaml.reader.Reader.NON_PRINTABLE.search(text): - return _PYYAML - try: - if _needs_pyyaml(text, fast): - return _PYYAML - return yaml.load(text, Loader=fast) # noqa: S506 - except Exception: # noqa: BLE001 - return _PYYAML @functools.lru_cache(maxsize=512) -def _cached_libyaml_load(text: str, fast: type, int_digits: int) -> Any: # noqa: ARG001 - """Return the libyaml result for ``text``, cached per loader and integer digit limit.""" - return _libyaml_load(text, fast) +def _parse(text: str, loader: type) -> Any: + """Return ``yaml.load(text, Loader=loader)``, parsed once per text and loader.""" + return yaml.load(text, Loader=loader) # noqa: S506 -def _fast_value(text: str, fast: type) -> Any: - """Return a fresh copy of the libyaml result for ``text``, or ``_PYYAML`` for PyYAML to decide. +def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: + """Return what ``yaml.load(stream, Loader=loader)`` returns or raises. - Only texts up to 128 KiB are cached, which covers the largest srt recipe at 110 KiB. Alias - chains too deep to copy go to PyYAML.""" - if len(text) > _CACHED_TEXT_CHARACTERS: - return _libyaml_load(text, fast) - value = _cached_libyaml_load(text, fast, sys.get_int_max_str_digits()) - if value is _PYYAML: - return _PYYAML + A ``str`` of at most 128 KiB is parsed once per process and loader, and every call gets a + fresh deep copy. Streams and larger texts are parsed on each call. Constructors added to a + loader after it parsed a text do not apply to that cached text. + """ + if type(stream) is not str or len(stream) > _CACHED_TEXT_CHARACTERS: + return yaml.load(stream, Loader=loader) # noqa: S506 + value = _parse(stream, loader) try: return fastcopy.deepcopy(value) except RecursionError: - return _PYYAML - - -def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: - """Return exactly what ``yaml.load(stream, Loader=loader)`` returns or raises, except - within a few frames of the interpreter's recursion limit, where either one can raise - RecursionError while the other returns. - - A ``str`` or a seekable ``io.TextIOWrapper``/``io.StringIO`` whose text libyaml accepts is - parsed by libyaml, which builds the same nodes as PyYAML; everything else, including every - error, goes to PyYAML from the stream's original position. Texts up to 128 KiB are parsed - once per process and returned as fresh deep copies. ``INFX_PURE_YAML=1`` turns libyaml off. - PyYAML internals patched before this module is imported, or mutated in place (such as - ``Scanner.ESCAPE_REPLACEMENTS``), are not detected. Each loader class is kept for the life - of the process. A stream whose ``tell()`` fails, as after iteration or closing, goes to PyYAML. - """ - fast = _fast_loader(loader) - if fast is not None and type(stream) is str: - value = _fast_value(stream, fast) - if value is not _PYYAML: - return value - elif fast is not None and type(stream) in (io.TextIOWrapper, io.StringIO): - try: - start = stream.tell() if stream.seekable() else None - except (OSError, ValueError): - start = None - if start is not None: - try: - value = _fast_value(stream.read(), fast) - except (OSError, ValueError): - value = _PYYAML - if value is not _PYYAML: - return value - stream.seek(start) - return yaml.load(stream, Loader=loader) # noqa: S506 + return yaml.load(stream, Loader=loader) # noqa: S506 def safe_load(stream: str | IO[str]) -> Any: - """Drop-in for ``yaml.safe_load`` on text.""" + """Return what ``yaml.safe_load(stream)`` returns, using the same cache as ``load``.""" return load(stream, yaml.SafeLoader) From 9388098acb57c48b6007746519fc3d223facaf85 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:59:25 -0500 Subject: [PATCH 202/203] chore(ci): remove comments from workflow files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop explanatory comments from GitHub Actions workflows. Keep zizmor, shellcheck and actionlint directives, action version pins and the permission comments zizmor's undocumented-permissions audit requires. 中文:删除 GitHub Actions 工作流中的说明性注释。保留 zizmor、shellcheck 和 actionlint 指令、action 版本标注,以及 zizmor undocumented-permissions 审计要求的权限注释。 --- .../workflows/benchmark-multinode-tmpl.yml | 22 ---------------- .github/workflows/benchmark-tmpl.yml | 12 --------- .github/workflows/ci.yml | 1 - .github/workflows/claude.yml | 2 -- .../workflows/codeowner-signoff-verify.yml | 10 ------- .github/workflows/collect-job-events.yml | 1 - .github/workflows/e2e-tests.yml | 1 - .github/workflows/klaud-candidate.yml | 7 ----- .github/workflows/klaud-plan.yml | 21 --------------- .github/workflows/merge-ingest.yml | 4 --- .github/workflows/operatorx-sweep.yml | 7 ----- .github/workflows/pr-infx-diff-comment.yml | 4 --- .github/workflows/profile.yml | 6 ----- .github/workflows/recover-reused-ingest.yml | 1 - .github/workflows/reuse-sweep-comment.yml | 2 +- .github/workflows/run-sweep.yml | 18 ------------- .github/workflows/runner-offline-digest.yml | 14 +--------- .github/workflows/speedbench-al.yml | 26 ------------------- .github/workflows/stage-results.yml | 1 - .github/workflows/trusted-external-sweep.yml | 6 ----- 20 files changed, 2 insertions(+), 164 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 8f5d86f7b6..9f61b9ecd7 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -73,7 +73,6 @@ on: type: string required: false default: '' - # Agentic-coding inputs. Fixed-seq-len jobs leave these empty. scenario-type: description: "Scenario type (fixed-seq-len or agentic-coding)" type: string @@ -118,10 +117,6 @@ env: SALLOC_TIME_LIMIT: '480' IS_MULTINODE: 'true' RANDOM_RANGE_RATIO: 0.8 - # Day-zero models resolved via hf: ids download from the Hub inside the - # slurm job (srtctl pre-download + dynamo hub fetch). Anonymous requests - # get 429-rate-limited when several workers pull a 444 GB snapshot at - # once; sbatch/srun inherit this env so the token reaches the workers. HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} EXP_NAME: ${{ fromJSON(inputs.config).exp-name }} RECIPE_FINGERPRINT: ${{ fromJSON(inputs.config)['recipe-fingerprint'] || '' }} @@ -150,7 +145,6 @@ env: POWER: ${{ fromJSON(inputs.config).power && '1' || '0' }} REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }} POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }} - # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} CONC: ${{ inputs.conc }} @@ -214,9 +208,6 @@ jobs: ) || format('[{0}]', toJSON(inputs.runner)) ) }} - # Full-context AgentX warmup can legitimately exceed the fixed-sequence - # eight-hour envelope at high concurrency. Keep one hour beyond the - # recipe's 12-hour Slurm limit for checkout, staging, and artifact upload. timeout-minutes: ${{ inputs.scenario-type == 'agentic-coding' && 780 || 480 }} name: >- ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }} @@ -241,8 +232,6 @@ jobs: done fi - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -267,7 +256,6 @@ jobs: with: ref: ${{ github.workflow_sha }} path: .result-tooling - # Only inferencex-e2e/ is read from this checkout. sparse-checkout: inferencex-e2e persist-credentials: false @@ -316,7 +304,6 @@ jobs: set -x RESULT_FILENAME=$(python3 -m infx.results.result_filename) export RESULT_FILENAME - # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" echo "EVAL_ARTIFACT_RECIPE=${RECIPE_FINGERPRINT:0:16}" >> "$GITHUB_ENV" eval_artifact_conc="$(python3 -c 'import hashlib,os; print(hashlib.sha256(os.environ["CONC_LIST"].encode()).hexdigest()[:12])')" @@ -324,7 +311,6 @@ jobs: source benchmarks/runtime_settings.sh source benchmarks/multi_node/runtime_settings.sh - # Assign data literally; recipe values must never become shell code. settings_json=$(jq -cen \ --argjson prefill "$PREFILL_ADDITIONAL_SETTINGS" \ --argjson decode "$DECODE_ADDITIONAL_SETTINGS" \ @@ -334,7 +320,6 @@ jobs: while IFS= read -r -d '' setting; do export "$setting" done < <(jq -j '.[] + "\u0000"' <<< "$settings_json") - # Resolve the workflow's documented automatic eval-concurrency selection. if [[ -z "$EVAL_CONC" ]]; then EVAL_CONC=$(python3 -c 'import os; print(max(map(int, os.environ["CONC_LIST"].split())))') fi @@ -343,7 +328,6 @@ jobs: "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run if [ "${EVAL_ONLY}" = "true" ]; then echo "Eval-only mode: skipping benchmark result file check" - # Verify eval produced results if ! ls results*.json 1>/dev/null 2>&1; then echo "Eval-only run failed: no results*.json files found." >&2 exit 1 @@ -357,9 +341,6 @@ jobs: echo "Run failed: expected $expected_count agentic results, found ${#agentic_results[@]}." >&2 exit 1 fi - # Existence is not enough: the agentic aggregation step writes the - # aggregate even when aiperf recorded zero valid requests. Require - # successful requests in every concurrency result. for result_file in "${agentic_results[@]}"; do echo "Found agentic result file: $result_file" ok=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(int(bool(d.get('num_requests_successful'))))" "$result_file" 2>/dev/null || echo 0) @@ -386,8 +367,6 @@ jobs: if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi source "$PYTHONPATH/benchmarks/check_env.sh" check_env_vars INFERENCEX_RESULTS_PYTHON - # Launcher stamp supplies the producer SHA unless the - # power-producer-sha input is set as a manual override if [ -z "$POWER_PRODUCER_SHA" ] && [ -f power-producer-sha.txt ]; then export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" @@ -451,7 +430,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: multinode_server_logs_${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }} - # multinode launchers package server logs into this tarball. path: | ${{ env.INFERENCEX_E2E_ROOT }}/multinode_server_logs.tar.gz ${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index ef25ec69ed..adb39e1962 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -54,7 +54,6 @@ on: description: "Git ref (branch/sha) to checkout" required: false type: string - # Agentic-coding inputs. Fixed-seq-len jobs leave these empty. scenario-type: description: "Scenario type (fixed-seq-len or agentic-coding)" required: false @@ -127,7 +126,6 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} EVAL_SUITE: ${{ inputs.eval-suite }} - # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} KV_OFFLOADING: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offloading) || '' }} @@ -202,8 +200,6 @@ jobs: done fi - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -228,7 +224,6 @@ jobs: with: ref: ${{ github.workflow_sha }} path: .result-tooling - # Only inferencex-e2e/ is read from this checkout. sparse-checkout: inferencex-e2e persist-credentials: false @@ -268,8 +263,6 @@ jobs: RUNNER_TYPE: ${{ inputs.runner }} JOB_CHECK_RUN_ID: ${{ job.check_run_id }} RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_a${{ github.run_attempt }}_${{ runner.name }} - # Suppress per-job eval markdown from being appended to the step summary. - # We'll publish a single combined eval table in the collection job instead. GITHUB_STEP_SUMMARY: '' run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi @@ -280,7 +273,6 @@ jobs: export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE)) echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV" - # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" source benchmarks/runtime_settings.sh @@ -288,7 +280,6 @@ jobs: if [ "${EVAL_ONLY}" = "true" ]; then echo "Eval-only mode: skipping benchmark result file check" - # Verify eval produced results if ! ls results*.json 1>/dev/null 2>&1; then echo "Eval-only run failed: no results*.json files found." >&2 exit 1 @@ -335,7 +326,6 @@ jobs: echo "no raw result to process: $RESULT_FILENAME.json" >&2 exit 1 fi - # A native power job's launcher stamp names the telemetry producer if [ -f power-producer-sha.txt ]; then export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" fi @@ -463,7 +453,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.eval-only && 'eval_server_logs_' || 'server_logs_' }}${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }} - # fixed-seq writes server.log at root; agentic writes logs under results/. path: | ${{ env.INFERENCEX_E2E_ROOT }}/server.log ${{ env.INFERENCEX_E2E_ROOT }}/results/*.log @@ -580,7 +569,6 @@ jobs: run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi rm -f meta_env.json || true - # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true rm -f -- ./*_report.json || true rm -f -- ./*_results.jsonl || true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6771b527e1..f37b4d869a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,5 +70,4 @@ jobs: if: matrix.check == 'Tests' run: | python -c "import torch; assert torch.version.cuda is None and torch.version.hip is None" - # SRT is initialized for our connector tests; its upstream suite runs in its own CI. python -m pytest -c inferencex-e2e/pyproject.toml inferencex-e2e/infx/tests/ inferencex-e2e/utils/ inferencex-e2e/runners/ collectivex/tests/ operatorx/tests/ --ignore=inferencex-e2e/utils/srt-slurm -n 4 diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 8eed6e55bb..7fd5fb8d7c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -14,7 +14,6 @@ jobs: concurrency: group: pr-review-${{ github.event.pull_request.number }} cancel-in-progress: false - # Only review PRs opened by someone with write access. if: contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) permissions: contents: read @@ -42,7 +41,6 @@ jobs: uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234 with: path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} - # Repository review credential; the job gates eligible review requests. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] track_progress: true allowed_bots: '' diff --git a/.github/workflows/codeowner-signoff-verify.yml b/.github/workflows/codeowner-signoff-verify.yml index c91e33bea4..956deb92cb 100644 --- a/.github/workflows/codeowner-signoff-verify.yml +++ b/.github/workflows/codeowner-signoff-verify.yml @@ -39,8 +39,6 @@ jobs: concurrency: group: codeowner-signoff-pr-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr-number }} cancel-in-progress: false - # The verifier agent reads untrusted PR content, so this job's token is read-only. - # The separate publish job holds the write permissions. permissions: contents: read pull-requests: read # Read PR metadata, diff, and review comments. @@ -103,7 +101,6 @@ jobs: SIGNOFF_KIND: ${{ steps.resolve.outputs.signoff-kind }} SIGNOFF_FETCH_CMD: ${{ steps.resolve.outputs.signoff-fetch-cmd }} run: | - # These placeholders must remain literal for envsubst. # shellcheck disable=SC2016 envsubst '${REPO} ${PR_NUMBER} ${HEAD_SHA} ${SIGNOFF_AUTHOR} ${SIGNOFF_KIND} ${SIGNOFF_FETCH_CMD}' \ < .github/codeowner-signoff-verify-prompt.md \ @@ -124,13 +121,11 @@ jobs: if: steps.resolve.outputs.proceed == 'true' uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234 env: - # Read-only job token: the agent reads untrusted PR content and never publishes. GH_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ github.token }} BASH_DEFAULT_TIMEOUT_MS: "1800000" BASH_MAX_TIMEOUT_MS: "3600000" with: - # Repository verifier credential; trusted code checks writer authorization before this step. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] github_token: ${{ github.token }} path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} @@ -141,8 +136,6 @@ jobs: settings: | {"fastMode": true} - # Bash is limited to the read-only commands the verifier prompt uses. - # /proc is denied so file tools cannot read credentials from process environments. claude_args: | --model 'claude-opus-5-5' --allowedTools "Read,Write,Glob,Grep,WebFetch,mcp__github_ci__*,Bash(gh pr view:*),Bash(gh pr diff:*),Bash(gh api:*),Bash(gh run view:*),Bash(gh run list:*),Bash(gh run download:*),Bash(git log:*),Bash(git show:*),Bash(uv run --project inferencex-e2e --locked python -m infx.workflows.pareto_coverage:*),Bash(uv run --project inferencex-e2e --locked python -m infx.workflows.signoff_validate:*)" @@ -153,8 +146,6 @@ jobs: already rendered with this run's PR number, head SHA, and sign-off metadata. - # The publisher reads the verdict only when verification succeeded, so success - # here guarantees an uploaded verdict artifact for the publish job. - name: Record verification outcome id: outcome if: always() && steps.resolve.outputs.proceed == 'true' @@ -214,7 +205,6 @@ jobs: inferencex-e2e/pyproject.toml inferencex-e2e/uv.lock - # Any download failure fails this job rather than posting a false rejection. - name: Download verdict if: needs.verify.outputs.verification-succeeded == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/collect-job-events.yml b/.github/workflows/collect-job-events.yml index cbf4d7c5a2..3d1380e81e 100644 --- a/.github/workflows/collect-job-events.yml +++ b/.github/workflows/collect-job-events.yml @@ -25,7 +25,6 @@ jobs: ref: ${{ inputs.tooling-ref }} persist-credentials: false - # Job records are diagnostics: they never fail the run. - name: Download job events continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 891e8e74e9..d0aa6d2936 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -4,7 +4,6 @@ run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || gith permissions: contents: read -# Independent GPU dispatches queue through the priority scheduler; newer runs must not cancel them. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: diff --git a/.github/workflows/klaud-candidate.yml b/.github/workflows/klaud-candidate.yml index 5a13b7d9d7..96bb728068 100644 --- a/.github/workflows/klaud-candidate.yml +++ b/.github/workflows/klaud-candidate.yml @@ -16,7 +16,6 @@ on: candidate: type: string required: true - # A resume run continues the session of this planner run, as session number `session`. parent-run: type: string required: false @@ -36,7 +35,6 @@ jobs: name: Klaud Cold runs-on: ubuntu-latest env: - # Repair attempts after the initial update; the baseline is already published. MAX_REPAIRS: '5' KLAUD_BRANCH: klaud/auto-${{ inputs.candidate }} KLAUD_PARENT_RUN: ${{ inputs.parent-run || github.run_id }} @@ -94,9 +92,6 @@ jobs: rm -r "$RUNNER_TEMP/plan" outcome_schema=$(uv run --project "$INFERENCEX_TOOLING_PROJECT" --locked python -P -m infx.klaud outcome-schema) echo "outcome-schema=$outcome_schema" >> "$GITHUB_OUTPUT" - # A planner run's own session runs in its first attempt. A resume run that recovery - # dispatched runs only the session it granted; anything else skips the steps below. - # lifecycle.GATE_STEP and AGENT_STEP name these steps. - name: Check whether this session should run id: gate env: @@ -119,7 +114,6 @@ jobs: id: claude if: steps.gate.outputs.run == 'true' continue-on-error: true - # Stops a session that runs out before the 6-hour job limit, so the steps below still run. timeout-minutes: 330 uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234 env: @@ -173,7 +167,6 @@ jobs: if: always() && steps.gate.outputs.run == 'true' with: name: klaud-candidate-${{ inputs.candidate }} - # Never upload the execution transcript or private API/scratch data. path: ${{ runner.temp }}/candidate-diagnostics.json retention-days: 14 - name: Require a verified candidate outcome diff --git a/.github/workflows/klaud-plan.yml b/.github/workflows/klaud-plan.yml index 3547706571..0c2003a351 100644 --- a/.github/workflows/klaud-plan.yml +++ b/.github/workflows/klaud-plan.yml @@ -1,15 +1,11 @@ name: Klaud Cold auto-sweep -# lifecycle.resume_title matches the resume runs' names. run-name: >- ${{ inputs.resume-parent && format('Klaud Cold resume {0} {1} session {2}', inputs.resume-parent, inputs.resume-candidate, inputs.resume-session) || 'Klaud Cold auto-sweep' }} -# Overlapping waves are intentional; candidate ownership claims prevent duplicate work. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: - # Recovery dispatches a resume run with these to continue an unfinished session; a manual - # dispatch leaves them empty and runs a normal wave. inputs: resume-parent: description: Planner run that owns the session to resume @@ -32,13 +28,9 @@ permissions: pull-requests: read # Read PR metadata. env: - # Total candidates per invocation, resumed sessions included; they run in parallel. MAX_CANDIDATES_PER_RUN: '5' jobs: - # The overlap review reads untrusted open-PR content (including fork PRs). It runs in its own - # job with no write-capable credential; the credentialed jobs never share its workspace, runner - # temp directory or tool caches and only consume its validated structured output as data. recover: name: Reconcile interrupted candidates if: github.ref == 'refs/heads/main' && github.run_attempt == 1 && inputs.resume-parent == '' @@ -50,7 +42,6 @@ jobs: outputs: tooling-ref: ${{ steps.checkout.outputs.commit }} blocked-families: ${{ steps.recover.outputs.blocked-families }} - # Unfinished sessions recovery resumed in new runs; they share this wave's cap. resumed: ${{ steps.recover.outputs.resumed }} steps: - id: checkout @@ -87,8 +78,6 @@ jobs: - name: Reconcile interrupted owned candidates id: recover env: - # Recovery mutates only sessions recorded by this workflow after their parent finishes. - # Repository automation credential; the planner runs only on main. GH_TOKEN: ${{ secrets.AGENT_PAT }} # zizmor: ignore[secrets-outside-env] run: | uv run --project "$INFERENCEX_PYTHON_PROJECT" --locked python -m infx.klaud recover --max-resumes "$MAX_CANDIDATES_PER_RUN" @@ -127,7 +116,6 @@ jobs: echo "INFERENCEX_PYTHON_PROJECT=$python_project" >> "$GITHUB_ENV" - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: - # Never save a cache the agent's job could have poisoned for credentialed jobs to restore. enable-cache: false python-version: '3.12' - name: Restore recovery-blocked families @@ -138,7 +126,6 @@ jobs: id: prepare env: GH_TOKEN: ${{ github.token }} - # Read-only dashboard key, used before the review agent starts and absent from its step. KLAUD_DASHBOARD_API_KEY: ${{ secrets.DASH_API_KEY }} # zizmor: ignore[secrets-outside-env] run: >- uv run --project "$INFERENCEX_PYTHON_PROJECT" --locked python -m infx.klaud plan @@ -146,7 +133,6 @@ jobs: --review-batch-size "$REVIEW_BATCH_SIZE" --cooldown-hours "$CANDIDATE_COOLDOWN_HOURS" --stale-pr-days "$STALE_PR_DAYS" - # Uploaded before the agent runs, so selection reads the planner's files, not agent-touched ones. - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: klaud-review-input @@ -178,7 +164,6 @@ jobs: KLAUD_EVIDENCE: ${{ runner.temp }}/klaud with: path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} - # Repository automation credential; the planner runs only on main. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] github_token: ${{ github.token }} track_progress: false @@ -233,7 +218,6 @@ jobs: if: always() && steps.review.outputs.execution_file != '' env: EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} - # Keep only the final result fields selection diagnostics read; never transcript content. run: | execution=$(jq -c ' [.[] | objects | select(.type == "result")] | last @@ -295,9 +279,7 @@ jobs: - name: Select reviewed candidates within the total cap id: select env: - # Repository automation credential; the planner runs only on main. GH_TOKEN: ${{ secrets.AGENT_PAT }} # zizmor: ignore[secrets-outside-env] - # Repository automation credential; the planner runs only on main. KLAUD_DASHBOARD_API_KEY: ${{ secrets.DASH_API_KEY }} # zizmor: ignore[secrets-outside-env] KLAUD_PR_REVIEW: ${{ needs.review.outputs.review }} KLAUD_REVIEW_OUTCOME: ${{ needs.review.outputs.outcome }} @@ -313,7 +295,6 @@ jobs: if: always() with: name: klaud-plan - # Deliberately exclude arbitrary agent scratch files. path: | ${{ runner.temp }}/klaud/candidates.json ${{ runner.temp }}/klaud/open-prs.json @@ -340,8 +321,6 @@ jobs: AGENT_PAT: ${{ secrets.AGENT_PAT }} DASH_API_KEY: ${{ secrets.DASH_API_KEY }} - # A resume run continues one unfinished session. Only recovery, as Klaud-Cold, starts one, and - # klaud-candidate.yml's gate starts its agent only with recovery's grant for that session. resume: name: Run candidate ${{ inputs.resume-candidate }} if: >- diff --git a/.github/workflows/merge-ingest.yml b/.github/workflows/merge-ingest.yml index 04a21de17c..25b47294e7 100644 --- a/.github/workflows/merge-ingest.yml +++ b/.github/workflows/merge-ingest.yml @@ -1,9 +1,6 @@ name: "Merge Ingest" run-name: Merge Ingest - ${{ github.event.head_commit.message }} -# Publishes a merged PR's reused run-sweep.yml artifacts. Pushes to main never -# run benchmarks: without a valid reuse authorization this run fails. - concurrency: group: merge-ingest-${{ github.sha }} cancel-in-progress: false @@ -101,7 +98,6 @@ jobs: SOURCE_RUN_ID: ${{ steps.reuse.outputs.reuse-source-run-id }} MERGE_RUN_ID: ${{ github.run_id }} with: - # Repository integration credential for the scoped ingest job. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | const agentic = process.env.AGENTIC === 'true'; diff --git a/.github/workflows/operatorx-sweep.yml b/.github/workflows/operatorx-sweep.yml index 7f3a5b8527..bf3b768696 100644 --- a/.github/workflows/operatorx-sweep.yml +++ b/.github/workflows/operatorx-sweep.yml @@ -90,7 +90,6 @@ jobs: set -eo pipefail source inferencex-e2e/benchmarks/check_env.sh check_env_vars OPERATORX_RUNNER OPERATORX_BACKENDS OPERATORX_TESTLISTS OPERATORX_WORLD_SIZES OPERATORX_MODE OPERATORX_CHUNK_SIZE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA GITHUB_OUTPUT - # Cases take InferenceX's srt-slurm recipes, expanded by InferenceX's own code (srtctl). git submodule update --init inferencex-e2e/utils/srt-slurm uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml --with ruamel.yaml \ --with marshmallow --with marshmallow-dataclass --with pydantic --with jinja2 --with requests --with rich \ @@ -123,8 +122,6 @@ jobs: timeout-minutes: 70 strategy: fail-fast: false - # Keep every shard eligible: a second matrix cap can strand priority labels - # on GitHub-held jobs. Runner/node admission is owned by the scheduler. matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} runs-on: >- # zizmor: ignore[self-hosted-runner] GPU Slurm runners by design; this job runs only on workflow_dispatch ${{ fromJSON(vars.PRIORITY_SCHEDULER_ENABLED == 'true' && @@ -141,8 +138,6 @@ jobs: PYTHONDONTWRITEBYTECODE: '1' OPERATORX_CLEANUP_SECONDS: '180' steps: - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -259,7 +254,6 @@ jobs: trigger-ingest: name: Trigger OperatorX database ingest - # A run with failed shards is still stored: its unrun cases read as missing. if: >- always() && github.event_name == 'workflow_dispatch' && inputs.ingest && (needs.summarize.result == 'success' || needs.summarize.result == 'failure') @@ -271,7 +265,6 @@ jobs: env: RUN_ID: ${{ github.run_id }} with: - # Cross-repository dispatch credential, as run-sweep.yml's database ingest uses. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | await github.rest.repos.createDispatchEvent({ diff --git a/.github/workflows/pr-infx-diff-comment.yml b/.github/workflows/pr-infx-diff-comment.yml index b68239cfab..7c1dcfe166 100644 --- a/.github/workflows/pr-infx-diff-comment.yml +++ b/.github/workflows/pr-infx-diff-comment.yml @@ -1,15 +1,11 @@ name: PR infx diff -# synchronize covers pushes to PR branches, including branches in forks. -# Use API data only: never check out or execute code from the PR. on: - # Control plane only: reads PR metadata through the API, never executes PR code. pull_request_target: # zizmor: ignore[dangerous-triggers] types: [opened, reopened, ready_for_review, synchronize, edited] permissions: {} -# Serialize comment writes; each run reads the current PR rather than old event data. concurrency: group: pr-infx-diff-${{ github.event.pull_request.number }} cancel-in-progress: false diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 449413ecc1..40dfb30df1 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -1,8 +1,5 @@ name: Profile -# Runs one single-node matrix point through infx.launch. The infx launch path -# has no profiler capture, so this workflow produces no trace. -# Independent profiling requests use priority-scheduled runner slots. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: @@ -234,8 +231,6 @@ jobs: done fi - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - name: Checkout code id: checkout continue-on-error: true @@ -256,7 +251,6 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: *checkout-inputs - # With clean: false above, nested checkouts also persist between jobs. - name: Checkout result tooling id: checkout-tooling continue-on-error: true diff --git a/.github/workflows/recover-reused-ingest.yml b/.github/workflows/recover-reused-ingest.yml index a6225d44f6..fd71c202fb 100644 --- a/.github/workflows/recover-reused-ingest.yml +++ b/.github/workflows/recover-reused-ingest.yml @@ -29,7 +29,6 @@ jobs: SOURCE_RUN_ID: ${{ inputs.source-run-id }} MERGE_RUN_ID: ${{ inputs.merge-run-id }} with: - # Cross-repository dispatch credential for maintainer-triggered ingest recovery. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | await github.rest.repos.createDispatchEvent({ diff --git a/.github/workflows/reuse-sweep-comment.yml b/.github/workflows/reuse-sweep-comment.yml index 2aaad24cfe..9b5ff9b87e 100644 --- a/.github/workflows/reuse-sweep-comment.yml +++ b/.github/workflows/reuse-sweep-comment.yml @@ -31,7 +31,7 @@ jobs: - name: Checkout trusted workflow code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.sha }} # issue_comment uses the default-branch commit + ref: ${{ github.sha }} persist-credentials: false - name: Validate request and update reaction diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index f8caac9179..5df90087ef 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -39,7 +39,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - # Labels opt into sweeps regardless of draft status; readiness only starts reviews. if: >- github.event.pull_request.head.repo.full_name == github.repository && ( @@ -235,7 +234,6 @@ jobs: with: path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} github_token: ${{ github.token }} - # Repository integration credential for the scoped sweep/ingest job. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] track_progress: false settings: | @@ -342,9 +340,6 @@ jobs: | if $idx == null then . else del(.[$idx]) end end; - # Canary is a benchmark-only smoke test — exclude entries - # whose primary purpose is eval (run-eval == true) so the - # picked canary never runs an eval pass. (((.single_node["1k1k"] // []) + (.single_node["8k1k"] // []) + (.single_node.agentic // [])) | map(select(.["run-eval"] != true))) as $single_candidates | ((.multi_node.agentic // []) | map(select(.["run-eval"] != true))) as $multi_candidates @@ -385,7 +380,6 @@ jobs: with: config: ${{ toJSON(matrix.config) }} pr-number: &pr-number ${{ github.event.pull_request.number }} - # Only same-repository PRs owned by Klaud receive background priority. klaud-run: &klaud-run ${{ github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login == 'Klaud-Cold' && (startsWith(github.head_ref, 'klaud/auto-') || startsWith(github.head_ref, 'klaude/auto-')) }} runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} @@ -620,9 +614,6 @@ jobs: run-eval: true eval-only: true - # Agentic GSM8K eval rows carry the agentic input shape, so they are - # dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len - # inputs (isl/osl/max-model-len, which agentic rows don't have). sweep-agentic-evals: needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] if: >- @@ -691,12 +682,6 @@ jobs: eval-only: true eval-conc: ${{ (matrix.config['eval-framework'] || 'lm-eval') == 'lm-eval' && matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} - # Multi-node agentic GSM8K eval rows carry the agentic input shape, so - # they are dispatched with sweep-multi-node-agentic's inputs rather than - # sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, - # which agentic rows don't have). Agentic selection uses one highest - # eval-conc per topology; fixed-sequence --all-evals rows may instead pass - # a joined concurrency list to lm-eval. sweep-multi-node-agentic-evals: needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] if: >- @@ -802,7 +787,6 @@ jobs: upload-changelog-metadata: name: upload-changelog-metadata - # collect-evals makes an eval-only rerun upload a manifest for the current run attempt. needs: [setup, collect-results, collect-evals] if: ${{ always() && needs.setup.result == 'success' }} runs-on: ubuntu-latest @@ -827,7 +811,6 @@ jobs: Path('sweep_manifest.json').write_text(json.dumps({ 'head': os.environ['SWEEP_HEAD'], 'run-id': int(os.environ['GITHUB_RUN_ID']), 'run-attempt': int(os.environ['GITHUB_RUN_ATTEMPT']), - # Klaud validates only full sweeps that ran fail-fast. 'full-sweep': sweep_labels == {'full-sweep-fail-fast'}, 'matrix': matrix, })) @@ -915,7 +898,6 @@ jobs: runs-on: ubuntu-latest env: - # Repository integration credential for the scoped sweep/ingest job. DATABASE_URL: ${{ secrets.DB_RO_URL }} # zizmor: ignore[secrets-outside-env] steps: diff --git a/.github/workflows/runner-offline-digest.yml b/.github/workflows/runner-offline-digest.yml index 13563c6a68..5af58f8a31 100644 --- a/.github/workflows/runner-offline-digest.yml +++ b/.github/workflows/runner-offline-digest.yml @@ -1,18 +1,8 @@ name: Runner Offline Digest -# Daily check that lists self-hosted runners reporting `offline` and posts -# a summary + threaded details to Slack. No post when nothing is offline. -# -# Required secrets: -# RUNNERS_PAT — PAT (or GitHub App token) with repo "Administration: Read" -# so it can list self-hosted runners. The default GITHUB_TOKEN -# cannot read this endpoint. -# SLACK_BOT_TOKEN — Slack bot token with `chat:write`. The bot must be a member -# of channel C09PULGMVNG. - on: schedule: - - cron: '13 14 * * *' # daily 14:13 UTC ≈ 8:13 AM CST (9:13 AM CDT during DST) + - cron: '13 14 * * *' workflow_dispatch: permissions: @@ -30,7 +20,6 @@ jobs: - name: List offline runners id: runners env: - # Repository runner/Slack integration credential for the scheduled digest. GH_TOKEN: ${{ secrets.RUNNERS_PAT }} # zizmor: ignore[secrets-outside-env] REPO: ${{ github.repository }} run: | @@ -65,7 +54,6 @@ jobs: - name: Post Slack digest if: steps.runners.outputs.offline_count != '0' env: - # Repository runner/Slack integration credential for the scheduled digest. SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} # zizmor: ignore[secrets-outside-env] CHANNEL: C09PULGMVNG OFFLINE_COUNT: ${{ steps.runners.outputs.offline_count }} diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index eb7b4c6088..e036c0c98c 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -1,12 +1,5 @@ name: SpeedBench AL Collection -# Push-button (workflow_dispatch) collection of a SPEED-Bench acceptance-length -# (AL) matrix: thinking_on/off x MTP levels, for the given model (defaults to -# DeepSeek-V4-Pro). Produces the golden curve consumed by the -# synthetic-acceptance framework and (optionally) opens a PR writing it to -# infx/golden_al_distribution/.yaml. - -# Independent collection requests use priority-scheduled runner slots. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: @@ -37,7 +30,6 @@ on: # zizmor: ignore[concurrency-limits] type: string default: '' image: - # vLLM collectors need vLLM >= 0.30: thinking-on cells rely on --chat-template-kwargs (vllm-project/vllm#44244). description: "Container image (vLLM >= 0.30 for vLLM collectors)" required: false type: string @@ -114,7 +106,6 @@ env: DP_ATTENTION: 'false' SPEC_DECODING: mtp SALLOC_TIME_LIMIT: ${{ inputs.salloc-time }} - # Matrix-collector tunables (propagated into the container via srun --export=ALL). MTP_LIST: ${{ inputs.mtp-list }} THINKING_MODES: ${{ inputs.thinking-modes }} CATEGORY: ${{ inputs.category }} @@ -132,8 +123,6 @@ jobs: priority: ${{ steps.score.outputs.priority }} queue-token: ${{ steps.score.outputs.queue-token }} steps: - # Fail before the GPU run if open-pr has no curve to write. Same name rule as - # infx.golden_al_distribution.curves.CURVE_NAME, which keeps the path in that directory. - name: Validate golden curve target if: ${{ inputs.open-pr }} env: @@ -216,7 +205,6 @@ jobs: steps: - name: Resource cleanup (pre-run) run: &resource-cleanup | - # Cleanup SLURM resources if command -v squeue >/dev/null 2>&1; then for job_name in "${RUNNER_NAME}" "inferencex-${RUNNER_NAME}"; do echo "[Slurm] Cleaning up jobs with name: ${job_name} ..." @@ -228,13 +216,9 @@ jobs: done fi - # Cleanup AL-matrix outputs from a prior job on this runner so a stale - # matrix from a previous run is never picked up as this job's output. rm -rf "${GITHUB_WORKSPACE}/speedbench_results" \ "${GITHUB_WORKSPACE}/inferencex-e2e/speedbench_results" 2>/dev/null || true - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -270,7 +254,6 @@ jobs: if [[ -n "$COLLECTOR_VARIANT" ]]; then collector+="_${COLLECTOR_VARIANT}" fi - # The name must stay a plain file in the collector directory. if [[ "$collector" == */* || "$collector" == *$'\n'* || ! -f "$collector_dir/$collector.sh" ]]; then echo "::error::No SPEED-Bench collector $collector_dir/$collector.sh at this ref. Available collectors are listed below." >&2 ls "$collector_dir" >&2 @@ -283,7 +266,6 @@ jobs: with: ref: ${{ github.workflow_sha }} path: .launch-tooling - # Only inferencex-e2e/ is read from this checkout. sparse-checkout: inferencex-e2e persist-credentials: false @@ -341,7 +323,6 @@ jobs: - name: Open PR updating golden AL curve if: ${{ inputs.open-pr && success() }} env: - # Repository write credential for authorized reference-data collection. GH_TOKEN: ${{ secrets.SPEEDBENCH_PAT }} # zizmor: ignore[secrets-outside-env] GOLDEN_CURVE: ${{ inputs.golden-curve }} run: | @@ -351,14 +332,10 @@ jobs: check_env_vars GOLDEN_CURVE INFERENCEX_LAUNCH_PYTHON BENCH_SCRIPT_OVERRIDE run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" curve_file="infx/golden_al_distribution/${GOLDEN_CURVE}.yaml" - # The README requires the source run URL as the first line of a committed curve. - # The whole file is replaced, so cells this run did not collect are dropped. { echo "# Source GitHub Actions run: ${run_url}" cat speedbench-reference-al.yaml } > "$curve_file" - # Load every cell through the golden curve CLI. A failed cell (N/A) or a malformed - # matrix stops here instead of reaching a PR. PYTHONPATH="$PWD" "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.golden_al_distribution show "$GOLDEN_CURVE" BRANCH="speedbench-al/${MODEL_PREFIX}-auto-${GITHUB_RUN_ID}" @@ -386,9 +363,6 @@ jobs: path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench_results/server_*.log if-no-files-found: ignore - # Per-request benchmark detail (vllm bench serve --save-detailed): includes - # the model's response text per request, so the AL of each cell can be - # audited for output correctness (sensible text + correct thinking mode). - name: Upload detailed benchmark results if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index 9c86ca2c10..2d2db195dc 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -86,7 +86,6 @@ jobs: REQUESTED_BY: ${{ steps.request.outputs.requested-by }} COMMENT_ID: ${{ steps.acknowledge.outputs.comment-id }} with: - # Cross-repository dispatch credential; trusted control code validates maintainer authorization. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | await github.rest.repos.createDispatchEvent({ diff --git a/.github/workflows/trusted-external-sweep.yml b/.github/workflows/trusted-external-sweep.yml index 6543dace46..6af540f2b8 100644 --- a/.github/workflows/trusted-external-sweep.yml +++ b/.github/workflows/trusted-external-sweep.yml @@ -1,13 +1,7 @@ name: Trusted External Sweep Dispatch run-name: Trusted external sweep - PR #${{ github.event.pull_request.number }} @ ${{ github.event.pull_request.head.sha }} -# This workflow is intentionally control-plane only. It never checks out or -# executes pull-request code. A write-authorized collaborator applies one primary -# sweep label to approve the exact external head SHA, then this trusted event -# dispatches the existing e2e workflow from main with repository secrets. -# Each authorized label event approves an exact PR head; no event may cancel another. on: # zizmor: ignore[concurrency-limits] - # Control plane only: verifies writer permission and the exact labeled head, never executes PR code. pull_request_target: # zizmor: ignore[dangerous-triggers] branches: - main From 107eeabdb780e9e65d9ca2c1ff71ec10096e1580 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:36:57 -0500 Subject: [PATCH 203/203] revert(ci): leave profile.yml as main has it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit profile.yml is under active development elsewhere, so this branch no longer changes it or its launch test. 中文:profile.yml 正由他人开发,本分支不再修改该文件及其启动测试。 --- .github/workflows/profile.yml | 173 ++++++++++++++++-- .../tests/workflows/test_launch_layout.py | 7 +- 2 files changed, 165 insertions(+), 15 deletions(-) diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 40dfb30df1..ce9feb53bb 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -1,5 +1,6 @@ name: Profile +# Independent profiling requests use priority-scheduled runner slots. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: @@ -17,6 +18,11 @@ on: # zizmor: ignore[concurrency-limits] required: false type: string default: '64' + moe-debug: + description: "Enable MoE debug patch and log (MOE_DEBUG_LOG)" + required: false + type: boolean + default: false ref: description: "Ref (branch/sha) to checkout" required: false @@ -30,16 +36,20 @@ env: INFERENCEX_E2E_ROOT: ${{ github.workspace }} PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' + DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' + MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' SALLOC_TIME_LIMIT: '480' + KEEP_LOGS: '0' EVAL_ONLY: 'false' RUN_EVAL: 'false' IS_AGENTIC: '0' IS_MULTINODE: 'false' + SCENARIO_SUBDIR: fixed_seq_len/ HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} HF_HUB_CACHE: '/mnt/hf_hub_cache/' RANDOM_RANGE_RATIO: '0.8' - REQUIRE_POWER: '0' + PERFETTO_RELAY_URL: https://semianalysisai.github.io/InferenceX-trace-storage PYTHONDONTWRITEBYTECODE: '1' PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache @@ -98,10 +108,12 @@ jobs: python -P -m infx.workflows.ci_priority \ --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ --event-name "${GITHUB_EVENT_NAME}" \ - --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}") + --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}" \ + --criteria-json "${PRIORITY_CRITERIA}") echo "raw=$CONFIG_JSON" >> "$GITHUB_OUTPUT" env: + PRIORITY_CRITERIA: ${{ inputs.moe-debug && '["patchwork"]' || '' }} INPUTS_CONFIG_FILE: ${{ inputs.config-file }} INPUTS_CONFIG_KEY: ${{ inputs.config-key }} INPUTS_CONC: ${{ inputs.conc }} @@ -198,8 +210,6 @@ jobs: ) }} env: EXP_NAME: ${{ matrix.config.exp-name }} - SRT_RECIPE: ${{ matrix.config.srt-recipe }} - RECIPE_FINGERPRINT: ${{ matrix.config.recipe-fingerprint }} MODEL: ${{ matrix.config.model }} MODEL_PREFIX: ${{ matrix.config.model-prefix }} ISL: ${{ matrix.config.isl }} @@ -217,6 +227,8 @@ jobs: CONC: ${{ matrix.config.conc }} SPEC_DECODING: ${{ matrix.config.spec-decoding }} DISAGG: ${{ matrix.config.disagg }} + MOE_DEBUG: '0' + MOE_DEBUG_LOG: ${{ (inputs.moe-debug) && '/workspace/moe_debug.tp0.log' || '' }} steps: - name: Slurm cleanup run: | @@ -231,6 +243,8 @@ jobs: done fi + # A cancelled or crashed job can leave git state in this persistent + # workspace that breaks checkout; re-cloning fixes it and is idempotent. - name: Checkout code id: checkout continue-on-error: true @@ -239,7 +253,6 @@ jobs: fetch-depth: 0 ref: ${{ inputs.ref || github.sha }} clean: false - submodules: true persist-credentials: false - name: Wipe workspace after failed checkout @@ -251,6 +264,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: *checkout-inputs + # With clean: false above, nested checkouts also persist between jobs. - name: Checkout result tooling id: checkout-tooling continue-on-error: true @@ -299,31 +313,59 @@ jobs: run: | "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.launch cleanup - - name: Launch job script + - name: Launch + Profile (single-node sglang/vllm) + id: run env: RUNNER_NAME: ${{ runner.name }} - JOB_CHECK_RUN_ID: ${{ job.check_run_id }} - RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_a${{ github.run_attempt }}_${{ runner.name }} + PROFILE: '1' + SGLANG_TORCH_PROFILER_DIR: /workspace/ + VLLM_TORCH_PROFILER_DIR: /workspace/ + VLLM_RPC_TIMEOUT: '1800000' shell: bash run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi export GITHUB_WORKSPACE="$PWD" echo "INFERENCEX_E2E_ROOT=$PWD" >> "$GITHUB_ENV" set -eo pipefail - RESULT_FILENAME=$(python3 -m infx.results.result_filename) - export RESULT_FILENAME - echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE)) echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV" + ep_val="${EP_SIZE:-1}" + res_name="${EXP_NAME}_${PRECISION}_${FRAMEWORK}_tp${TP}-pp${PP_SIZE}-dcp${DCP_SIZE}-pcp${PCP_SIZE}_ep${ep_val}_dpa_${DP_ATTENTION}_conc${CONC}_${RUNNER_NAME}" + export RESULT_FILENAME="${res_name}" + echo "RESULT_FILENAME=${res_name}" >> "$GITHUB_ENV" source benchmarks/runtime_settings.sh "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run - if [ ! -f "${RESULT_FILENAME}.json" ]; then - echo "Run failed: Benchmark result ${RESULT_FILENAME}.json not found." >&2 + if [ ! -f "${res_name}.json" ]; then + echo "Run failed: Benchmark result ${res_name}.json not found." >&2 exit 1 fi + trace_path="profile_${res_name}.trace.json.gz" + if [ -f "$trace_path" ]; then + echo "trace=$PWD/$trace_path" >> "$GITHUB_OUTPUT" + if [ "${FRAMEWORK}" = "sglang" ]; then + # Try to locate corresponding TP-0 traces produced by SGLang profiler + merged_latest=$(ls -t profiles/merged-*.trace.json.gz 2>/dev/null | head -n1 || true) + if [ -n "${merged_latest}" ] && [ -f "${merged_latest}" ]; then + ts_name="${merged_latest##*/}" + ts_name="${ts_name#merged-}" + ts_name="${ts_name%.trace.json.gz}" + tp0_decode="profiles/${ts_name}-TP-0-DECODE.trace.json.gz" + tp0_extend="profiles/${ts_name}-TP-0-EXTEND.trace.json.gz" + if [ -f "${tp0_decode}" ]; then + echo "tp0_decode=$PWD/${tp0_decode}" >> "$GITHUB_OUTPUT" + fi + if [ -f "${tp0_extend}" ]; then + echo "tp0_extend=$PWD/${tp0_extend}" >> "$GITHUB_OUTPUT" + fi + fi + fi + else + echo "Profile trace not found: $trace_path" >&2 + fi + - name: Process result (json -> agg) env: RUNNER_TYPE: ${{ matrix.config.runner }} @@ -333,3 +375,108 @@ jobs: source "$PYTHONPATH/benchmarks/check_env.sh" check_env_vars INFERENCEX_RESULTS_PYTHON "$INFERENCEX_RESULTS_PYTHON" -P -m infx.results.fixed_sequence + + - name: Upload profile as artifact + if: ${{ steps.run.outputs.trace != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: profile_${{ env.RESULT_FILENAME }} + path: ${{ steps.run.outputs.trace }} + if-no-files-found: ignore + + - name: Upload TP-0-DECODE trace as artifact + if: ${{ steps.run.outputs.tp0_decode != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: profile_${{ env.RESULT_FILENAME }}_TP0_DECODE + path: ${{ steps.run.outputs.tp0_decode }} + if-no-files-found: ignore + + - name: Upload TP-0-EXTEND trace as artifact + if: ${{ steps.run.outputs.tp0_extend != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: profile_${{ env.RESULT_FILENAME }}_TP0_EXTEND + path: ${{ steps.run.outputs.tp0_extend }} + if-no-files-found: ignore + + - name: Upload MoE debug log as artifact + if: ${{ env.MOE_DEBUG == '1' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: moe_debug_${{ env.RESULT_FILENAME }} + path: ${{ env.INFERENCEX_E2E_ROOT }}/moe_debug.tp0.log + if-no-files-found: ignore + + - name: Checkout storage repo + id: checkout-storage + if: ${{ steps.run.outputs.trace != '' }} + continue-on-error: true + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: &storage-inputs + repository: SemiAnalysisAI/InferenceX-trace-storage + path: storage + ref: master + # Storage-repository deploy key for authorized profiling dispatches. + ssh-key: ${{ secrets.PROFILER_STORAGE_DEPLOY_KEY }} # zizmor: ignore[secrets-outside-env] + fetch-depth: 0 + # This separate storage checkout pushes trace commits using its scoped deploy key. + persist-credentials: true # zizmor: ignore[artipacked] + + - name: Wipe storage repo after failed checkout + if: ${{ steps.checkout-storage.outcome == 'failure' }} + run: rm -rf "$GITHUB_WORKSPACE/storage" + + - name: Retry storage repo checkout + if: ${{ steps.checkout-storage.outcome == 'failure' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: *storage-inputs # zizmor: ignore[artipacked] + + - name: Push profile to storage repo + if: ${{ steps.run.outputs.trace != '' }} + id: push + env: + MATRIX_CONFIG_RUNNER: ${{ matrix.config.runner }} + MATRIX_CONFIG_EXP_NAME: ${{ matrix.config['exp-name'] }} + MATRIX_EP: ${{ matrix.config.ep || 1 }} + TRACE_LOCAL: ${{ steps.run.outputs.trace }} + shell: bash + run: | + set -eo pipefail + + dest_dir="storage/profiles/${GITHUB_SHA}/${MATRIX_CONFIG_RUNNER}/${FRAMEWORK}/${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}" + mkdir -p "$dest_dir" + cp "$TRACE_LOCAL" "$dest_dir/trace.json.gz" + + pushd storage >/dev/null + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git add -A + git commit -m "Add profile: ${GITHUB_SHA} ${MATRIX_CONFIG_EXP_NAME} tp${TP} pp${PP_SIZE} dcp${DCP_SIZE} pcp${PCP_SIZE} ep${MATRIX_EP} conc${CONC}" || echo "Nothing to commit" + git push + STORAGE_SHA="$(git rev-parse HEAD)" + popd >/dev/null + + export RAW_URL="https://raw.githubusercontent.com/SemiAnalysisAI/InferenceX-trace-storage/${STORAGE_SHA}/profiles/${GITHUB_SHA}/${MATRIX_CONFIG_RUNNER}/${FRAMEWORK}/${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}/trace.json.gz" + export TITLE="${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}" + + enc_src="$(python3 -c 'import os,urllib.parse; print(urllib.parse.quote(os.environ["RAW_URL"], safe=""))')" + enc_title="$(python3 -c 'import os,urllib.parse; print(urllib.parse.quote(os.environ["TITLE"], safe=""))')" + + relay="${PERFETTO_RELAY_URL%/}" + RELAY_URL="${relay}/?src=${enc_src}&title=${enc_title}" + + echo "raw_url=$RAW_URL" >> "$GITHUB_OUTPUT" + echo "relay_url=$RELAY_URL" >> "$GITHUB_OUTPUT" + + - name: Print Perfetto link (relay) + if: ${{ steps.push.outputs.relay_url != '' }} + env: + RELAY_URL: ${{ steps.push.outputs.relay_url }} + RAW_URL: ${{ steps.push.outputs.raw_url }} + shell: bash + run: | + set -eo pipefail + echo "RAW trace URL: $RAW_URL" + echo "Perfetto Relay URL: $RELAY_URL" + printf "\n**Perfetto (Relay):** %s\n" "$RELAY_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/inferencex-e2e/infx/tests/workflows/test_launch_layout.py b/inferencex-e2e/infx/tests/workflows/test_launch_layout.py index 47f16969b2..1e3ab04092 100644 --- a/inferencex-e2e/infx/tests/workflows/test_launch_layout.py +++ b/inferencex-e2e/infx/tests/workflows/test_launch_layout.py @@ -27,6 +27,7 @@ result = os.environ["RESULT_FILENAME"] Path(result + ".json").write_text("{}\\n") Path(result + "_conc1.json").write_text('{"num_requests_successful": 1}\\n') +Path("profile_" + result + ".trace.json.gz").write_bytes(b"fixture trace") Path("speedbench-reference-al.yaml").write_text("fixture: 1\\n") """ @@ -71,7 +72,7 @@ def workflow_step(workflow: str, step_name: str) -> dict: [ ("benchmark-tmpl.yml", "Launch job script"), ("benchmark-multinode-tmpl.yml", "Launch multi-node job script"), - ("profile.yml", "Launch job script"), + ("profile.yml", "Launch + Profile (single-node sglang/vllm)"), ("speedbench-al.yml", "Collect AL matrix"), ], ) @@ -132,4 +133,6 @@ def test_launch_runs_the_python_entrypoint_from_the_project_root(tmp_path, workf assert "fixture: 1" in summary.read_text() else: assert (project / (exported["RESULT_FILENAME"] + ".json")).read_text() == "{}\n" - + if workflow == "profile.yml": + output = dict(line.split("=", 1) for line in github_output.read_text().splitlines()) + assert Path(output["trace"]).read_bytes() == b"fixture trace"