Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
)
TRT_VARIABLE = "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS"
TILERT_VARIABLES = ("TILERT_SIMULATE_ACC_LEN", "TILERT_SIMULATE_ACC_METHOD")
# Synthetic acceptance streams unverified draft tokens. Kimi's parsers stop streaming on them.
KIMI_VLLM_PARSER_ARGS = ("enable-auto-tool-choice", "tool-call-parser", "reasoning-parser")


def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]:
Expand Down Expand Up @@ -299,6 +301,10 @@ def build_overrides(
worker_spec.update(
rejection_sample_method="synthetic", synthetic_acceptance_length=worker_al
)
if environment["MODEL_PREFIX"].startswith("kimi"):
for key in KIMI_VLLM_PARSER_ARGS:
if key in (role.get("args") or {}):
overrides += ["--unset", f"{prefix}.args.{key}"]
Comment on lines +304 to +307

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 Kimi vLLM synthetic throughput benchmarks now run without their parsers, but no perf-changelog entry is added, so the re-run the PR describes won't be scheduled. CONTRIBUTING.md:30 requires a new entry at the end of inferencex-e2e/perf-changelog.yaml for any change to what a benchmark runs. The diff only touches synthetic_acceptance.py and its test. Fix: append one entry listing every affected Kimi vLLM synthetic-acceptance config key, including kimik3.1-fp4-b300-vllm-agentic-dspark. python -m infx.matrix.changed --base origin/main lists them.

Why this was flagged

synthetic_acceptance.py:304-307 now adds --unset roles.<name>.args.{enable-auto-tool-choice,tool-call-parser,reasoning-parser} for every vLLM worker whose MODEL_PREFIX starts with "kimi" and that receives a synthetic acceptance length. That changes the server command, and the PR's own numbers show it changes ITL and error counts. CONTRIBUTING.md:30 says such a PR MUST append a perf-changelog.yaml entry for the affected config keys, and the changelog diff is what selects the sweep. git diff --stat shows only two changed files and no perf-changelog.yaml. So kimik3.1-fp4-b300-vllm-agentic-dspark (nvidia-master.yaml:5727) is not re-run. Its published ITL and interactivity stay at the 17–26% optimistic values from the parsers-on run until some unrelated changelog entry triggers that key.

Verification: The diff touches only inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py and its test, and perf-changelog.yaml is unchanged. Lines 304-307 add --unset {prefix}.args.{key} for the parsers when MODEL_PREFIX starts with "kimi". infx.matrix.changed does not see script changes like this one, so without an entry no sweep or result collection is triggered.

elif (
worker_spec.get("rejection_sample_method") == "synthetic"
or "synthetic_acceptance_length" in worker_spec
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -978,3 +978,25 @@ def test_an_eval_server_verifies_for_real(tmp_path: Path) -> None:
overrides = build_overrides(recipe, "dynamo-vllm", env, golden_dir=tmp_path / "absent")
real = apply_native(recipe, overrides)["roles"]["agg"]["args"]
assert json.loads(real["speculative-config"]) == {**spec, "rejection_sample_method": "block"}


@pytest.mark.parametrize(("prefix", "eval_only"), [("kimik3", "false"), ("kimik3", "true"), ("dsv4", "false")])
def test_synthetic_throughput_drops_kimi_vllm_parsers(
golden_dir: Path, prefix: str, eval_only: str
) -> None:
recipe = vllm_recipe("dspark", draft_sample_method="probabilistic")
parsers = {
"enable-auto-tool-choice": True,
"tool-call-parser": "kimi_k3",
"reasoning-parser": "kimi_k3",
}
recipe["roles"]["agg"]["args"].update(parsers, **{"kv-cache-dtype": "fp8"})
env = {**ENV, "MODEL_PREFIX": prefix, "SPEC_DECODING": "dspark", "EVAL_ONLY": eval_only}
overrides = build_overrides(recipe, "vllm", env, golden_dir=golden_dir)
args = apply_native(recipe, overrides)["roles"]["agg"]["args"]
assert args["kv-cache-dtype"] == "fp8"
if eval_only == "true" or prefix == "dsv4":
assert parsers.items() <= args.items()
else:
assert parsers.keys().isdisjoint(args)

Loading