diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index d3531d6043..4373b27f42 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -43,6 +43,8 @@ ) TRT_VARIABLE = "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS" TILERT_VARIABLES = ("TILERT_SIMULATE_ACC_LEN", "TILERT_SIMULATE_ACC_METHOD") +# Synthetic acceptance streams unverified draft tokens. Kimi's parsers stop streaming on them. +KIMI_VLLM_PARSER_ARGS = ("enable-auto-tool-choice", "tool-call-parser", "reasoning-parser") def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: @@ -299,6 +301,10 @@ def build_overrides( worker_spec.update( rejection_sample_method="synthetic", synthetic_acceptance_length=worker_al ) + if environment["MODEL_PREFIX"].startswith("kimi"): + for key in KIMI_VLLM_PARSER_ARGS: + if key in (role.get("args") or {}): + overrides += ["--unset", f"{prefix}.args.{key}"] elif ( worker_spec.get("rejection_sample_method") == "synthetic" or "synthetic_acceptance_length" in worker_spec diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index bfca6bf560..4c5fa24ae4 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -978,3 +978,25 @@ def test_an_eval_server_verifies_for_real(tmp_path: Path) -> None: overrides = build_overrides(recipe, "dynamo-vllm", env, golden_dir=tmp_path / "absent") real = apply_native(recipe, overrides)["roles"]["agg"]["args"] assert json.loads(real["speculative-config"]) == {**spec, "rejection_sample_method": "block"} + + +@pytest.mark.parametrize(("prefix", "eval_only"), [("kimik3", "false"), ("kimik3", "true"), ("dsv4", "false")]) +def test_synthetic_throughput_drops_kimi_vllm_parsers( + golden_dir: Path, prefix: str, eval_only: str +) -> None: + recipe = vllm_recipe("dspark", draft_sample_method="probabilistic") + parsers = { + "enable-auto-tool-choice": True, + "tool-call-parser": "kimi_k3", + "reasoning-parser": "kimi_k3", + } + recipe["roles"]["agg"]["args"].update(parsers, **{"kv-cache-dtype": "fp8"}) + env = {**ENV, "MODEL_PREFIX": prefix, "SPEC_DECODING": "dspark", "EVAL_ONLY": eval_only} + overrides = build_overrides(recipe, "vllm", env, golden_dir=golden_dir) + args = apply_native(recipe, overrides)["roles"]["agg"]["args"] + assert args["kv-cache-dtype"] == "fp8" + if eval_only == "true" or prefix == "dsv4": + assert parsers.items() <= args.items() + else: + assert parsers.keys().isdisjoint(args) +