From 5ec97bbffe013914c55b0dd348a9b42797065f8f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 5 Nov 2026 11:01:35 -0600 Subject: [PATCH 1/3] fix(synthetic-acceptance): drop Kimi vLLM parsers on synthetic throughput servers Synthetic acceptance streams unverified draft tokens. The kimi_k3 tool and reasoning parsers stop streaming on some of them, which empties or truncates the response the client sees. Evals and acceptance gates verify for real and keep their parsers. --- .../infx/srt_slurm/synthetic_acceptance.py | 6 +++++ .../srt_slurm/test_synthetic_acceptance.py | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index d3531d6043..4373b27f42 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -43,6 +43,8 @@ ) TRT_VARIABLE = "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS" TILERT_VARIABLES = ("TILERT_SIMULATE_ACC_LEN", "TILERT_SIMULATE_ACC_METHOD") +# Synthetic acceptance streams unverified draft tokens. Kimi's parsers stop streaming on them. +KIMI_VLLM_PARSER_ARGS = ("enable-auto-tool-choice", "tool-call-parser", "reasoning-parser") def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: @@ -299,6 +301,10 @@ def build_overrides( worker_spec.update( rejection_sample_method="synthetic", synthetic_acceptance_length=worker_al ) + if environment["MODEL_PREFIX"].startswith("kimi"): + for key in KIMI_VLLM_PARSER_ARGS: + if key in (role.get("args") or {}): + overrides += ["--unset", f"{prefix}.args.{key}"] elif ( worker_spec.get("rejection_sample_method") == "synthetic" or "synthetic_acceptance_length" in worker_spec diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index bfca6bf560..4c5fa24ae4 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -978,3 +978,25 @@ def test_an_eval_server_verifies_for_real(tmp_path: Path) -> None: overrides = build_overrides(recipe, "dynamo-vllm", env, golden_dir=tmp_path / "absent") real = apply_native(recipe, overrides)["roles"]["agg"]["args"] assert json.loads(real["speculative-config"]) == {**spec, "rejection_sample_method": "block"} + + +@pytest.mark.parametrize(("prefix", "eval_only"), [("kimik3", "false"), ("kimik3", "true"), ("dsv4", "false")]) +def test_synthetic_throughput_drops_kimi_vllm_parsers( + golden_dir: Path, prefix: str, eval_only: str +) -> None: + recipe = vllm_recipe("dspark", draft_sample_method="probabilistic") + parsers = { + "enable-auto-tool-choice": True, + "tool-call-parser": "kimi_k3", + "reasoning-parser": "kimi_k3", + } + recipe["roles"]["agg"]["args"].update(parsers, **{"kv-cache-dtype": "fp8"}) + env = {**ENV, "MODEL_PREFIX": prefix, "SPEC_DECODING": "dspark", "EVAL_ONLY": eval_only} + overrides = build_overrides(recipe, "vllm", env, golden_dir=golden_dir) + args = apply_native(recipe, overrides)["roles"]["agg"]["args"] + assert args["kv-cache-dtype"] == "fp8" + if eval_only == "true" or prefix == "dsv4": + assert parsers.items() <= args.items() + else: + assert parsers.keys().isdisjoint(args) + From 58e68f7b3abbc3e78b900a281c2effcd8bac5d70 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 5 Nov 2026 14:16:53 -0600 Subject: [PATCH 2/3] chore(changelog): re-run Kimi-K3.1-Pro vLLM B300 with synthetic parsers off --- inferencex-e2e/perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index bc808e6451..c01e338f77 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12099,3 +12099,11 @@ - "Disaggregated: TP8/DCP8 prefill with a HiCache host tier feeding either one TP16 (DCP8 x attention DP2, EP16 MegaMoE) decode worker (3P1D c300, 2P1D c180, 1P1D c76) or TP8/DCP8 decode workers (1P3D c60 and c36, 1P4D c32)." - "Aggregated low-latency points: one TP16 EP1 worker with the NVIDIA megakernel (--enable-nvidia-megakernel), max-running-requests equal to the concurrency (c1, c2, c4)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4811 + +- config-keys: + - kimik3.1-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Re-run Kimi-K3.1-Pro vLLM AgentX on B300 with the tool and reasoning parsers off on synthetic-acceptance throughput servers; evals keep them." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4840 From d9c01aace0b5e6ebdddc30aa19ee3787ec2398c2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 5 Nov 2026 14:23:59 -0600 Subject: [PATCH 3/3] chore(changelog): drop the B300 re-run entry --- inferencex-e2e/perf-changelog.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c01e338f77..bc808e6451 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12099,11 +12099,3 @@ - "Disaggregated: TP8/DCP8 prefill with a HiCache host tier feeding either one TP16 (DCP8 x attention DP2, EP16 MegaMoE) decode worker (3P1D c300, 2P1D c180, 1P1D c76) or TP8/DCP8 decode workers (1P3D c60 and c36, 1P4D c32)." - "Aggregated low-latency points: one TP16 EP1 worker with the NVIDIA megakernel (--enable-nvidia-megakernel), max-running-requests equal to the concurrency (c1, c2, c4)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4811 - -- config-keys: - - kimik3.1-fp4-b300-vllm-agentic-dspark - scenario-type: - - agentic-coding - description: - - "Re-run Kimi-K3.1-Pro vLLM AgentX on B300 with the tool and reasoning parsers off on synthetic-acceptance throughput servers; evals keep them." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4840