From 2b6efcb51c5785298ee903b5edfb3d13fc5a562d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 6 Nov 2026 04:31:38 -0600 Subject: [PATCH 1/3] feat(kimik3.1): re-sweep GB200 vLLM with the updated DSpark draft [skip-sweep] --- .../vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml index ef97d6303d..bf2f986c83 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml @@ -36,6 +36,8 @@ base: env: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" + # AIPerf reuses a session's connection across turns; outlast uvicorn's 5 s idle timeout. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "600" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c730b50783..f46283a7e9 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12285,3 +12285,13 @@ description: - "Add day-0 Kimi K3.1 Pro vLLM AgentX on GB200: aggregated TP8 x PP2 x DCP8 and disaggregated 1P1D TP16 + EP16 with DSpark (K3.1 golden AL curve) and Simple CPU KV offload." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4799 + +- config-keys: + - kimik3.1-fp4-gb200-vllm-agentic-dspark-tp8pp2-agg + - kimik3.1-fp4-gb200-vllm-agentic-dspark-tp16-disagg + scenario-type: + - agentic-coding + description: + - "Re-sweep Kimi-K3.1-Pro GB200 vLLM AgentX with the updated DSpark draft (Inferact/Kimi-K3.1-Pro-DSpark-preview @ 6a2ef20a) and the #4851 golden curve, so throughput forces AL 3.41 at 5 draft tokens and 3.19 at 4." + - "Set VLLM_HTTP_TIMEOUT_KEEP_ALIVE to 600 s on the aggregated server so AIPerf's reused session connections outlive uvicorn's 5 s idle timeout." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From a006870e36473ee0d8e3b077112902909f453f8d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 6 Nov 2026 04:31:54 -0600 Subject: [PATCH 2/3] chore(changelog): link the GB200 vLLM re-sweep entry to #4864 --- inferencex-e2e/perf-changelog.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f46283a7e9..bc106ac5e7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic @@ -10287,7 +10287,7 @@ description: - "Update the image from lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 to lmsysorg/sglang:nightly-dev-cu13-20261013-7224fce1." - "Set TP4 concurrency 2 max-running-requests to 2 and sweep concurrency 1 to 16 on TP4 and TP8." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp @@ -10694,7 +10694,7 @@ description: - "Replace the Muse Spark Open B300 TP8/EP8 search space with per-concurrency AgentX settings: TP8/EP8 with GPU-resident KV at concurrency 1, 4, 8 and 14; TP4/EP4 with GPU-resident KV at concurrency 4; TP4/EP4 with 120-175 GB per-rank DRAM HiCache at concurrency 10, 16, 20, 28, 36 and 40." - "Tune request admission, decode graph batch limits, memory fraction, SWA/full ratio, prefill-decode interval and chunked prefill per concurrency; enable layernorm SP and MegaMoE at the HiCache points and shortest-prefill-first from concurrency 16." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -10703,7 +10703,7 @@ description: - "Configure Muse Spark Open B200 SGLang AgentX request admission, scheduling, memory allocation, HiCache, and draft attention per concurrency, adding TP4/EP4 at concurrency 8, 10, 16, and 24 and TP8 at concurrency 6, 10, and 56." - "Drop TP8 concurrency 12, 16, 24, 32, 48, and 64." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - qwen3.5-fp8-b300-sglang-mtp @@ -10789,7 +10789,7 @@ - "Disagg: replace the DEP8 prefill + Mooncake store ladder with TP1 x PP8 prefill workers that keep a 2x HiCache host tier, and run W4A4 MegaMoE for the target model on prefill and decode while the DSpark draft MoE stays at the image default (W4A8)." - "Disagg points: 1P+DEP16 at concurrency 80, 160 and 200 with the KV router and token-balanced decode; 1P+DEP16 c300, 2P+DEP16 c720, 2P+DEP12 c800 and 2P+DEP8 c880 with a round-robin frontend and request-balanced decode. DSpark block size 6, 6, 5, 5 and 4, matched on prefill and decode." - "Agg: TP8 over two nodes at concurrency 1; TP4 on one node at concurrency 1 and 4 with the fastokens tokenizer. DSpark block size 6." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg @@ -10810,7 +10810,7 @@ description: - "Update the Muse Spark Open GB300 curve to lmsysorg/sglang:nightly-dev-20261021-c1c287a4 and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 and --stream-interval 20 at every concurrency." - "Lower max-running-requests to 3 at concurrency 10, raise prefill-decode-interval to 96 at concurrency 24, switch HiCache to write_back, and replace concurrency 16, 20 and 40 with 28, 30, 44 and 48." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -11250,7 +11250,7 @@ - minimaxm31flash-fp4-gb300-sglang-agentic-dspark description: - "Add TP2 points with DRAM KV offload, and adjust the memory settings on the DRAM KV offload points." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 scenario-type: [agentic-coding] - config-keys: @@ -12195,7 +12195,7 @@ description: - "Update the Muse Spark Open B200 curve to lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc and retune request admission, prefill-decode interval and stream interval per concurrency." - "Add TP4/EP4 concurrency 7 and 40; drop TP8 concurrency 2, 6, 8 and 56 and TP4 concurrency 8 and 10." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - musespark-fp4-gb200-sglang-agentic-dflash @@ -12204,7 +12204,7 @@ description: - "Retune the Muse Spark Open GB200 curve and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 at every concurrency; run TP4/EP4 at concurrency 1 without KV offload and at concurrency 4, 7, 10, 14, 16, 28, 36, 40 with HiCache, dropping concurrency 2, 6, 8, 20, 24 and 32." - "Use prefill-decode interval 64-256 and stream interval 20 at every HiCache point, shortest-prefill-first except at concurrency 7, request admission 2-6 at concurrency 4-16, 16K chunked prefill at concurrency 28-40, and at concurrency 40 mem-fraction-static 0.93, a 160 GB HiCache pool and CUDA graphs up to 48." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 - config-keys: - glm5.3flash-fp8-b200-sglang-agentic-mtp @@ -12294,4 +12294,4 @@ description: - "Re-sweep Kimi-K3.1-Pro GB200 vLLM AgentX with the updated DSpark draft (Inferact/Kimi-K3.1-Pro-DSpark-preview @ 6a2ef20a) and the #4851 golden curve, so throughput forces AL 3.41 at 5 draft tokens and 3.19 at 4." - "Set VLLM_HTTP_TIMEOUT_KEEP_ALIVE to 600 s on the aggregated server so AIPerf's reused session connections outlive uvicorn's 5 s idle timeout." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 From b961604f3aa1c940e35144b61214de771e7ccd71 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 6 Nov 2026 04:32:41 -0600 Subject: [PATCH 3/3] chore(changelog): restore historical placeholder links; link only the #4864 entry --- inferencex-e2e/perf-changelog.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index bc106ac5e7..9d06261111 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic @@ -10287,7 +10287,7 @@ description: - "Update the image from lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 to lmsysorg/sglang:nightly-dev-cu13-20261013-7224fce1." - "Set TP4 concurrency 2 max-running-requests to 2 and sweep concurrency 1 to 16 on TP4 and TP8." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp @@ -10694,7 +10694,7 @@ description: - "Replace the Muse Spark Open B300 TP8/EP8 search space with per-concurrency AgentX settings: TP8/EP8 with GPU-resident KV at concurrency 1, 4, 8 and 14; TP4/EP4 with GPU-resident KV at concurrency 4; TP4/EP4 with 120-175 GB per-rank DRAM HiCache at concurrency 10, 16, 20, 28, 36 and 40." - "Tune request admission, decode graph batch limits, memory fraction, SWA/full ratio, prefill-decode interval and chunked prefill per concurrency; enable layernorm SP and MegaMoE at the HiCache points and shortest-prefill-first from concurrency 16." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -10703,7 +10703,7 @@ description: - "Configure Muse Spark Open B200 SGLang AgentX request admission, scheduling, memory allocation, HiCache, and draft attention per concurrency, adding TP4/EP4 at concurrency 8, 10, 16, and 24 and TP8 at concurrency 6, 10, and 56." - "Drop TP8 concurrency 12, 16, 24, 32, 48, and 64." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b300-sglang-mtp @@ -10789,7 +10789,7 @@ - "Disagg: replace the DEP8 prefill + Mooncake store ladder with TP1 x PP8 prefill workers that keep a 2x HiCache host tier, and run W4A4 MegaMoE for the target model on prefill and decode while the DSpark draft MoE stays at the image default (W4A8)." - "Disagg points: 1P+DEP16 at concurrency 80, 160 and 200 with the KV router and token-balanced decode; 1P+DEP16 c300, 2P+DEP16 c720, 2P+DEP12 c800 and 2P+DEP8 c880 with a round-robin frontend and request-balanced decode. DSpark block size 6, 6, 5, 5 and 4, matched on prefill and decode." - "Agg: TP8 over two nodes at concurrency 1; TP4 on one node at concurrency 1 and 4 with the fastokens tokenizer. DSpark block size 6." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg @@ -10810,7 +10810,7 @@ description: - "Update the Muse Spark Open GB300 curve to lmsysorg/sglang:nightly-dev-20261021-c1c287a4 and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 and --stream-interval 20 at every concurrency." - "Lower max-running-requests to 3 at concurrency 10, raise prefill-decode-interval to 96 at concurrency 24, switch HiCache to write_back, and replace concurrency 16, 20 and 40 with 28, 30, 44 and 48." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -11250,7 +11250,7 @@ - minimaxm31flash-fp4-gb300-sglang-agentic-dspark description: - "Add TP2 points with DRAM KV offload, and adjust the memory settings on the DRAM KV offload points." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX scenario-type: [agentic-coding] - config-keys: @@ -12195,7 +12195,7 @@ description: - "Update the Muse Spark Open B200 curve to lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc and retune request admission, prefill-decode interval and stream interval per concurrency." - "Add TP4/EP4 concurrency 7 and 40; drop TP8 concurrency 2, 6, 8 and 56 and TP4 concurrency 8 and 10." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - musespark-fp4-gb200-sglang-agentic-dflash @@ -12204,7 +12204,7 @@ description: - "Retune the Muse Spark Open GB200 curve and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 at every concurrency; run TP4/EP4 at concurrency 1 without KV offload and at concurrency 4, 7, 10, 14, 16, 28, 36, 40 with HiCache, dropping concurrency 2, 6, 8, 20, 24 and 32." - "Use prefill-decode interval 64-256 and stream interval 20 at every HiCache point, shortest-prefill-first except at concurrency 7, request admission 2-6 at concurrency 4-16, 16K chunked prefill at concurrency 28-40, and at concurrency 40 mem-fraction-static 0.93, a 160 GB HiCache pool and CUDA graphs up to 48." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4864 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - glm5.3flash-fp8-b200-sglang-agentic-mtp