From 49a86c244a9464f878696b54cdd85b130287a186 Mon Sep 17 00:00:00 2001 From: "Eric.Chin.AMD" Date: Thu, 15 Oct 2026 00:11:22 +0800 Subject: [PATCH 1/5] [AMD][AgentX] Add GLM-5.2 MXFP4 MI355X SGLang TP4/EP4 DCP4 points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TP4/EP4 decode-context-parallel (DCP4) points at concurrency 16, 24, 40 and 48 to glm5.2-fp4-mi355x-sglang-agentic-mtp. The latent KV is striped across the four ranks, so these points run without HiCache, which DSA DCP rejects. Validate decode-context-parallel-size against the matrix DCP size for SGLang recipes, as for vLLM and ATOM. 中文:為 glm5.2-fp4-mi355x-sglang-agentic-mtp 新增 TP4/EP4 搭配 decode context parallel(DCP4)的 concurrency 16、24、40、48 點位。latent KV 依位置分散到四張卡上,因此這些點位不使用 HiCache(DSA DCP 不支援 HiCache)。SGLang recipe 的 decode-context-parallel-size 也會與 matrix 的 DCP size 比對驗證,與 vLLM、ATOM 一致。 --- .../glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml | 61 +++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 10 ++- inferencex-e2e/infx/srt_slurm/single_node.py | 3 +- .../tests/srt_slurm/test_srt_single_node.py | 14 +++++ inferencex-e2e/perf-changelog.yaml | 11 ++++ 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml index 5e3e101597..fb52811040 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -152,6 +152,67 @@ override_tp4_c16_hicache: CONC: '16' KV_OFFLOADING: dram +# DSA DCP rejects HiCache; DCP4 stripes the latent KV across the TP4 ranks instead. +override_tp4_dcp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_dcp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 48 + cuda-graph-max-bs-decode: 48 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: none + +override_tp4_dcp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 80 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: none + +override_tp4_dcp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 96 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: none + override_tp8_c1: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 24f60ae790..085a7da146 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1405,13 +1405,16 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: dp-attn: true # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Two arms: +# decoding. Three arms: # (1) TP4/EP4 with HiCache KV offloading to DRAM at concurrency [4, 8, 10, 12, 14, 16]; # serves the high-concurrency, throughput-dominant regime. -# (2) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4]; +# (2) TP4/EP4 with DCP4 at concurrency [16, 24, 40, 48]; striping the latent KV +# across the four ranks holds the working set on device without HiCache, +# which DSA DCP rejects. +# (3) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4]; # EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid # concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. -# c4 intentionally overlaps both arms to compare the topology crossover. +# c4 intentionally overlaps arms (1) and (3) to compare the topology crossover. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260930 @@ -1427,6 +1430,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.85 search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, dcp-size: 4, kv-offloading: none, conc-list: [16, 24, 40, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } # GLM-5.2 FP8 agentic-coding benchmark on MI355X via SGLang with MTP speculative diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index c9f8b6c8ac..dc3db574d0 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -140,7 +140,8 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N expected[name] = (str(workload[name]), environment[name]) if engine == "atom": expected["ATOM TP"] = (role["gpus"], int(environment["TP"])) - dcp = str(args.get("decode-context-parallel-size", 1)) if engine in {"vllm", "atom"} else "1" + dcp_engines = {"vllm", "atom", "sglang"} + dcp = str(args.get("decode-context-parallel-size", 1)) if engine in dcp_engines else "1" for name, value in {"PP_SIZE": "1", "DCP_SIZE": dcp, "PCP_SIZE": "1"}.items(): expected[name] = (environment[name], value) for name, (actual, wanted) in expected.items(): diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index 4d88d9c86f..78c16f50f6 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -182,6 +182,20 @@ def test_dp_attention_is_validated_without_replacing_recipe_topology(point): runtime_arguments(f"{path}:base", {**env, "DP_ATTENTION": "false"}) +def test_sglang_dcp_is_validated_against_the_point(point): + path, recipe, env = point + recipe["roles"]["agg"]["args"]["decode-context-parallel-size"] = 4 + path.write_text(yaml.safe_dump({"base": recipe})) + actual = copy.deepcopy(recipe) + argv = runtime_arguments(f"{path}:base", {**env, "DCP_SIZE": "4"}) + apply_overrides_to_recipe(actual, parse_overrides(argv[1::2], [])) + assert actual["roles"]["agg"]["args"]["decode-context-parallel-size"] == 4 + with pytest.raises(ValueError, match=r"DCP_SIZE: recipe/matrix '1' != '4'"): + runtime_arguments(f"{path}:base", env) + with pytest.raises(ValueError, match=r"DCP_SIZE: recipe/matrix '2' != '4'"): + runtime_arguments(f"{path}:base", {**env, "DCP_SIZE": "2"}) + + def test_trt_binding_keeps_engine_options_and_sets_eval_token_budget(point): path, recipe, env = point recipe["engine"] = {"type": "trtllm", "served_model_name": "test/model"} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 6f2ebc3425..17487c3238 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9738,3 +9738,14 @@ - "Enable INT4 QuickReduce with a 256 MiB size limit and FlyDSL stage-2 FP8 on prefill and decode workers." - "Configure MoRIIO READ and prefill-only SimpleCPUOffloadConnector through native role arguments. Use recipe-scoped draft checkpoint and RDMA device/provider mounts. Use the upstream discovery-template fix from srt-slurm #508 through the shared v2.51.1 dependency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3582 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48; the latent KV is striped across the four ranks, so these points run without HiCache, which DSA DCP rejects." + - "Keep the EAGLE draft with 5 steps, top-k 1 and 6 draft tokens, FP8 KV, the triton DSA prefill and decode backends, admission at twice the concurrency and a decode graph limit of min(2 x concurrency, 64)." + - "Validate decode-context-parallel-size against the matrix DCP size for SGLang recipes as well as vLLM and ATOM." + - "Requires an SGLang ROCm image with sgl-project/sglang#43735 and the triton sparse-MLA LSE path for DCP." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From f87158de72c62b249889a302995ad55e90714fb2 Mon Sep 17 00:00:00 2001 From: "Eric.Chin.AMD" Date: Thu, 15 Oct 2026 00:44:25 +0800 Subject: [PATCH 2/5] [AMD][AgentX] Point the DCP4 changelog entry at PR 4064 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the real pull request number in the perf-changelog entry for the GLM-5.2 MXFP4 MI355X SGLang DCP4 points. 中文:把 perf-changelog 的 pr-link 換成實際的 PR 4064。 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 17487c3238..22555f6e30 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9748,4 +9748,4 @@ - "Keep the EAGLE draft with 5 steps, top-k 1 and 6 draft tokens, FP8 KV, the triton DSA prefill and decode backends, admission at twice the concurrency and a decode graph limit of min(2 x concurrency, 64)." - "Validate decode-context-parallel-size against the matrix DCP size for SGLang recipes as well as vLLM and ATOM." - "Requires an SGLang ROCm image with sgl-project/sglang#43735 and the triton sparse-MLA LSE path for DCP." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4064 From a600ec97d6b7f03c74bb9f05f79b19175ad95443 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 14 Oct 2026 17:58:45 +0000 Subject: [PATCH 3/5] fix changelog --- inferencex-e2e/perf-changelog.yaml | 4828 +++++++++++++++++++++++++++- 1 file changed, 4827 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 49b4bd8f53..940ec5d8a2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -4934,4 +4934,4830 @@ description: - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs." - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" - - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at co \ No newline at end of file + - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2241 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + description: + - "Bump Qwen3.5-FP8 MI355X SGLang disagg image from lmsysorg/sglang-rocm:v0.5.11-rocm700-mi35x-20260511 to lmsysorg/sglang:v0.5.14-rocm720-mi35x (ROCm 7.0 → 7.2, sglang v0.5.11 → v0.5.14)." + - "Fix disagg SLURM exclude list: add mia1-p01-g09 (broken pyxis) to submit.sh — was missing from multi-node path, causing prefill worker 503 failures." + - "Expand mi355x-disagg runner pool from 3 to 9 nodes." + - "Concurrency sweep: 1K1K and 8K1K both [ 8, 16, 32, 64, 128 ]." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2094 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm-mtp + description: + - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic + description: + - "Add GLM-5.2 NVFP4 B300 SGLang single-node agentic benchmarks from the SGLang cookbook B300 NVFP4 recipes (STP only, no spec decoding)" + - "Image: lmsysorg/sglang:v0.5.15.post1-cu130; model nvidia/GLM-5.2-NVFP4" + - "Low-latency arm: TP8 (fp8 KV cache, cutedsl bf16 GEMM backend) at conc [1, 2, 4, 8, 16, 32]" + - "High-throughput arm: TP8/DP8 attention-DP behind sglang-router consistent hashing at conc [48, 64, 96, 128, 192, 256, 512]" + - "New 1M-context model prefix glm5.2 added to the unfiltered agentic corpus branch in benchmark_lib.sh" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2268 + +- config-keys: + - kimik2.5-int4-b200-vllm + - kimik2.5-int4-b300-vllm + description: + - "Bump vLLM image to v0.25.0" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic + description: + - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: DSv4-style host/device token-capacity ratio, tuned to 0.75 for GLM-5.2 (DSA/MLA-family: the 169.98 GB device pool is replicated on all 8 ranks, so ratio 0.75 = ~1.0 TB pinned host pools; ratio 2 and GB-based sizing both OOM-killed 3 TB nodes), io-backend direct, mem-layout page_first_direct, write policy write_back; validated on-node with an overflow bench forcing eviction through the DSA KV+INDEXER host pools" + - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" + - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2279 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic + description: + - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: DSv4-style host/device token-capacity ratio, tuned to 0.75 for GLM-5.2 (DSA/MLA-family: the 169.98 GB device pool is replicated on all 8 ranks, so ratio 0.75 = ~1.0 TB pinned host pools; ratio 2 and GB-based sizing both OOM-killed 3 TB nodes), io-backend direct, mem-layout page_first_direct, write policy write_back; validated on-node with an overflow bench forcing eviction through the DSA KV+INDEXER host pools" + - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" + - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic + description: + - "Convert the high-throughput arm to DEP: add ep: 8 to the TP8/DP8 attention-DP arm so the MoE layers run expert-parallel across all 8 ranks (--ep-size 8) instead of TP-sharded (the arm previously left ep unset -> EP_SIZE=1). Attention stays data-parallel behind sglang-router consistent hashing; the recipe already passes --ep-size EP_SIZE unconditionally, so only the master-config search space changes" + - "DEP arm runs conc-list [48] only (was [48, 64, 96, 128, 192, 256, 512]): conc 48 is the frontier peak (163 tok/s/gpu out, ttft 2.8s); conc >=64 overflows the HBM+host radix cache (GPU hit 0.93->0.57 at 48->64) and thrashes on re-prefill, so it is strictly dominated" + - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg + description: + - "Retune the non-MTP 8k1k disagg search space: drop the 2P1D DEP8 sweep (conc 128/256/512/1024 across 2 prefill nodes) and the 1P1D DEP8 conc=1024 point" + - "Add 1P1D DEP8 (TP8/EP8 + dp-attention, 1 prefill node) at conc 256/512, and a new 1P1D pure-TP8 (no EP, no DP-attention) scenario sweeping conc [1, 2, 4, 8, 16, 32, 64, 128]" + - "Fix 2P1D eval 503s: speed up router circuit-breaker recovery (--cb-timeout-duration-secs 15 --retry-max-retries 3) and gate benchmarking on an end-to-end router readiness canary (one real generation through the router), so the eval no longer starts against a router whose prefill path isn't actually serving yet" + - "Fix the 8k1k conc-500 straggler tail: decode now derives its admission context_length from models.yaml (falling back to prefill's value when unset) so decode and prefill reject over-length requests symmetrically instead of decode hanging on a KV transfer prefill never sends" + - "Bump DeepSeek-V4-Pro prefill max_total_tokens from 262144 to 2097152 to match the model's actual KV budget" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2293 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic + scenario-type: + - agentic-coding + description: + - "Add LMCache 0.5.1 DRAM KV-offload arms to the official B300 AgentX config from PR #2241 (sparse DSV4 FlashInfer attention, AMXF4 mega-MoE, FULL_DECODE_ONLY CUDA graphs, image vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec), so LMCache is benchmarked in the same sweep as the SimpleCPU and GPU-resident arms" + - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector per PR #2153/#2231; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB; PYTORCH_ALLOC_CONF=expandable_segments:True dropped on the lmcache arms only (VMM allocations cannot be CUDA-IPC-exported to the LMCache server, same failure mode as cuMem on B200); gpu-memory-utilization 0.92 on lmcache DEP8 (matches the official DEP8 derate) and 0.94 on lmcache TP4/DEP4 (at 0.96 the PR #2232 bring-up sweeps hit torch-pool, DeepGEMM-JIT, and cuBLAS-workspace OOMs; 23 lmcache points validated green in runs 29463061871/29535333851 after the derate)" + - "LMCache ladders offset the official arms: TP4 +4 conc [32, 36, 40, 44] vs SimpleCPU [28, 32, 36, 40]; DEP4 +4 conc [36, 44, 52, 60, 68, 76] vs SimpleCPU [32, 40, 48, 56, 64, 72]; DEP8 +8 conc capped at 208 [72, 104, 120, 136, 152, 168, 184, 200] vs GPU-resident [64, 96, 112, 128, 144, 160, 176, 192, 224]" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2232 + + +- config-keys: + - dsv4-fp4-b200-vllm-agentic + scenario-type: + - agentic-coding + description: + - "Add LMCache 0.5.1 DRAM KV-offload arm to the official B200 AgentX sweep alongside the vllm-simple and Mooncake arms, on the same tuned recipe and image (vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec) from PR #2224" + - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector per PR #2153; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB; --enable-cumem-allocator dropped on the lmcache arm only (cuMem/VMM allocations cannot be CUDA-IPC-exported to the LMCache server)" + - "LMCache TP8 points mirror the vllm-simple ladder (conc [8, 12, 16]); LMCache DEP8 points mirror the Mooncake ladder (conc [12, 20, 28, 36, 44, 52, 60, 68, 76]) for a like-for-like backend comparison; the PR #2231 bring-up sweep validated the recipe across DEP8 conc 8-80 (peak ~25.7k total tok/s/GPU at conc 72, 96-98% cache hit)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2231 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + description: + - "Add MTP speculative-decoding for the B300 vLLM AgentX recipe as a separate config-key, so the existing dsv4-fp4-b300-vllm-agentic aggregate is not re-run." + - "dsv4-fp4-b300-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms plus a new TP8 GPU-resident arm -- TP8 GPU-resident [1,2,4,6,8], TP4 GPU-resident [1,2,4,6,8,12,16,20], TP4 SimpleCPU [20,24,28,32,36,40], DEP4 [32,40,48,56], DEP8 [64,96,112,128,144,160,176,192,224]; routed via spec-decoding=mtp to dsv4_fp4_b300_vllm_mtp.sh (FULL_DECODE_ONLY cudagraph capture sizes in TOKENS = num_seqs*4)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2258 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm + description: + - "Close the high-concurrency gap vs the ATOM recipe on MiniMax-M3 MXFP4 MI355X single-node vLLM STP. Bump image to latest nightly vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1." + - "Add INT4 quantized all-reduce (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0, VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256) and fp8 KV cache (--kv-cache-dtype fp8). No --hf-overrides indexer-skipping knobs (disallowed: reduces model architecture FLOPs per PR_REVIEW_CHECKLIST.md)." + - "Simplify search space to a single TP4 conc 1-512 sweep for 8k1k (drop TP8, TP4/EP4, TP2/EP2, and dp-attn variants), matching minimaxm3-fp8-mi355x-vllm (#2003); 1k1k deprecated per #2263." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2104 + + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + description: + - "Add MTP speculative-decoding for the B200 vLLM AgentX recipe as a separate config-key, so the existing dsv4-fp4-b200-vllm-agentic aggregate is not re-run." + - "dsv4-fp4-b200-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms -- TP8 GPU-resident, TP8 SimpleCPU, DEP8 SimpleCPU, DEP8 Mooncake, each mirroring its non-MTP conc-list; routed via spec-decoding=mtp to dsv4_fp4_b200_vllm_mtp.sh." + - "MTP capture sizes are in TOKENS: --max-cudagraph-capture-size scaled to MAX_NUM_SEQS*(1+num_speculative_tokens) (= MAX_NUM_SEQS*4) so FULL_DECODE_ONLY covers the largest decode batches." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2259 + +- config-keys: + - kimik2.5-fp4-b200-dynamo-trt + description: + - "Add Kimi K2.5 NVFP4 B200 disaggregated TRT-LLM benchmarks via Dynamo (11 STP configs, ISL 8k OSL 1k)" + - "Container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 with Dynamo installed from source (hash 4025cb6a8992d19fd19eead36065e08ea5301f35)" + - "Topologies: 1p4d-dep4-tep8 (conc 4/24), 1p5d-dep4-tep4 (conc 5/30/60/115/195), 1p1d-dep4-dep8 (conc 308), 2p1d-dep4-dep8 (conc 615), 3p1d-dep4-dep8 (conc 1127), 4p1d-dep4-dep8 (conc 2151)" + - "Runner updated to clone srt-slurm at sa-submission-q2-2026 and copy local recipes into recipes/trtllm/kimi-k25-nvfp4/b200-fp4/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2249 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm-disagg + description: + - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "All-TP4 prefill/decode (drop TP8): a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU; expert parallelism off (ep:1, single-node TP8 sweep showed EP -14% to -27% slower than dense)" + - "Split the 8k/1k P/D topology by concurrency: low conc (1,2,4) on 1P(TP4)/1D(TP4); high conc (8,16,32,64,128,256,512) on 1P(TP4)/2D(TP4) so decode load/KV is spread across two decode engines (the single-decode engine is unstable in the high-conc tail)" + - "Sync per-worker vLLM serve flags/env with the single-node recipe: --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256, --max-num-batched-tokens 32768; VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults; drop the --compilation-config PIECEWISE pin (use vLLM default FULL_AND_PIECEWISE)" + - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)" + - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic + description: + - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" + - "Image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2118 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm-mtp + - minimaxm3-fp4-b300-dynamo-vllm-mtp-legacy-dep4 + description: + - "Refresh five MiniMax-M3 NVFP4 B300 8k1k disaggregated Dynamo-vLLM EAGLE3 recipes from vllm/vllm-openai:nightly-8e981630c9336233ca9de91452f68918bddbc4e2 to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + - "Remove the deprecated VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm override from the five refreshed prefill and decode environments, matching the image compatibility adjustment established by PR #2120" + - "Retain the 2P1D DEP4 recipe on its legacy image and all-reduce settings under a separate matrix key" + - "Preserve all six existing 8k1k EAGLE3 topologies, speculative-token settings, chat template, random range ratio, and 12 total concurrency points" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2314 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + description: + - "Add a Qwen3.5-397B-A17B-FP8 GB200 disaggregated Dynamo-SGLang MTP configuration for 8k/1k." + - "Add seven SGLang recipes for TP4, TEP8, and DEP4/DEP16 prefill/decode topologies with EAGLE speculative decoding and Mooncake KV transfer." + - "Enable chat-formatted benchmark inputs for every MTP recipe." + - "Use a digest-pinned SGLang v0.5.14-cu130 image and validate Enroot imports before publishing shared squash files." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2324 + +- config-keys: + - dsr1-fp8-b300-sglang-mtp + description: + - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc for all dp-on cells (avoids dp-attn OOM) and conc>=64; TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2327 + +- config-keys: + - minimaxm3-fp8-b200-vllm + - minimaxm3-fp8-b300-vllm + description: + - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9; set --stream-interval 32; drop the sparse_attention_msa contiguity patch (shipped in the nightly image)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2336 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-mtp + description: + - "Add MTP variant of dsv4-fp4-mi355x-sglang-disagg (new config-key dsv4-fp4-mi355x-sglang-disagg-mtp), Clones the base disagg search space and topologies exactly (image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713, mori KV transfer + MoE a2a, sglang-router --pd-disaggregation on :30000); enables DeepSeek-V4 built-in MTP via spec-decoding: mtp, same enablement path as dsr1-fp8-mi355x-sglang-disagg-mtp" + - "Topologies: 1P1D pure-TP8/EP1 (conc 1-128); 1P1D DEP8 (TP8/EP8 + dp-attention) conc 256 and conc 512. All arms at DECODE_MTP_SIZE=1 (draft depth 1)" + - "Recipe fix (model): adds mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to the DeepSeek-V4-Pro-DI models.yaml entry; without it DECODE_MTP_SIZE>0 emitted --speculative-num-steps/--speculative-num-draft-tokens with no --speculative-algorithm, so sglang never engaged spec decoding. EAGLE with in-checkpoint draft (no draft-model-path); NEXTN (the V3/R1 keyword) crashes the dsv4 decode server at init" + - "Recipe fix (launcher): build_server_config appended the speculative flags to the decode server only, so prefill never allocated the nextn (MTP draft) KV layer -- prefill registered 3 PD state components, decode 4. SGLang PD-disaggregation requires matching speculative config on both roles (the draft/MTP layers participate in both prefill KV computation and decode verification): v0.5.15 hard-fails the mismatch (mori 'state component count mismatch'), while v0.5.14 tolerated it silently but decode's nextn state was never seeded from prefill, corrupting greedy EAGLE verification (gsm8k 0.96 -> 0.88, worse at depth 2). Fix: apply the speculative flags to both prefill and decode" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2305/changes + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + description: + - "Add GB300 Dynamo-vLLM AgentX MTP3 points: aggregate TP8 c1, aggregate TP4 c4, P/D DEP4/DEP8 c128, and P/D DEP8/DEP8 c384; use upstream vLLM 426e59f, srt-slurm v1.0.36, and random session-affinity routing for P/D." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2341 + + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + description: + - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - dsv4-fp4-mi355x-atom-mtp + description: + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on; tp4 tried but removed — tp4+MTP+dp-on OOMs, KV budget negative). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1 + - minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1-tep4 + description: + - "Update the non-speculative MiniMax-M3 NVFP4 B300 8k1k disaggregated vLLM image from nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 to nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + - "Remove VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm from the 1P1D TP1-prefill/TP8-decode recipe; retain the existing FP8 indexer KV cache configuration" + - "Add a one-node 4P1D TP1-prefill/TEP4-decode concurrency-4096 point on the refreshed image while preserving the legacy 4P2D DEP2-prefill/TEP4-decode point unchanged" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2310 + + +- config-keys: + - kimik3-fp4-b300-vllm-agentic + description: + - "Add day-zero single-node agentic-coding recipe for Kimi-K3 (MXFP4, 2.8T MoE, 896 experts, KDA/MLA hybrid, 1M native context) on B300 with vLLM. Agentic-coding scenario only -- no fixed-seq-len (1k1k / 8k1k) arms." + - "Image vllm/vllm-openai:kimi-k3 (vLLM 0.1.dev19262+gb6bbf29dd, pushed 2026-07-27): K3 ships as the out-of-tree vllm.models.kimi_k3 plugin package (DSpark MLA + KDA kernels, registering KimiK3ForConditionalGeneration / KimiLinearForCausalLM / KimiK3MTPModel), so a generic vllm-openai release cannot serve this checkpoint." + - "Weights are pre-staged node-local at /scratch/models/Kimi-K3 (96/96 shards, 1.5 TB, verified on all 18 b300 nodes); Kimi-K3 is already in the b300-nv launcher STAGED_MODELS list, so MODEL_PATH resolves to the read-only staged mount and the runner performs no download." + - "Search space is two TP8 arms at conc 1-24: GPU-resident (kv-offloading none) and host-DRAM offload (kv-offloading dram, SimpleCPUOffloadConnector via kv-offload-backend vllm-simple). Both share one ladder so the arms are directly comparable at equal concurrency. TP8 is the only single-node layout that fits: the MXFP4 checkpoint is ~1.5 TB (~188 GB/GPU across 8 B300s), whereas TP4 would need ~375 GB/GPU against 288 GB of HBM." + - "DRAM arm: dram-utilization 0.63 resolves to total-cpu-dram-gb 1889 on cluster:b300-nv, which the script divides across the 8 TP ranks into cpu_bytes_to_use_per_rank (~220 GiB/rank), with eager offload (lazy_offload false). Note K3 is a KDA/MLA hybrid -- the KDA layers hold a recurrent state rather than paged KV blocks, so only the MLA layers' paged blocks are expected to offload; the bring-up sweep is the check for KV-geometry errors at server init." + - "Serve flags follow the K3 production recipe already exercised by the DSpark AL collector: --load-format fastsafetensors, --moe-backend auto, --enable-prefix-caching, --kv-cache-dtype fp8 with --attention-config mla_prefill_backend=FLASHINFER + use_prefill_query_quantization, kimi_k3 reasoning/tool-call parsers, NCCL_DMABUF_ENABLE=0, VLLM_ALLREDUCE_USE_FLASHINFER=1, VLLM_USE_RUST_FRONTEND=1." + - "Sets VLLM_HTTP_TIMEOUT_KEEP_ALIVE=900 and AIPERF_HTTP_TCP_USER_TIMEOUT=900000. AIPerf pins one pooled keep-alive connection per agentic session and reuses it across turns, while the Rust frontend defaults to a 5s keep-alive; an inter-turn idle gap past that lets the client reuse a socket exactly as the server closes it, producing an aiohttp ServerDisconnectedError that AIPerf escalates to a terminal warmup failure. This aborted the dram c4 arm ~15 min into run 30324907690 against a healthy server. Same mitigation as glm5.2_fp4_b300_sglang.sh's SGLANG_TIMEOUT_KEEP_ALIVE=900." + - "Bring-up validated in run 30326393603: all 12 configs green, zero ServerDisconnectedError after the keep-alive fix. GPU KV resolves to 42.23 GiB / 3,249,215 tokens, i.e. ~3.1 max-length requests, against MAX_NUM_SEQS = 2*CONC." + - "Measured behaviour: below conc 8 the GPU-resident and DRAM arms are within run-to-run noise (1-5%). At conc 16 and 24 the GPU-resident arm thrashes -- prefix cache hit rate 2.7%, TTFT p50 86s and 191s, 49.6 and 54.9 output tok/s -- because the working set exceeds GPU KV and prefixes are recomputed. The DRAM arm holds TTFT p50 0.85s and 6.2s for 245.0 and 260.6 output tok/s (4-5x), with the CPU tier serving a 62% external prefix cache hit rate at conc 24. The high-conc GPU-resident points are retained deliberately as the honest baseline that makes the offload gain legible." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2371 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic + description: + - "Kimi K3 B300 Day0 performance" + - "Tried single-node TP8 vLLM with GPU-resident KV and host-DRAM KV offload at concurrency 1, 2, 4, 8, 16, and 24. TP4 was not viable because the 1.5 TB checkpoint would require ~375 GB/GPU, above B300's 288 GB HBM." + - "Both arms were within 1-5% below concurrency 8. At concurrency 16 and 24, DRAM offload delivered 245 and 261 output tok/s with p50 TTFT of 0.85s and 6.2s, versus 50 and 55 output tok/s with 86s and 191s TTFT for GPU-resident (4-5x higher throughput)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2397 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + description: + - "Add the DSpark speculative-decoding variant of kimik3-fp4-b300-vllm-agentic: Kimi-K3 (MXFP4, 2.8T MoE, KDA/MLA hybrid, 1M native context) on B300 with vLLM, agentic-coding scenario only -- no fixed-seq-len (1k1k / 8k1k) arms." + - "Speculative config: method dspark on the Inferact/Kimi-K3-DSpark draft head, num_speculative_tokens 2, attention_backend FLASHINFER_MLA, draft_sample_method probabilistic, with rejection_sample_method synthetic and synthetic_acceptance_length 2.51 -- the committed golden AL at K=2 on the probabilistic curve (golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml). This follows the AgentX policy in golden_al_distribution/README.md: a submission may choose any supported draft length but may not substitute a different acceptance target, because InferenceX measures inference-system performance rather than the fitness of a benchmark-specific speculative head. The greedy/standard curve gives 2.45 at K=2; the curves are not mixed." + - "EVAL_ONLY runs switch to real block verification. Synthetic acceptance commits drafted tokens regardless of the target logits, so generated text is wrong and SWE-bench scores 0.0000 -- the same split dsv4_fp4_b300_vllm_mtp.sh makes, and which kimik2.5_fp4_b300_mtp.sh omits; dsv4 is the pattern followed here." + - "Level 2 rather than 7. Draft length is the submission's choice and verify width is a real system cost synthetic acceptance does not remove: at K=7 the target verifies 8 positions per step, which loses to no-speculation once the GPU saturates even at the forced golden AL of 3.84 (K=2 clears break-even at both ends of the cost range; K=3 is a wash at saturation; K=7 is negative). cudagraph capture consequently enumerates multiples of (1+2)=3, since vLLM rounds configured sizes to multiples of (1+N) and would otherwise cover only MAX_NUM_SEQS/3 sequences. The drafter's KV footprint costs ~25% of GPU KV regardless of K and remains a genuine cost of enabling spec decoding on 100k+ prompts." + - "The Inferact/Kimi-K3-DSpark draft is not pre-staged on the B300 cluster, so it is downloaded once to the launcher's writable models dir (/data/models, same placement as DeepSeek-V4-Pro-DSpark) and cached across the sweep. It must not go next to the target: Kimi-K3 is in STAGED_MODELS, so dirname(MODEL_PATH) is the read-only /scratch/models mount." + - "Config key uses the -dspark suffix rather than the repo-wide -mtp convention, to name the actual speculative method. Note the schema field stays spec-decoding: mtp (its Literal has no dspark member), so generated exp-names, artifacts and job labels still read _spec-mtp, and the benchmark script remains kimik3_fp4_b300_vllm_mtp.sh because the launcher derives its _mtp suffix from that field rather than from the key." + - "Serve flags stay on the values that ran 12/12 green in run 30326393603, with one exception adopted from recipes.vllm.ai/moonshotai/Kimi-K3: --attention-config mla_prefill_backend=TRTLLM_RAGGED (the non-DSpark sibling gets the same swap in #2385). Adopting the rest of the upstream Blackwell/NVIDIA block at the same time -- gpu-memory-utilization 0.95, max-num-seqs 32, max-model-len 1000000, max-num-batched-tokens 32768, --no-enable-flashinfer-autotune, VLLM_USE_V2_MODEL_RUNNER=1 -- killed the engine core on every completed concurrency (c1/c2/c4/c8, both KV arms, run 30378755933) about 13-16 min in during agentic warmup, with an assertion in the FlashInfer MoE runner's shared-experts output buffer (vllm/model_executor/layers/fused_moe/runner/shared_experts.py:165, assert self._output[self._output_idx] is None) firing on all 8 TP ranks at once. So max-num-seqs stays at 2*CONC and cudagraph capture enumerates 2*CONC token-multiples of (1+7)=8 rather than a fixed 32/256 ladder." + - "Measured acceptance does NOT reproduce the golden curve on this workload. A partial run (30337911387, 8 of 24 jobs before an unrelated enroot/pyxis flake cancelled the rest) recorded mean acceptance length 1.18-1.68 against the golden 3.84, per-position acceptance decaying 0.26 / 0.12 / 0.087 / 0.056 / 0.037 / 0.019 / 0.007 and an 8-10% average draft acceptance rate -- drafting 489-515 tok/s to accept 41-48. Throughput regressed 20-63% vs the non-MTP arms at conc 4 and 8. The spec config itself matches upstream exactly, and the draft supports the full 1M context (max_position_embeddings 1048576), so the gap is workload: the golden curve was collected on SPEED-Bench coding at max_model_len 16384 with short single-turn prompts, whereas agentic replays 118-230k-token multi-turn tool-calling traces. Treat the SPEED-Bench curve as non-transferable here and re-measure AL on the agentic corpus before trusting a level-7 configuration. Note rejection_sample_method=block is what exposed this: a synthetic_acceptance_length pinned to 3.84 would have reported a phantom speedup." + - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs." + - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2380 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic + description: + - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" + - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.90 (0.95 OOMs: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's pool on the first forward), --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" + - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 1/8/16/32" + - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so the launcher clones the functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch, which carries PR #278 plus the multi-node extension (vLLM-native --master-addr/--nnodes/--node-rank serve, headless non-leader ranks) for the 2-node TP8xPP2 topology" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" + - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" + - "Agentic experiment Variant G (of the #2359 direct-vllm Variant D): identical GPU-resident config plus VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0. Any positive value must be a multiple of Kimi-K3's KDA-hybrid scheduler_block_size 3145728 (the GB recipes' 32768 is hard-rejected at engine init, verified earlier on this PR family), so 0 is the only setting below one 3.1M-token scheduler block" + - "Agentic experiment Variant K (probe, of the #2374 Variant G): REMOVES the kimi-k3-container-deps.sh in-container patch (setup_script, script file, and launcher copy) to verify whether the mamba_hybrid index_fill_ dtype patch is still required by the current vllm/vllm-openai:kimi-k3 image — expected to fail at the first decode step if it is" + - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2391 + +- config-keys: + - minimaxm3-fp4-b200-vllm + - minimaxm3-fp4-b200-vllm-mtp + - minimaxm3-fp4-b300-vllm + - minimaxm3-fp4-b300-vllm-mtp + description: + - "Update vllm image, remove setting trtllm allreduce backend" + - "Use fp8 indexer cache dtype for the B200/B300 non-eagle configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2120 + +- config-keys: + - minimaxm3-fp4-b200-dynamo-vllm + description: + - "Add MiniMax-M3 NVFP4 B200 Dynamo-vLLM disaggregated benchmarks at 8k/1k." + - "Use configurations from concurrency 1 through 1024 with spread worker placement for CI transport validation." + - "Image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9; Dynamo: 1.3.0.dev20260710." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2230 + +- config-keys: + - minimaxm3-fp8-b300-dynamo-vllm + description: + - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 and dynamo to 1.3.0.dev20260710; drop the minimax-m3-vllm-fixes.sh setup patch (shipped in the nightly image)" + - "conc 4096: drop the 4p2d-dep2-tep4 arm; add a 3p2d-dep2-dep8 arm for conc [1024, 2048] (prefill 3xDEP2, decode 2xDEP8)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2338 + +- config-keys: + - minimaxm3-fp8-gb200-dynamo-vllm + description: + - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 and dynamo to 1.3.0.dev20260710; drop the minimax-m3-gb200-vllm-fixes.sh setup patch (shipped in the nightly image)" + - "Collapse every 2xDEP8 decode into a single DEP16 worker (decode_workers 2->1, gpus_per_decode 8->16, data-parallel-size 8->16); recipes renamed Xp2d-dep4-dep8 -> Xp1d-dep4-dep16 (1p2d/2p2d/3p2d/5p2d)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2339 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm + description: + - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 and dynamo to 1.3.0.dev20260710; drop the minimax-m3-gb300-vllm-fixes.sh setup patch (shipped in the nightly image)" + - "Collapse the 2xDEP8 decode into a single DEP16 worker (decode_workers 2->1, gpus_per_decode 8->16, data-parallel-size 8->16); recipe renamed 3p2d-dep2-dep8 -> 3p1d-dep2-dep16 (conc 512). The remaining decodes are 1xDEP8 or TEP and are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2340 + +- config-keys: + - minimaxm3-fp8-b200-vllm-mtp + - minimaxm3-fp8-b300-vllm-mtp + description: + - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9; set --stream-interval 32; consolidate --attention-config with indexer_kv_dtype fp8; drop the sparse_attention_msa contiguity patch (shipped in the nightly image)" + - "EAGLE3 draft head Inferact/MiniMax-M3-EAGLE3 -> Inferact/MiniMax-M3-EAGLE3-GQA; speculative attention_backend TRITON_ATTN -> FLASH_ATTN; num_speculative_tokens kept at 3" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2337 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" + - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" + - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1; runner: gb300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang-mtp + description: + - "Add Qwen3.5-397B-A17B-FP8 GB300 disaggregated Dynamo-SGLang MTP configurations for 8k/1k." + - "Cover 1P1D TP4/TP4, 1P1D TEP8/TEP8, and 3P/4P/6P/7P/8P DEP4/DEP16 topologies." + - "Enable chat-formatted benchmark inputs for every MTP recipe." + - "Use Mooncake KV transfer and lmsysorg/sglang:v0.5.14-cu130." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2316 + +- config-keys: + - dsv4-fp4-b300-dynamo-sglang + description: + - "Add DSV4 B300 disaggregated Dynamo SGLang STP configuration for 8k/1k" + - "Add 5 checked-in srt-slurm recipes selected one-to-one by the STP search space" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2362 + +- config-keys: + - dsv4-fp4-b300-dynamo-sglang-mtp + description: + - "Add DSV4 B300 disaggregated Dynamo SGLang MTP configuration for 8k/1k" + - "Add 8 checked-in srt-slurm recipes with EAGLE speculative decoding and chat-template inputs" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2363 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Update the GB200 DeepSeek-V4-Pro Dynamo-vLLM stack to Dynamo 1.3.0.dev1, vLLM 0.25.1, and the MXFP4 checkpoint" + - "Enable multiple frontends and MessagePack request-plane encoding, and reduce 1P4D decode GPU memory utilization to 0.85" + - "Drop the recipe setup_script entirely (removes the DeepGEMM NVLink barrier timeout patch); required container deps now ship in vLLM 0.25.1, on NVIDIA/srt-slurm v1.0.31" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2139 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + description: + - "Add the DSpark speculative-decoding variant of kimik3-fp4-b300-vllm-agentic: Kimi-K3 (MXFP4, 2.8T MoE, KDA/MLA hybrid, 1M native context) on B300 with vLLM, agentic-coding scenario only -- no fixed-seq-len (1k1k / 8k1k) arms." + - "Speculative config: method dspark on the Inferact/Kimi-K3-DSpark draft head, num_speculative_tokens 2, attention_backend FLASHINFER_MLA, draft_sample_method probabilistic, with rejection_sample_method synthetic and synthetic_acceptance_length 2.51 -- the committed golden AL at K=2 on the probabilistic curve (golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml). This follows the AgentX policy in golden_al_distribution/README.md: a submission may choose any supported draft length but may not substitute a different acceptance target, because InferenceX measures inference-system performance rather than the fitness of a benchmark-specific speculative head. The greedy/standard curve gives 2.45 at K=2; the curves are not mixed." + - "EVAL_ONLY runs switch to real block verification. Synthetic acceptance commits drafted tokens regardless of the target logits, so generated text is wrong and SWE-bench scores 0.0000 -- the same split dsv4_fp4_b300_vllm_mtp.sh makes, and which kimik2.5_fp4_b300_mtp.sh omits; dsv4 is the pattern followed here." + - "Level 2 rather than 7. Draft length is the submission's choice and verify width is a real system cost synthetic acceptance does not remove: at K=7 the target verifies 8 positions per step, which loses to no-speculation once the GPU saturates even at the forced golden AL of 3.84 (K=2 clears break-even at both ends of the cost range; K=3 is a wash at saturation; K=7 is negative). cudagraph capture consequently enumerates multiples of (1+2)=3, since vLLM rounds configured sizes to multiples of (1+N) and would otherwise cover only MAX_NUM_SEQS/3 sequences. The drafter's KV footprint costs ~25% of GPU KV regardless of K and remains a genuine cost of enabling spec decoding on 100k+ prompts." + - "The Inferact/Kimi-K3-DSpark draft is not pre-staged on the B300 cluster, so it is downloaded once to the launcher's writable models dir (/data/models, same placement as DeepSeek-V4-Pro-DSpark) and cached across the sweep. It must not go next to the target: Kimi-K3 is in STAGED_MODELS, so dirname(MODEL_PATH) is the read-only /scratch/models mount." + - "Config key uses the -dspark suffix rather than the repo-wide -mtp convention, to name the actual speculative method. Note the schema field stays spec-decoding: mtp (its Literal has no dspark member), so generated exp-names, artifacts and job labels still read _spec-mtp, and the benchmark script remains kimik3_fp4_b300_vllm_mtp.sh because the launcher derives its _mtp suffix from that field rather than from the key." + - "Serve flags stay on the values that ran 12/12 green in run 30326393603, with one exception adopted from recipes.vllm.ai/moonshotai/Kimi-K3: --attention-config mla_prefill_backend=TRTLLM_RAGGED (the non-DSpark sibling gets the same swap in #2385). Adopting the rest of the upstream Blackwell/NVIDIA block at the same time -- gpu-memory-utilization 0.95, max-num-seqs 32, max-model-len 1000000, max-num-batched-tokens 32768, --no-enable-flashinfer-autotune, VLLM_USE_V2_MODEL_RUNNER=1 -- killed the engine core on every completed concurrency (c1/c2/c4/c8, both KV arms, run 30378755933) about 13-16 min in during agentic warmup, with an assertion in the FlashInfer MoE runner's shared-experts output buffer (vllm/model_executor/layers/fused_moe/runner/shared_experts.py:165, assert self._output[self._output_idx] is None) firing on all 8 TP ranks at once. So max-num-seqs stays at 2*CONC and cudagraph capture enumerates 2*CONC token-multiples of (1+7)=8 rather than a fixed 32/256 ladder." + - "Measured acceptance does NOT reproduce the golden curve on this workload. A partial run (30337911387, 8 of 24 jobs before an unrelated enroot/pyxis flake cancelled the rest) recorded mean acceptance length 1.18-1.68 against the golden 3.84, per-position acceptance decaying 0.26 / 0.12 / 0.087 / 0.056 / 0.037 / 0.019 / 0.007 and an 8-10% average draft acceptance rate -- drafting 489-515 tok/s to accept 41-48. Throughput regressed 20-63% vs the non-MTP arms at conc 4 and 8. The spec config itself matches upstream exactly, and the draft supports the full 1M context (max_position_embeddings 1048576), so the gap is workload: the golden curve was collected on SPEED-Bench coding at max_model_len 16384 with short single-turn prompts, whereas agentic replays 118-230k-token multi-turn tool-calling traces. Treat the SPEED-Bench curve as non-transferable here and re-measure AL on the agentic corpus before trusting a level-7 configuration. Note rejection_sample_method=block is what exposed this: a synthetic_acceptance_length pinned to 3.84 would have reported a phantom speedup." + - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs." + - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418 + +- config-keys: + - qwen3.5-fp4-rtx6000pro-sglang + scenario-type: + - fixed-seq-len + description: + - "Add Qwen3.5-397B-A17B NVFP4 single-node SGLang benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner" + - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention)" + - "Pin SGLang v0.5.16-cu130, the first tag whose auto backend resolution knows that trtllm-gen MoE is SM100-only and routes modelopt_fp4 experts to FlashInfer CUTLASS on SM120" + - "Use FlashInfer CUTLASS for both the NVFP4 GEMMs and the routed experts and the FlashInfer attention backend on SM120, with FP8 KV cache, bf16 Mamba SSM state, the no_buffer Mamba scheduler strategy, radix cache disabled, and decode CUDA graph capture capped to each tested concurrency" + - "Use standard NCCL collectives on the PCIe topology (custom all-reduce disabled explicitly) and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM" + - "Deviate from the B300 SGLang recipe where 96 GiB per GPU forces it: mem-fraction-static 0.7 with a 2-request prefill chunk (0.8 with a 32k chunk sized the KV pool at 2.2M tokens and OOM'd on the first 8k prefill), and a single tokenizer worker (the B300 recipe's six each pin a ~0.7 GiB CUDA context on GPU 0)" + - "Teach the RTX PRO 6000 launcher to prefer a framework-tagged benchmark script so vLLM and SGLang recipes for the same model can coexist" + - "Validated on the node through the real launcher at all four corners of the sweep, every request successful: TEP4 concurrency 64 640/640 at 6198 tok/s, TP4 concurrency 64 640/640 at 5999 tok/s, TEP4 concurrency 1 10/10 at 672 tok/s, TP4 concurrency 1 10/10 at 658 tok/s" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 + +- config-keys: + - qwen3.5-fp4-rtx6000pro-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Add an MTP arm to the Qwen3.5-397B-A17B NVFP4 RTX PRO 6000 SGLang sweep, driving the checkpoint's built-in MTP draft head through SGLang's EAGLE path with 3 steps, top-k 1, and 4 draft tokens" + - "Sweep the same 8k/1k grid as the non-MTP config: concurrency 1, 4, 16, and 64 for both TP4 and TEP4" + - "Raise mem-fraction-static to 0.80 for this arm: the fraction bounds weights plus the KV and Mamba pools, so the 4.05 GiB per-rank draft head has to be paid for by raising it, not lowering it (0.65 and 0.75 both left the pools empty on this 96 GiB SKU, and 0.85 came up with only 4.56 GiB free per rank)" + - "Size max-running-requests to the tested concurrency instead of a fixed 128, since the client never opens more connections and the oversized Mamba state pool is what starved the draft head" + - "Pass --use-chat-template to the benchmark, as required for EAGLE-style speculative decoding" + - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + description: + - "Add Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration." + - "Add seven checked-in srt-slurm recipes selected one-to-one by the 8K/1K prefill/decode topology and concurrency entries." + - "Image: vllm/vllm-openai:v0.25.1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Set the vLLM ROCm image to the immutable vllm/vllm-openai-rocm:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 build (the latest official ROCm nightly, commit-pinned), which ships the AITER MXFP4 MoE bump (>= v0.1.16.post5 from vllm-project/vllm#48683) including the ROCm/aiter#3832 gfx950 MXFP4 MoE backend" + - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" + - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - qwen3.5-fp4-b300-sglang-agentic-mtp + description: + - "Add Qwen3.5-397B-A17B NVFP4 AgentX benchmark on B300 with SGLang native NEXTN MTP" + - "Use the 256k trace dataset and golden synthetic acceptance length 3.39" + - "Use the shared AIPerf watchdog to cap whole-trajectory runtime idle gaps at 300 seconds" + - "Image: lmsysorg/sglang:v0.5.16-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2421 + +- config-keys: + - qwen3.5-fp4-b200-sglang-agentic-mtp + description: + - "Add Qwen3.5-397B-A17B NVFP4 AgentX benchmark on B200 with SGLang native NEXTN MTP" + - "Use the 256k trace dataset and golden synthetic acceptance length 3.39" + - "Use the shared AIPerf watchdog to cap whole-trajectory runtime idle gaps at 300 seconds" + - "Image: lmsysorg/sglang:v0.5.16-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 + + +- config-keys: + - minimaxm3-fp4-b200-dynamo-vllm + description: + - "Bump vLLM nightly image to nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 (fixed-len MiniMax-M3 fix); update all B200 FP4 8k1k recipe YAML container fields to match" + - "Enable VLLM_MINIMAX_M3_MSA_DECODE_BACKEND=cutlass in prefill and decode environments" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2472 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1 + - minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1-tep4 + description: + - "Bump vLLM nightly image to nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 (fixed-len MiniMax-M3 fix); update B300 FP4 8k1k tp1 recipe YAML container fields to match" + - "Enable VLLM_MINIMAX_M3_MSA_DECODE_BACKEND=cutlass in prefill and decode environments" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2473 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm-mtp + description: + - "Bump vLLM nightly image to nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 (fixed-len MiniMax-M3 fix); update B300 FP4 MTP recipe YAML container fields to match (excluding 2p1d-dep2-dep4 used by legacy-dep4)" + - "Enable VLLM_MINIMAX_M3_MSA_DECODE_BACKEND=cutlass in prefill and decode environments" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2471 + +- config-keys: + - minimaxm3-fp4-b300-dynamo-vllm-mtp-legacy-dep4 + description: + - "Bump vLLM nightly image to nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 (fixed-len MiniMax-M3 fix); update 2p1d-dep2-dep4 recipe YAML container to match" + - "Enable minimax_m3_msa_decode_backend=cutlass in decode attention-config" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2483 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Bump vLLM nightly image to nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 (fixed-len MiniMax-M3 fix)" + - "Enable VLLM_MINIMAX_M3_MSA_DECODE_BACKEND=cutlass" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2470 + +- config-keys: + - minimaxm3-fp4-b200-vllm + description: + - "Bump vLLM nightly image to nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 (fixed-len MiniMax-M3 fix)" + - "Enable VLLM_MINIMAX_M3_MSA_DECODE_BACKEND=cutlass" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2468 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm + scenario-type: + - fixed-seq-len + description: + - "Extend the search space to include the TP2EP1 configuration for fixed seq len 8k1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2446 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + description: + - "Add single-node agentic-coding recipe for Kimi-K3 (MXFP4, 2.8T MoE, 896 experts, KDA/MLA hybrid, 1M native context) on MI355X (gfx950) with vLLM + DSpark MTP speculative decoding. Agentic-coding scenario only -- no fixed-seq-len (1k1k / 8k1k) arms." + - "TP8 only: the ~1.56 TB MXFP4 checkpoint is ~195 GB/GPU across 8 gfx950 288 GB parts; TP4 would need ~390 GB/GPU and cannot load, matching upstream strategy_min_gpus." + - "In-container filesystem patches applied via an idempotent git-apply wrapper (apply_k3_container_patches.sh): KDA fused_recurrent state_indices coercion (PIECEWISE cudagraph boot), triton_mla UNIFORM_BATCH cudagraph support for the non-causal DSpark draft group, rocm_aiter_mla small-head (<16 heads at TP8) multi-token MTP-verify paged-KV buffers (vLLM PR #50619 path), and aiter mla_gluon int64 KV stride on the >2GB path (aiter PR #4474); plus Triton 3.7.0 (ROCm 7.2.0) + tabulate for Gluon MLA PaddedSharedLayout." + - "Piecewise cudagraph capture: compilation-config mode 3 with an explicit even-step cudagraph_capture_sizes list up to 2*CONC*(1+SPEC_NUM_TOKENS) (6*CONC at spec=2), no cudagraph_mode. Capture range is decoupled from --max-num-seqs (1*CONC) so it covers the DSpark verify batch (each decode slot expands to 1+num_speculative_tokens rows) even though the served slot cap is lower." + - "Speculative config: method dspark on the Inferact/Kimi-K3-DSpark draft head, num_speculative_tokens 2, attention_backend TRITON_MLA, draft_sample_method probabilistic, rejection_sample_method synthetic with synthetic_acceptance_length 2.45." + - "Serve flags: --moe-backend auto, --load-format fastsafetensors, --language-model-only, --kv-cache-dtype auto (bf16 KV), --enable-prefix-caching, gpu-memory-utilization 0.9, kimi_k3 reasoning/tool-call parsers, max-model-len 1048576. Env: VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1, AITER_BF16_FP8_MOE_BOUND=0, VLLM_USE_BREAKABLE_CUDAGRAPH=0, SAFETENSORS_FAST_GPU=1." + - "Search space is one TP8 GPU-resident arm (kv-offloading none) at conc-list [4, 8, 10, 12, 14, 16]; the LMCache/vllm-native/SimpleCPUOffload DRAM-offload backends are wired in the recipe but off this sweep's ladder." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2508 + +- config-keys: + - minimaxm3-fp4-b200-dynamo-vllm-mtp + description: + - "Add MiniMax M3 NVFP4 B200 disagg EAGLE3 MTP recipes, mirrored from B300" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2462 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - "Add MiniMax-M3-MXFP8 GB300 8k1k disaggregated Dynamo-vLLM EAGLE3 MTP Pareto on vLLM nightly 5e35a6f4f" + - "The six dep8 decode configurations enable the CUTLASS MSA sparse decode backend and FULL_DECODE_ONLY cudagraph mode; the five low-concurrency c1/c4/c8 configurations use neither" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2486 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-disagg + description: + - "Update the MiniMax-M3 MXFP4 MI355X vLLM disagg image to vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 (from rocm/vllm-dev:vllm-0.23.1-rocm723-mi35x-mori-0625) for AITER MoE and shared-expert fusion support." + - "Export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 for MiniMax-M3-MXFP4 (both prefill and decode)." + - "Enable prefill-only INT4 quick-reduce: set VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 and VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 on the prefill workers via a new prefill_env channel (mirrors the existing decode_env path in server_vllm.sh)." + - "Cap the 1P1D TP4 concurrency sweep at 256 (was 512); drop the 2P1D TP4 layout (128/256/512) as it is CI-flaky with negligible curve impact." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1943 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add five MiniMax-M3 NVFP4 B300 single-node aggregate vLLM AgentX points: TP8 concurrency 1; TP4 concurrency 1, 2, and 16; TP2 concurrency 2." + - "Use the Inferact/MiniMax-M3-EAGLE3-GQA draft model with three speculative tokens and synthetic rejection sampling; thinking-on synthetic acceptance length is 2.78." + - "Use vllm/vllm-openai:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 with prefix caching enabled, default KV-cache dtype, FlashInfer TRT-LLM attention with FP8 indexer KV, and the EAGLE3 drafter on FLASH_ATTN." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2328 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm + description: + - "Decode: gpu-memory-utilization 0.85->0.9, add max-model-len 9280, disable NCCL symmetric memory, add no-enable-flashinfer-autotune, drop stream-interval (unsupported by rust frontend)" + - "Prefill: gpu-memory-utilization 0.9->0.95, add max-model-len 9280, add max-num-seqs 16 + max-num-batched-tokens 32768, enable rust frontend, enable randomize-dp-dummy-inputs" + - "EPP: switch from max-score-picker to weighted-random-picker for better load distribution under high concurrency" + - "Low-latency recipe: same prefill optimizations, decode NCCL symmetric memory disabled + rust frontend enabled, drop stream-interval" + - "Sweep: drop low-middle point (c256/512, 1P×4D) from llmd-vllm key; low-latency recipe runs at c1 only" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2498 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2433 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-mtp + description: + - "Add Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated TensorRT-LLM+Dynamo MTP (3-token) benchmarks for 8k1k." + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 with Dynamo installed at runtime (version 1.3.0.dev20260708, TCP request plane)." + - "9 MTP configs: 1P1D DEP2+1D TEP16 (conc 48), 1P4D DEP2+4D TP16 (conc 4), 1P7D DEP2+7D TEP8 (conc 42), 2P15D DEP2+15D TP4 (conc 90), 3P4D DEP2+4D TEP8 (conc 192), 3P7D DEP2+7D TEP8 (conc 168), 4P1D DEP2+1D DEP16 (conc 384), 6P5D DEP2+5D TEP8 (conc 480), 9P1D DEP2+1D DEP16 (conc 1536)." + - "Runner: launch_gb300-nv.sh updated to clone NVIDIA/srt-slurm@v1.0.29 and overlay local recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/ for the dynamo-trt+qwen3.5+fp4 path; --no-preflight added for model on compute-node /scratch." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2218 + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + description: + - "Run the Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration on the b200-new runner." + - "Remove the 1P/1D DEP4-to-DEP8 concurrency-1024 configuration." + - "Image: vllm/vllm-openai:v0.25.1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2438 + +- config-keys: + - dsr1-fp4-b200-sglang + - dsr1-fp4-b200-sglang-mtp + description: + - "Update the aggregated DSR1 FP4 B200 SGLang STP and MTP configurations to v0.5.16 with CUDA 13" + - "Image: lmsysorg/sglang:v0.5.16-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2492 + +- config-keys: + - dsv4-fp4-b200-dynamo-vllm-mtp + description: + - "Add DeepSeek-V4-Pro FP4 B200 disaggregated Dynamo vLLM MTP benchmarks for 8k/1k" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2390 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Bump SGLang image to lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 for the 8k1k scenarios." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2099 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + description: + - "Add the GLM-5.2 NVFP4 B300 AgentX (agentic-coding) recipe with EAGLE/MTP speculative decoding, routed to benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang_mtp.sh. This is the arm that replaces glm5.2-fp4-b300-sglang-agentic once agentic non-MTP is deprecated for GLM-5.2 on 2026-08-03 (MODELS.md); the STP entry is left untouched here so the two can be compared before the cut-over." + - "Speculative config is GLM-5.2's built-in nextn head via SGLang EAGLE (no external draft model): --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4, i.e. 3 speculative tokens per verification step. Same shape as the GLM-5.2 GB300 dynamo-sglang agentic recipes." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.99 (GLM-5.2 MTP curve at K=3, thinking_on), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang. EVAL_ONLY runs leave simulated acceptance off and keep real verification, since simulated acceptance commits drafted tokens regardless of the target logits and would score the eval at ~0." + - "Commits that curve as golden_al_distribution/glm5.2_mtp.yaml (SPEED-Bench coding, output-len 4096, temperature 1.0, top_p 0.95, thinking_on, num_speculative_tokens 1-8), verbatim from the speedbench-al.yml artifact of run 28058352479, and adds the GLM-5.2 row to golden_al_distribution/README.md and README_zh.md. The curve was collected on the FP8 checkpoint; one curve is committed per model, and the NVFP4 checkpoint ships the same nextn head." + - "Image bumped from lmsysorg/sglang:v0.5.15.post1-cu130 (the STP entry) to lmsysorg/sglang:v0.5.16-cu130. SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 -- SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back, so an older image would silently ignore the token-mode half of the AgentX contract -- and v0.5.16-cu130 is the tag the GLM-5.2 GB300 dynamo-sglang agentic recipes already run." + - "Serve shape is the validated STP script unchanged outside the speculative and simulated-acceptance blocks: cookbook B300 NVFP4 low-latency arm (TP8, --kv-cache-dtype fp8_e4m3, --bf16-gemm-backend cutedsl, --max-prefill-tokens 8192, --chunked-prefill-size 8192, --mem-fraction-static 0.85, glm47 tool-call parser, glm45 reasoning parser) with HiCache host-DRAM offload at hicache-ratio 0.75 / write_back / direct / page_first_direct. mem-fraction-static stays at 0.85, which is also what the GLM-5.2 GB300 aggregated MTP recipe uses; the hicache ratio is relative to the device pool, so the nextn layer's own KV only shrinks the host pool." + - "Search space is one arm on the AgentX MTP concurrency grid: TP8 + HiCache at conc [1, 4, 8, 12, 16]. Steps of at least 2 because single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and a hard stop at conc 16. The STP entry's DEP (attention-DP + EP8) throughput arm is not carried over: its frontier peak is conc 48, above that cap, and at conc <= 16 attention-DP leaves 2 sessions per rank and is strictly dominated by TP8, so it would spend one GPU job per point re-measuring a worse curve. The DEP branch is kept in the benchmark script -- including the --speculative-moe-a2a-backend none / --speculative-moe-runner-backend triton pair that GLM-5.2-NVFP4's unquantized bf16 nextn layer needs once expert parallelism puts an all-to-all in the MoE path -- so the throughput arm can be added later without re-deriving it." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2447 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Refresh the Qwen3.5 GB300 AgentX frontier after aiperf 381758a restored real conversation pacing; replace the stale nine-point sweep with a complete fourteen-point locally validated curve: disaggregate TP4/TP4 c8/c16/c32/c64/c96/c128, disaggregate TP2/TP2 c72, and aggregate TP2 c1/c24/c32/c40/c48/c52/c64." + - "The 30-minute local re-sweep covered low, mid, and high concurrency across aggregate TP2, disaggregate TP2/TP2, and disaggregate TP4/TP4. All 33 new-client steady-state samples completed with zero request errors and zero retract events; using the same historical output-token-throughput-per-user p50 metric as the old curve, the fourteen submitted points are the strict non-dominated set. The InferenceMAX sweep keeps the canonical 60-minute AgentX benchmark duration." + - "Align both configurations on the 2026-07-24 SGLang container, current aiperf be758d6, the merged Dynamo #12081 commit 5a638087, X-Dynamo-Session-ID affinity with the removed legacy conv-aware CLI path explicitly disabled, and NVIDIA/srt-slurm v1.0.38 with logical worker metric endpoint injection." + - "Enable SGLang backend metrics on every aggregate, prefill, and decode engine, and fail before publishing a partial trace artifact if required sglang: metrics are absent." + - "Record the recipes' active HiCache host-DRAM tier and, for disaggregated points, Dynamo router commit 5a638087 in nvidia-master metadata so generated artifacts no longer report kv_offloading=none, allocated_cpu_dram_gb=0, or a null router." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Capture the vLLM backend Prometheus endpoint for every topology and require vllm: metrics before publishing AgentX results." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2517 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add a DeepSeek-V4-Pro FP4 MI355X vLLM AgentX MTP submission with three speculative tokens." + - "Pin throughput to the committed DeepSeek-V4-Pro golden synthetic acceptance length of 2.49; eval-only runs retain real target verification." + - "Use the immutable vllm/vllm-openai-rocm:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c image because the base submission's retired nightly tag is no longer published; rely on its upstream scheduler instead of replacing scheduler.py out of tree." + - "Explicitly pass the vLLM backend metrics endpoint to AIPerf for every topology and require vllm: metrics before publishing AgentX results." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2522 + +- config-keys: + - kimik3-fp4-h200-vllm-agentic + description: + - "Add Day-0 Kimi K3 FP4 AgentX H200 performance recipes from the official vLLM guidance, using fit-safe TP/DP/EP topologies, optional SimpleCPU KV offload, and a five-minute reconstructed trace idle-gap cap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2353 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic + description: + - "Add Kimi K3 MXFP4 GB200 AgentX performance points following the official vLLM recipe: TP16 (conc 1-8), TEP16 (conc 8-32), and TP4 x DP4 = EP16 (conc 32-256). https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200" + - "Test vLLM Simple CPU KV offloading on the Kimi K3 GB200 DEP16 throughput profile at AgentX concurrency 128, 192, 256, and 384." + - "Correct the pure TP16 result identity to EP1 so it remains distinct from the TEP16 point at concurrency 8." + - "Use the upstream vllm/vllm-openai:kimi-k3 image with the pinned Kimi K3-enabled Dynamo 1.3.0 source build installed in-container, and enable Kimi K3 DSpark level 2 with the prescribed 2.51 golden acceptance length." + - "Use FlashInfer TRT-LLM MoE for DSpark DEP to avoid DeepGEMM grid-barrier failures under MRV2." + - "Load DEP weights with standard safetensors to avoid fastsafetensors transient HBM exhaustion with FlashInfer MoE." + - "Size DEP CUDA-graph captures to the actual per-engine frontier concurrency (64 regular, 96 offload), using even-sequence captures for the high-concurrency offload arm to preserve the full 1M context while leaving FlashInfer runtime workspace." + - "Pin the Qwen B200/B300 AgentX AIPerf warmup-handoff revision, cap recorded per-trace idle gaps at 300 seconds, report vLLM cached prompt tokens, and let low-concurrency TP collect a larger live error sample before applying the unchanged 10% final validity gate." + - "Allow high-concurrency full-context AgentX jobs to complete the canonical warmup and one-hour profile with a 12-hour Slurm limit and 13-hour workflow envelope." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2404 + +- config-keys: + - glm5.1-fp8-b200-tilert + description: + - "Add GLM-5.1 FP8 B200 prefill/decode-disaggregated benchmark via TileRT: vLLM prefill (TileRTConnector, kv_producer) + TileRT decode_server + OpenAI-compatible pd_router, framework=tilert (non-dynamo custom runtime, decode-only engine)" + - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" + - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" + - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B FP8 AgentX on B200 with SGLang native NEXTN MTP" + - "Mirror the FP4 AgentX MTP concurrency and HiCache grids, scaling TP2 to TP4 and TP4 to TP8 for the larger FP8 checkpoint" + - "Use golden synthetic acceptance length 3.39" + - "Image: lmsysorg/sglang:v0.5.16-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2514 + +- config-keys: + - qwen3.5-fp8-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B FP8 AgentX on B300 with SGLang native NEXTN MTP" + - "Mirror the FP4 AgentX MTP TP4 and TP2/EP2 concurrency grids, including the TP2/EP2 HiCache arm" + - "Use golden synthetic acceptance length 3.39" + - "Image: lmsysorg/sglang:v0.5.16-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2515 + + +- config-keys: + - qwen3.5-fp8-h100-sglang-agentic-mtp + description: + - "Add the MTP speculative-decoding (spec-decoding: mtp) variant of qwen3.5-fp8-h100-sglang-agentic: Qwen3.5-397B-A17B FP8 on H100 with SGLang, agentic-coding scenario only, routed via spec-decoding=mtp to benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh." + - "Speculative config mirrors fixed_seq_len/qwen3.5_fp8_h100_mtp.sh: SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4 -- 3 speculative tokens per verification step." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the exact triple golden_al_distribution/README.md prescribes for SGLang and the same AL the GB300 Qwen3.5 AgentX srt-slurm recipes pin. EVAL_ONLY runs leave simulated acceptance off and keep real verification, since committing drafted tokens regardless of the target logits would score the eval at ~0." + - "Image moves from the non-MTP agentic sibling's lmsysorg/sglang:v0.5.12-cu130 to v0.5.16-cu130: SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 (SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back), so an older image would silently ignore the token-mode half of the AgentX contract. v0.5.14 is already the version the fixed-seq-len H100 Qwen3.5 MTP entry runs, so this is one minor version beyond an exercised build." + - "Serve shape is otherwise identical to the non-MTP agentic sibling (SGLang HiCache host-DRAM offload with page-size 64 / kernel IO / page_first layout, flashinfer attention on sm_90, --quantization fp8, --kv-cache-dtype fp8_e4m3, --mamba-ssm-dtype bfloat16, --mem-fraction-static 0.75, qwen3 reasoning / qwen3_coder tool-call parsing inherited from the shared agentic replay path, the multi_tokenizer cached_tokens_details patch), so the spec-decode delta is readable at equal concurrency." + - "Search space now runs on the AgentX MTP concurrency grid: the GPU-resident row [1, 4, 8, 12, 16] and the HiCache row [4, 8, 12, 16]. Every step is at least 2 concurrency apart and every arm stops hard at conc 16 -- single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and past conc 16 these SKUs are into the post-HBM-cliff thrashing regime that the non-MTP sweeps already characterized, which is not worth one GPU job per point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2427 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Kimi K2.5 NVFP4 B200 vLLM: nightly image, TP/DEP/TEP sweep, DEP gmu 0.80" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2512 + +- config-keys: + - kimik2.5-fp4-b300-vllm + description: + - "Kimi K2.5 NVFP4 B300 vLLM: nightly image, TP/DEP/TEP sweep, TP8 conc-1 only, DEP gmu 0.85" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2513 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm + description: + - "Sweep: drop 2P×1D DEP8 conc=4096; j-llm-d characterized 2P at c512/1024/2048 with Pareto at c1024, and c4096 hit DeepGEMM NVLink barrier timeouts under sustained load" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2498 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the MiniMax-M3 NVFP4 B200 AgentX (agentic-coding) recipe: nvidia/MiniMax-M3-NVFP4 on single-node vLLM with EAGLE3-GQA speculative decoding, routed to benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B, because synthetic acceptance already makes spec-decode results comparable across submissions. This is MiniMax-M3's only active B200 configuration: the Single-turn 8k1k scenario that carried the previous B200 MiniMax-M3 curves (minimaxm3-fp4-b200-vllm, minimaxm3-fp4-b200-vllm-mtp) was deprecated and archived under configs/deprecated/ in #2493." + - "Speculative config is the Inferact/MiniMax-M3-EAGLE3-GQA draft head at three speculative tokens with the drafter pinned to FLASH_ATTN, matching the B300 sibling minimaxm3-fp4-b300-vllm-agentic-mtp. Throughput runs pin vLLM synthetic rejection sampling to the committed golden AL: rejection_sample_method=synthetic with synthetic_acceptance_length 2.78, the thinking_on K=3 entry of golden_al_distribution/minimaxm3_eagle3_gqa.yaml. EVAL_ONLY runs drop synthetic acceptance and keep real target verification, since synthetic acceptance bypasses verification and zeroes the eval score." + - "Image vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 -- the tag the B300 AgentX MTP sibling runs, and the same tag the deprecated minimaxm3-fp4-b200-vllm 8k1k entry was bumped to in #2468, so it is already proven on this model, precision and SKU." + - "Engine flags are the B300 script verbatim so the two SKU curves stay comparable: FlashInfer TRT-LLM attention with FP8 indexer KV, --kv-cache-dtype fp8, --block-size 128 (mandatory for the MSA sparse/index cache), --language-model-only, --enable-prefix-caching, --max-cudagraph-capture-size 512, --max-num-batched-tokens 16384, --stream-interval 20, the minimax_m3 reasoning and tool-call parsers, and thinking_mode enabled. DRAM offload arms use vLLM's SimpleCPUOffloadConnector in lazy mode. --gpu-memory-utilization stays 0.9 (the value both the B300 sibling and the deprecated 8k1k B200 NVFP4 MTP recipe ran) and is now overridable via GPU_MEMORY_UTILIZATION for bring-up debugging." + - "First B200 delta: the checkpoint-resolution guard. runners/launch_b200-dgxc.sh resolves the checkpoint to /scratch/fsw/models/MiniMax-M3-NVFP4 and then rewrites MODEL to that path, so `hf download \"$MODEL\"` cannot work there the way it does on b300-nv, where MODEL stays the HF repo id. The script keeps the repo id in HF_MODEL_ID, verifies the checkpoint by walking model.safetensors.index.json rather than trusting an ls -A emptiness test, and serializes any download behind a flock so the concurrencies of one sweep do not race as writers on the shared path." + - "Second B200 delta: draft staging. The B300 sibling stages the EAGLE3-GQA head under /data/models, which does not exist on b200-dgxc; the launcher bind-mounts only $MODEL_PATH itself, so the draft is staged in the container-local parent directory instead of polluting the shared checkpoint directory. Same approach the deprecated 8k1k B200 MiniMax-M3 MTP recipe used." + - "Search space is TP4-only: TP4 GPU-resident conc [1, 2, 5, 8, 10, 12, 15, 20] and TP4 SimpleCPU conc [20, 30, 40]. The 250 GB NVFP4 checkpoint leaves roughly 398 GB of KV at TP4 on B200's 180 GB HBM3e against 787 GB on B300, so the host tier picks up one step earlier than the B300 sibling. Same B200/B300 asymmetry the qwen3.5 NVFP4 AgentX MTP pair already publishes." + - "The B300 sibling's TP2 arm is dropped rather than ported: TP2 does not fit on B200. With the checkpoint resident, the KV pool is 31.72 GiB at --gpu-memory-utilization 0.9 while a single request at the model's 1,048,576-token max_model_len needs 37.62 GiB, so vLLM raises in _check_enough_kv_cache_memory and the engine never reaches serving. Observed on run 31141913741, where both TP2 cells (c1 and c2) died identically at engine init while every TP4 cell passed. The failure is structural, not a concurrency cliff -- vLLM estimates the max model length at TP2 as 883,840, below the agentic corpus context -- so no conc-list would have made the arm run." + - "The whole recipe stops at concurrency 40. The B300 sibling's SimpleCPU arm runs to 75, but on B200 everything past 40 is the host tier absorbing a working set that never fit in the ~398 GB TP4 pool to begin with, so that tail traces offload bandwidth rather than the SKU and costs one engine start per point to do it. Cutting it drops the recipe from 18 configs to 13." + - "dram-utilization 0.683 is carried over unchanged from the B300 entry and resolves to the same 1,024 GB engine-level CPU KV budget at TP4: b200-dgxc and b300-nv both report more installed DRAM than MAX_AGENTIC_AVAILABLE_CPU_DRAM_MIB, so both clamp to the same 3 TB AgentX ceiling before the proportional-GPU rule applies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2511 + +- config-keys: + - glm5.2-fp4-b200-sglang-agentic-mtp + description: + - "Add the first GLM-5.2 AgentX (agentic-coding) recipe on B200: GLM-5.2-NVFP4 with SGLang, spec-decoding=mtp only, routed to benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B, because synthetic acceptance already makes spec-decode results comparable across submissions." + - "Speculative config is GLM-5.2's built-in nextn head via SGLang EAGLE (no external draft model): --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4, i.e. 3 speculative tokens per verification step. Same shape as the B300 sibling and the GLM-5.2 GB300 dynamo-sglang agentic recipes." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.99 (GLM-5.2 MTP curve at K=3, thinking_on, golden_al_distribution/glm5.2_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang. EVAL_ONLY runs leave simulated acceptance off and keep real verification. The golden curve file itself is committed by the B300 sibling PR #2447, which should merge first." + - "Image lmsysorg/sglang:v0.5.16-cu130. SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 (SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back), so an older image would silently ignore the token-mode half of the AgentX contract; v0.5.16-cu130 is also the tag the B300 GLM-5.2 MTP entry and the GLM-5.2 GB300 dynamo-sglang agentic recipes run." + - "Serve shape is the B300 GLM-5.2 MTP script with two B200 deltas. --mem-fraction-static drops from 0.85 to 0.83 (overridable via MEM_FRACTION_STATIC): B200's 180 GB HBM3e leaves only 27 GB of non-static headroom at 0.85 against B300's 43 GB, and the EAGLE draft head's verification-batch activations and the extra CUDA-graph capture at 4 draft tokens come out of that pool on top of GLM-5.2's DSA indexer temporaries; 0.83 restores ~31 GB. Everything else is unchanged -- cookbook low-latency TP8 arm (--kv-cache-dtype fp8_e4m3, --bf16-gemm-backend cutedsl, --max-prefill-tokens 8192, --chunked-prefill-size 8192, glm47 tool-call parser, glm45 reasoning parser), HiCache at hicache-ratio 0.75 / write_back / direct / page_first_direct, and the default 1M-context agentic corpus that glm5.2 selects in resolve_trace_source, so the B200 and B300 curves are directly comparable." + - "Second delta: the checkpoint-resolution guard. runners/launch_b200-dgxc.sh rewrites MODEL to the cluster-local checkpoint path for single-node runs, so `hf download \"$MODEL\"` cannot work there; the script keeps the HF repo id in HF_MODEL_ID and downloads from it only when MODEL_PATH is missing or empty." + - "Adds glm5.2/fp4 routing to runners/launch_b200-dgxc.sh, which previously hard-failed with 'Unsupported model prefix/precision' for this model. Probes /lustre/fsw/models/GLM-5.2-NVFP4, /scratch/fsw/models/GLM-5.2-NVFP4 and /lustre/fsw/gharunners/models/GLM-5.2-NVFP4 in that order (same shape as the dsv4 branch), falling back to the sa-shared-writable gharunners path and creating it, since --container-mounts needs the host path to exist and the bench script's download guard populates it on first use." + - "Search space mirrors glm5.2-fp4-b300-sglang-agentic-mtp exactly so the two SKUs are comparable: one TP8 + HiCache arm at conc [1, 4, 8, 12, 16]. Steps of at least 2 because single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and a hard stop at conc 16. TP8-only for memory as well: the ~433 GB NVFP4 checkpoint needs ~54 GB/GPU across 8 B200s and does not fit below 8. The DEP (attention-DP + EP8) throughput arm is not wired up for the same reason as on B300 -- its frontier peak sits well above the conc-16 cap -- but the branch, including the --speculative-moe-a2a-backend none / --speculative-moe-runner-backend triton pair that GLM-5.2-NVFP4's unquantized bf16 nextn layer needs under expert parallelism, is kept in the script." + - "Staging fix after run 30729467646, in which all five cells failed identically. /lustre/fsw/gharunners/models/GLM-5.2-NVFP4 already existed on b200-dgxc holding config.json, generation_config.json, hf_quant_config.json, chat_template.jinja, README.md and .quant_summary.txt and no weights or tokenizer at all -- an aborted or metadata-only pull. The `ls -A` emptiness guard accepted it, every cell skipped the download, SGLang read config.json fine and then died in AutoTokenizer.from_pretrained with \"Couldn't instantiate the backend tokenizer\". The B300 sibling ran 5/5 green on the same image and the same MTP + simulated-acceptance config in run 30729399405, so this was staging only, not the recipe or the image." + - "Replaces the emptiness guard with a completeness check -- tokenizer_config.json, a tokenizer.json or tokenizer.model, model.safetensors.index.json, and every shard the index names -- and serializes the ~433 GB download behind an flock on .download.lock so one cell stages the checkpoint and the other four wait instead of five allocations racing the same Lustre path. hf download resumes into a partially-populated --local-dir, so it downloads on top of the stub. The launcher probe now also requires a candidate to hold at least one weight shard rather than merely exist, so a stub in one staged tree cannot be preferred over a real copy in another." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2448 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-mtp + description: + - "Add the first Qwen3.5 AgentX (agentic-coding) recipe on H200: Qwen3.5-397B-A17B FP8 with SGLang, spec-decoding=mtp only, routed to benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh. Shipped spec-decode-only per the AgentX policy in MODELS.md -- agentic recipes are run and published with speculative decoding enabled rather than as an STP/MTP A/B, because synthetic acceptance already makes spec-decode results comparable across submissions." + - "Speculative config mirrors fixed_seq_len/qwen3.5_fp8_h200_mtp.sh: SGLANG_ENABLE_SPEC_V2=1 with --speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4 -- 3 speculative tokens per verification step." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=3.39 (Qwen3.5 MTP curve at K=3, thinking_on, golden_al_distribution/qwen3.5_mtp.yaml), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -- the triple golden_al_distribution/README.md prescribes for SGLang, and the same AL the GB300 Qwen3.5 AgentX srt-slurm recipes pin. EVAL_ONLY runs leave simulated acceptance off and keep real verification." + - "Image lmsysorg/sglang:v0.5.16-cu130. SGLANG_SIMULATE_ACC_TOKEN_MODE is only read from v0.5.16 (SIMULATE_ACC_LEN / SIMULATE_ACC_METHOD exist further back), so an older image would silently ignore the token-mode half of the AgentX contract; v0.5.14 is what the fixed-seq-len H200 Qwen3.5 entries already run." + - "Serve shape follows the proven H100 Qwen3.5 AgentX replay path (SGLang HiCache host-DRAM offload with page-size 64 / kernel IO / page_first layout, flashinfer attention, --quantization fp8, --kv-cache-dtype fp8_e4m3, --mamba-ssm-dtype bfloat16, the multi_tokenizer cached_tokens_details patch, the 256k-capped with-subagents trace corpus) with --mem-fraction-static 0.8 instead of 0.75, matching the fixed-seq-len H200 recipe's use of H200's 141 GB HBM3e." + - "Search space now runs on the AgentX MTP concurrency grid: the GPU-resident row [4, 8, 12, 16] and the HiCache row [4, 8, 12, 16]. Every step is at least 2 concurrency apart and every arm stops hard at conc 16 -- single-step sampling cannot separate configurations by more than run-to-run noise on the agentic corpus, and past conc 16 these SKUs are into the post-HBM-cliff thrashing regime that the non-MTP sweeps already characterized, which is not worth one GPU job per point." + - "Drops conc 1 from the GPU-resident arm. The TP8/EP8 c1 cell was the one job that never returned on this recipe while the other 18 completed, so it gated the PR without adding a point: at conc 1 a single session cannot exercise the EP8 all-to-all this arm exists to measure, and the low-interactivity corner is already covered by conc 4." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2428 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Capture the vLLM backend Prometheus endpoint for every topology and require vllm: metrics before publishing AgentX results." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2516 + +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Retain the 2P2D PCP8 prefill, TP8/DP8 decode, Mooncake, 1M-context topology, and c8/c12/c16 search space; use GPU-resident decode KV cache because released SGLang does not support combining HiSparse with MTP" + - "Use EAGLE with three speculative steps, top-k 1, four draft tokens, and the committed GLM-5.2 thinking-on golden synthetic acceptance length of 2.99; evals retain real verification" + - "Use lmsysorg/sglang:v0.5.16-cu130 and the latest NVIDIA/srt-slurm release, v1.0.44, which includes logical-worker Prometheus URL injection for AIPerf" + - "Enable SGLang cache reporting on both prefill and decode workers so AIPerf records prompt-cache read-token usage alongside all logical-worker Prometheus metrics" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2529 + +- config-keys: + - glm5.2-fp8-mi325x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh Actions run 29657732517 as an MTP sibling while preserving the single-node TP8/EP1, GPU-resident BF16 KV, TileLang DSA, 1M-context topology, and c1/c2/c3/c4/c5/c6/c8 search space" + - "Use EAGLE with three speculative steps, top-k 1, four draft tokens, and the committed GLM-5.2 thinking-on golden synthetic acceptance length of 2.99; evals retain real verification" + - "Use the official lmsysorg/sglang:v0.5.16-rocm720-mi30x image, enable SGLang prompt-cache reporting, and pass the backend Prometheus metrics endpoint explicitly to AIPerf" + - "Allow up to 30 minutes for healthy AgentX responses admitted near the end of the measurement window to drain before AIPerf finalizes profile metric coverage" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2528 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Collect every SGLang worker's metrics through AgentX and fail when required sglang: series are absent." + - "Enable SGLang metrics on every aggregate, prefill, and decode engine." + - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + description: + - "Add GLM-5.2 FP4 SGLANG Single Node Agentic Support" + - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2488 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Upgrade Dynamo from 1.3.0.dev1 to 1.3.0.dev20260718 so router queueing is disabled by default at high concurrency" + - "Migrate AgentX session affinity from nvext.session_control to X-Dynamo-Session-ID headers with a 3600-second router TTL" + - "Remove the DYN_ROUTER_TEMPERATURE=10000000 override and restore Dynamo's deterministic 0.0 default" + - "Upgrade SGLang from nightly-dev-cu13-20260711-7de33ce8 to nightly-dev-cu13-20260719-99f5a6f4, the latest nightly before the scheduler WAR-barrier regression in SGLang #31687" + - "Restore the pre-watchdog AgentX concurrency grid and use AIPerf's default 300-second per-trajectory idle-gap cap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2319 + +- config-keys: + - qwen3.5-fp4-b200-sglang + - qwen3.5-fp4-b200-sglang-mtp + - qwen3.5-fp4-b300-sglang + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Switch Qwen3.5 FP4 SGLang configs to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint" + - "Model pre-staged on all clusters as Qwen3.5-397B-A17B-NVFP4-V2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2205 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt + description: + - "Add Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated multinode TensorRT-LLM+Dynamo benchmarks (STP, no MTP) for 8k1k." + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 with Dynamo installed at runtime (version 1.3.0.dev20260708, TCP request plane)." + - "8k1k: 9 STP configs spanning 1P13D DEP2+13D TP4 (conc 78), 1P4D DEP2+4D TEP16 (conc 24), 1P8D DEP2+8D TEP8 (conc 48), 2P15D DEP2+15D TP4 (conc 180/216), 3P1D DEP2+1D DEP4 (conc 1536), 7P12D DEP2+12D TEP4 (conc 1152), 8P9D DEP2+9D TEP4 (conc 1728), 10P3D DEP2+3D DEP8 (conc 4608)." + - "Runner: launch_gb300-nv.sh updated to clone NVIDIA/srt-slurm@v1.0.29 and overlay local recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/ for the dynamo-trt+qwen3.5+fp4 path, and to skip srtctl preflight for this path." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2206 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang + description: + - "Bump SGLang image to v0.5.14-cu130 for the Qwen3.5 FP4 GB300 disaggregated dynamo-sglang config" + - "Switch to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint (model pre-staged on cluster as Qwen3.5-397B-A17B-NVFP4-V2)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2238 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + description: + - "Add EAGLE/MTP speculative-decoding arm for the DSv4 MI355X agentic disagg HiCache benchmark, mirroring the existing dsv4-fp4-mi355x-sglang-disagg-agentic-hicache TP8/EP1 and TP8/EP8/DPA topologies with spec-decoding=mtp and DECODE_MTP_SIZE=3." + - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to DeepSeek-V4-Pro-AgentX; num-steps/num-draft-tokens are derived from DECODE_MTP_SIZE in server_sglang.sh. DeepSeek-V4's MTP head is a native NextN/EAGLE draft shipped with the model, so no --speculative-draft-model-path is needed." + - "Two search-space arms: TP8/EP1 no-DP at conc-list [2,4,8,16,32], and TP8/EP8/DPA (ep=8, dp-attn=true) at conc-list [64,96,128], both on image lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2309 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + description: + - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + description: + - "Refresh B300 Kimi K3 DSpark submission with corrected AgentX harness" + - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" + - "Match the current published upstream Kimi K3 FP8-KV profile by using flashinfer MLA prefill with prefill query quantization" + - "Keep DSpark K=2 probabilistic throughput on synthetic golden AL 2.51 while the generated EVAL_ONLY row uses real block verification" + - "Cap the GPU-resident search at concurrency 8 after concurrency 16 failed to complete deterministic warmup; retain concurrency 16 for the DRAM-offload capacity tier" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2476 + +- config-keys: + - dsv4-fp8-h200-dynamo-sglang-agentic-agg + description: + - "Add one H200 TP8 aggregated DeepSeek-V4-Pro AgentX recipe with MTP and HiCache." + - "Generate Pareto points by sweeping concurrency [1, 2, 4, 8, 16] while keeping the serving topology fixed." + - "Use Dynamo header-based session affinity and AIPerf agentx-v1.0.1 (b7b16cf851885567988a643282266bce74e34437)." + - "Enable SGLang engine metrics and require the sglang: server-metric prefix so the published trace artifacts carry backend data." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2364 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on a two-node TP8/DP2/EP16 DSpark topology with tensor parallelism local to each B200" + - "Add a TP8/DP2/EP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Keep throughput on DSpark level 7 probabilistic drafting with committed golden synthetic AL 3.84; EVAL_ONLY selects committed rejection_sample_method=block recipes for real target verification" + - "Use the digest-pinned 2026-08-10 upstream vLLM nightly and direct Inferact/Kimi-K3-DSpark model ID as shipped, with no local compatibility layer or installed serving-engine rewrite" + - "Use node-local TP8 FlashInfer collectives and cross-node DP2/EP16 so the digest-pinned upstream serving engine runs without a compatibility layer or source rewrite" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + description: + - "Add the MI355X MiniMax-M3 FP4 vLLM AgentX MTP submission on vLLM v0.27.1, using the committed EAGLE3 synthetic acceptance length of 3.35 for throughput and real target verification for eval." + - "Sweep TP2/TP4 GPU-resident configurations and TP4 LMCache MP DRAM-offload points around the capacity knee, with vLLM server metrics enabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2487 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B FP8 AgentX on MI300X with SGLang native EAGLE MTP and golden synthetic acceptance length 3.39" + - "Use TP8/EP1 and TP8/EP8 Pareto sweeps at c4/c16/c20/c24/c32; fast discovery found peak throughput at c24 and a post-knee regression at c32" + - "Exclude HiCache because SGLang v0.5.16 rejects Qwen3.5's layer-first hybrid cache layout" + - "Collect and require SGLang Prometheus metrics from the aggregate engine endpoint" + - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2563 + +- config-keys: + - minimaxm3-fp8-h100-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 MXFP8 AgentX on H100 with vLLM v0.27.1, EAGLE3 golden AL 2.78, resident TP8 c1-c5, and Mooncake DRAM offload c6/c8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2564 + +- config-keys: + - qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp + description: + - "Add the measured GB200 Qwen3.5 NVFP4 AgentX Pareto frontier across aggregate TP4, TP2/EP2, and TP4 HiCache with K=3 MTP." + - "Use stable SGLang v0.5.17-cu130 and NVIDIA/srt-slurm v1.0.45." + - "Enable SGLang metrics/cache reporting and require nonempty sglang: server metrics before publishing results." + - "Use K=3 golden synthetic acceptance length 3.39 for throughput; eval retains real MTP verification." + - "Give the released SGLang router one hour to register the 28-minute-loading backend and route AgentX sessions by correlation ID." + - "Bound hybrid-state pools to the measured resident and HiCache operating ranges instead of reserving capacity that cannot fit beside target and MTP weights." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2568 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MI325X Qwen3.5 FP8 AgentX with SGLang native EAGLE MTP and golden synthetic acceptance length 3.39" + - "Use lmsysorg/sglang:v0.5.16-rocm720-mi30x, FP8 KV cache, AITER attention, and required SGLang Prometheus metrics" + - "Select a 32-point official TP2/EP2, TP4, TEP4, TP8, and TEP8 matrix from the complete 62-point fast discovery sweep" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2566 + +- config-keys: + - minimaxm3-fp8-h200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 MXFP8 AgentX on H200 with vLLM v0.27.1, EAGLE3 golden AL 2.78, resident TP8 c1/c2/c4/c6/c8/c10, and Mooncake DRAM offload c12/c14." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2565 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + description: + - "Add Qwen3.5-397B-A17B MXFP4 MI355X SGLang AgentX with native EAGLE MTP and the committed golden synthetic acceptance length." + - "Cover the measured resident TP2/EP2 and TP4 Pareto ranges through their HBM capacity knees, with required SGLang metrics exports." + - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang-mtp + description: + - "Add a DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP configuration for B200 with five 8k/1k disaggregated topologies." + - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + description: + - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement" + - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570 + +- config-keys: + - minimaxm3-fp8-mi325x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, and the measured resident TP8 frontier through its c16-c18 latency knee." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2579 + +- config-keys: + - qwen3.5-fp4-b200-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" + - "Rides on the NVFP4-V2 checkpoint switch from #2205" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang + - dsv4-fp4-gb300-dynamo-sglang + - qwen3.5-fp4-gb300-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Extend official DCGM power collection to the recipe-gated FP4 GB200 and GB300 Dynamo-SGLang lanes." + - "Synchronize all DSV4 GB300 recipes on lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3." + - "Pin the srt-slurm power producer and record its exact checkout SHA for strict provenance validation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2507 + +- config-keys: + - qwen3.5-fp8-b300-sglang-agentic-power-ab + - qwen3.5-fp4-b300-sglang-agentic-power-ab + scenario-type: + - agentic-coding + description: + - "Add a controlled Qwen3.5 B300 AgentX power matrix across FP8/FP4, HBM-only/HiCache, and concurrency 16/32 using identical SGLang v0.5.16 TP2/EP2 MTP settings." + - "新增受控的 Qwen3.5 B300 AgentX 功耗矩阵:在相同的 SGLang v0.5.16、TP2/EP2 和 MTP 配置下,对比 FP8/FP4、仅 HBM/HiCache,以及并发 16/32。" + - "Allow workflow_dispatch AgentX jobs to fail closed when strict power validation is requested." + - "允许通过 workflow_dispatch 启动的 AgentX 任务在启用严格功耗校验时遇到无效遥测后直接失败。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2601 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + description: + - "Add DeepSeek-V4-Pro FP4 single-node SGLang AgentX recipe on MI355X with EAGLE/MTP (num-steps 3, num-draft-tokens 4) and SGLANG_SIMULATE_ACC_LEN=2.49 from the committed thinking_on golden AL curve" + - "Image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2600 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, measured resident TP8 c2-c10, and an LMCache MP DRAM-offload c16 point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2580 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the purged B200 DeepSeek-V4-Pro SGLang AgentX resident, DP-attention, and HiCache grid on SGLang v0.5.17." + - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." + - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, and keep pooled AgentX connections alive across inter-turn gaps." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2578 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the purged B300 DeepSeek-V4-Pro SGLang AgentX TP4, TP8, DP-attention, and HiCache grid on SGLang v0.5.17." + - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." + - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX with K=2 MTP and golden synthetic acceptance length 2.49." + - "Compare the B200 TP8 and DEP8 concurrency range and extend throughput with 1P1D and 2P1D DEP8 disaggregation." + - "Use KV-aware Dynamo routing with 4-hour correlation-ID affinity, authoritative vLLM KV events, KV-cache token metrics, and every logical vLLM server-metrics endpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2-NVFP4 GB300 AgentX (agentic-coding) Dynamo-SGLang benchmarks over a 7-point sweep: aggregated single-worker TP8 (conc 2/4), disaggregated low-latency TP4-prefill/TP4-decode 1P2D (conc 48), 1P4D (conc 48), 1P6D (conc 45), and high-throughput 2P1D DEP8-prefill/DEP16-decode (conc 128/192)" + - "EAGLE speculative decoding, hierarchical KV cache (hicache-size 135), MTP, decode mem-fraction-static 0.9, nixl KV transfer with UCX_TLS=ib, and X-Dynamo-Session-ID session affinity; driven from the NVIDIA/srt-slurm base+zip_override recipes via CONFIG_FILE selectors" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642; runner: gb300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2608 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the MiniMax-M3 B300 AgentX submission as eight EAGLE3-GQA points: no-offload TP8 C1 and TP4 C1/C5/C10/C15/C20, TP4 C30 with 1,024 GB SimpleCPU KV offload, and TP2 C24 with the proportional-policy 750 GB SimpleCPU KV-offload cap." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2610 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Inject the committed synthetic MTP acceptance length only for GB300 AgentX throughput; keep eval-only jobs on real target verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add the tuned GB200 MiniMax-M3 FP4 AgentX frontier with EAGLE3, the B200 TP4 baseline, DEP4/DEP8, SimpleCPU offload, and KV-routed P/D." + - "Use full-decode-only CUDA graphs for TP4 stability." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the MiniMax-M3 B200 AgentX submission: no-offload TP8 C1 and TP4 C1/C5/C10/C15/C20, plus a TP4 SimpleCPU KV-offload sweep at C15–C40 using the full-capacity DRAM budget." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2611 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Move the B300 Kimi K3 recipe to TP8 x DCP8 with Mooncake as the external KV tier" + - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 40, 48, 56 and 70, drafting with DSpark level 7 at concurrency 8 and below, DSpark level 3 at concurrency 16" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2616 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B-NVFP4-V2 GB300 TRT-LLM disaggregated AgentX with Dynamo router 1.4.0.dev20260807 and NIXL KV transfer." + - "Cover 6 disaggregated topologies (1P1D through 3P2D) with MTP speculative decoding across conc 7-704 using TRT-LLM release:1.3.0rc24." + - "Mixed attention-DP (dep) and non-DP (tep) decode shapes; recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/." + - "EVAL_ONLY runs switch to real block verification by removing synthetic acceptance (TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS stripped from all agentx recipes before srtctl)." + - "Add SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 to all 6 agentx recipe env blocks and make build_replay_cmd prefer SERVED_MODEL_NAME over $MODEL for the aiperf --model flag." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2612 + +- config-keys: + - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GB200 GLM-5.2 FP4 Dynamo-SGLang AgentX with EAGLE MTP, HiCache DRAM offload, and KV-aware correlation-ID affinity." + - "Measure the TP8 aggregate curve and tuned 1P1D TP4 HiCache c10/c12 disaggregated frontier with every logical SGLang metrics endpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2620 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Retune B200 DeepSeek-V4-Pro vLLM AgentX MTP with SimpleCPU TP8 [1, 4, 8, 16, 24] and DEP8 [32, 64, 96, 128, 160, 196]." + - "Use vLLM 426e59f with DEP scheduling, MTP CUDA graphs, and host-DRAM sizing while keeping the B200 0.90 GPU-memory default." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2621 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg + scenario-type: + - agentic-coding + description: + - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2619 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg + scenario-type: + - agentic-coding + description: + - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 + + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5 FP8 H200 SGLang AgentX MTP points with DRAM HiCache." + - "Use TP8/EP1 concurrency 2 through 24, published FlashInfer 0.6.17 CUDA 13 wheels, and golden MTP acceptance length 3.39." + - "Use the lmsysorg/sglang:nightly-dev-cu13-20260815-a5ba081f image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2592 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use a 270 GB/rank absolute HiCache target pool for glm5.2-fp4-b300-sglang-agentic-mtp, replacing the previous ratio-based sizing." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2651 + +- config-keys: + - glm5.2-fp4-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use a 270 GB/rank target HiCache pool for the GLM-5.2 B200 TP8 c12 and c16 AgentX points to avoid the host-cache capacity cliff." + - "Keep the existing ratio-0.75 pool for c1, c4, and c8 while retaining the SGLang EAGLE MTP and golden synthetic acceptance setup." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2652 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3 GB300 AgentX configurations." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2637 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + scenario-type: + - fixed-seq-len + description: + - "Bump qwen3.5-fp8-mi355x-sglang-disagg image from v0.5.14 to v0.5.16 (3-5% throughput improvement) and switch from TP8P+TP8D (16 GPU) to TP4P+TP8D (12 GPU) — TP4 prefill delivers identical decode interactivity with 24-31% better throughput/GPU." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2606 + +- config-keys: + - minimaxm3-fp4-b200-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 NVFP4 B200 single-node TensorRT-LLM AgentX with EAGLE3-GQA speculative decoding." + - "Use the nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2659 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Repin the image to the 2026-08-17 ROCm nightly (nightly-311b3513af33bc29b4acb2fde2e9313e5e9966a0, digest sha256:4b403315f9a0d9b0ed4221dfc7ea04538701b4de0ce283c0e2cee622b217ff89). Docker Hub last pushed it at 2026-08-17T05:32:09Z and the tag commit is vllm-project/vllm@311b3513." + - "Update the pure TP8 AgentX search-space to concurrency 1, 4, 8, 16, 32, 40, 48, 56, and 64; keep only concurrency 64 for the DP-attention arm." + - "The pinned nightly includes vLLM #51473 (native 384-shard MXFP4 TP8 allocation), #52212 (gfx950 Triton sparse-MLA decode), and #52401, the merged equivalent of Fangzhou's #52488 regression fix for ROCm DeepSeek V4 MRV1 plus the wide eager attention region." + - "Pin max-num-batched-tokens to 8192 for both arms instead of the nightly default of 16384; the TP8 initialization check increased GPU KV cache capacity from 4,730,981 to 8,524,228 tokens and reduced peak activation memory from 11.44 GiB to 8.9 GiB." + - "Remove the MoE padding from 384 to 512 and replace it with the native 384-expert kernels, save 30% redundant dummy computation." + - "Optimze the MLA kernel under long context scenario, where the num_kv_split is further tuned for best hardware utilization." + - "Cache fp32 upcast of static e8m0 weight scale in AITER scaled_mm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2590 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 NVFP4 B300 single-node TensorRT-LLM AgentX with EAGLE3-GQA speculative decoding (3 draft tokens)." + - "Use the nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2658 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update the B200 SGLang AgentX HiCache MTP DEP8 configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2656 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the MI355X GLM-5.2 FP4 ATOM AgentX MTP submission: TP4 at concurrency 2, 4, 8, and 10 uses LMCache DRAM offload, while GPU-resident TP8 at concurrency 1, 2, and 4 uses no KV offload." + - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." + - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the MI355X Kimi-K3 FP4 ATOM AgentX submission with DSpark speculative decoding (2 draft tokens): TP8 at concurrency 1 and 4 is GPU-resident with no KV offload, while concurrency 8 and 10 add the LMCache DRAM tier together with ATOM's CPU state-offload tier." + - "Both offload tiers are switched together because Kimi-K3 is a hybrid: Kimi Delta Attention carries a per-request recurrent state alongside the paged KV, so a resumed agentic turn needs the state back and the paged KV tier alone cannot restore one." + - "Use AITER INT4 quick-reduce, AITER SITUV2 A4W4, FlyDSL stage-2 FP8, ptpc_fp8 online quantization excluding embeddings, lm_head, conv1d projections, experts, and the multimodal tower, an FP8 KV cache, and a 128-token page." + - "Set the simulated acceptance length to 2.54, giving spec-decode-acceptance-rate (2.54 - 1) / 2 = 0.77, which is the value the submitted ATOM numbers were measured under. This is 1.2% above the committed golden 2.51 that the vLLM arm feeds to synthetic_acceptance_length, and the script carries a review note saying so; set it to 2.51 for a strictly like-for-like sweep. Evaluations use real acceptance." + - "Divide the aggregate TOTAL_CPU_DRAM_GB budget by TP for LMCACHE_MAX_LOCAL_CPU_SIZE, which is a per-rank setting; dram-utilization 0.085 lands that at roughly 32 GB per rank." + - "Set OFFLOAD_GPU_STAGING_CHUNKS to 16 because the staging buffer defaults to 2 chunks (8 MiB) and one Kimi-K3 state entry is 54.78 MiB; a buffer too small to hold one entry makes the state tier decline to build after a single log line." + - "Select max-num-seqs, max-num-batched-tokens, gpu-memory-utilization, and in-GPU state-checkpoint slots by concurrency, and place state checkpoints at the prompt-end anchor only." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2670 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro FP4 ATOM MTP AgentX on one 8x MI355X node at concurrency 1, 2, 4, 8, 16, 32, and 48, with no KV offload and max-num-seqs set to twice concurrency." + - "Use rocm/atom-dev:nightly_202608201032, FP8 KV/index caches, prefix caching, 32K state checkpoints, 16K batching/prefill chunks, and FULL cudagraph mode." + - "Use three-token MTP with the committed DeepSeek-V4 thinking-mode golden AL 2.49 (synthetic acceptance length 3) for throughput, while eval-only runs measure real MTP acceptance." + - "Increase the AgentX warmup grace period from the 1800-second default to 3600 seconds at concurrency 32 and 48, matching the DeepSeek-V4-Pro SGLang saturation recipe so in-flight warmup requests can drain and reused long prefixes are fully primed before profiling." + - "Keep concurrency 1 through 16 at the default 1800 seconds; the profiling duration remains 3600 seconds for every concurrency arm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2698 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add new B200 multi-node configs with PP2, DCP8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2672 + +- config-keys: + - glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp + - glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 NVFP4 GB200 AgentX with Dynamo-SGLang across aggregated TP8 and disaggregated recipe variants." + - "Use aggregated TP8 variants at concurrency 2, 4, and 8, and disaggregated TP4 and DEP16 variants at concurrency 45, 48, and 128." + - "Use NIXL KV transfer for disaggregated variants, prefill HiCache DRAM offload, native MTP with committed thinking-on golden acceptance lengths, and lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642." + - "Leave DYN_ROUTER_TEMPERATURE unset across the recipe set." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2642 + +- config-keys: + - qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh GB200 Qwen3.5 NVFP4-V2 AgentX on lmsysorg/sglang:nightly-dev-20260818-c0b6474b without engine patches." + - "Add a TP2/EP2 HiCache branch and update the resident recipes with extra_buffer_lazy, ReplaySSM speculative verification, and checkpoint page-cache release." + - "Select TP4 C1/C2/C4/C8/C12/C20/C24, TP2/EP2 resident C4/C8/C12/C20, and TP2/EP2 HiCache C22/C24/C28/C32/C40 with golden synthetic acceptance length 3.39." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2660 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg + scenario-type: + - agentic-coding + description: + - "Add GB200 Kimi K3 AgentX config." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2639 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260811 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260818" + - "Serve the AgentX arm with an fp8_e4m3 KV cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2677 + +- config-keys: + - minimaxm3-fp4-b200-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh with lower stream interval to collect correct client metrics" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2 + scenario-type: + - agentic-coding + description: + - "Update the GB200 Pareto by replacing the non-DSpark DCP16 candidates at concurrency 16, 24, 32, 36, and 44 with TP8 x PP2/DCP8 Mooncake candidates at concurrency 16, 32, 48, 72, and 96." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2700 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump the SGLang image from lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817." + - "Add concurrency 48 and 64 to the TP8/EP1 sweep, and 64 and 192 to the TP8/EP8 DP-attention sweep." + - "Increase prefill mem_fraction_static from 0.72 to 0.85 and reduce HICACHE_RATIO from 4 to 3." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2643 + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt-agentx + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro GB300 Dynamo-TensorRT-LLM AgentX recipes for concurrencies 4, 24, 388, 736, 1152, and 2626." + - "Disable TensorRT-LLM return_perf_metrics in the prefill and decode workers." + - "Route the selected GB300 workloads to batch_3." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2690 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Append GPU-resident TP4 concurrency 1, 2, 4, and 6 points to the existing B300 DeepSeek-V4-Pro vLLM AgentX MTP curve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2647 + append-only: true + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Expand the GB300 AgentX aggregate TP4 sweep to concurrency 1, 2, 4, 6, and 8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2676 + + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from lmsysorg/sglang:v0.5.17-cu130 to lmsysorg/sglang:dev-nightly-0820 to pick up the prefill-delayer scheduling support used by the updated B200 recipe." + - "On the DP-attention path, add --enable-prefill-delayer with --prefill-decode-interval 10. Under speculative decoding plus DP attention, decode is synchronized globally; bounding consecutive decode steps prevents one rank's queued prefill from being delayed indefinitely." + - "On the DP path, replace DeepEP with --moe-a2a-backend megamoe plus --enable-deepseek-v4-fp4-indexer, --disable-shared-experts-fusion, and --disable-flashinfer-autotune. Export the MegaMoE FP4/MXF4 activation settings and increase the per-rank token workspace from 8192 to 8320 for headroom." + - "Scale chunked-prefill-size with dp_size (8192 * TP). SGLang divides this global budget by dp_size, preserving the tuned 8192-token per-rank budget for every DP-attention topology." + - "Set --cuda-graph-max-bs to 2x concurrency for every topology, matching max-running-requests so AgentX branch fan-out remains covered by CUDA graphs without an upper cap." + - "Enable the DeepSeek-V4 FP4 indexer on the TP-only path as well, while retaining flashinfer_mxfp4, chunked-prefill 8192, mem-fraction-static 0.88, and swa-full-tokens-ratio 0.1." + - "Keep the published TP-only rows, drop the DP-attention no-offload point, and replace the DP-attention HiCache concurrency list [16, 40, 48] with [32, 64, 96]. The DP path retains HiCache ratio 8, mem-fraction-static 0.88, swa-full-tokens-ratio 0.02, and the existing 2x max-running-requests budget." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2703 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add one aggregated TP8 and six disaggregated GLM-5.2 NVFP4 GB300 AgentX recipes for Dynamo and TensorRT-LLM with MTP speculative decoding." + - "Use the concurrency-1 aggregated Pareto point and disable TensorRT-LLM performance-metrics responses for all seven points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 + + +- config-keys: + - minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add the MiniMax-M3 NVFP4 GB300 disaggregated AgentX config with EAGLE3-GQA MTP and Mooncake DRAM KV offload." + - "Use vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 with golden synthetic AL 2.78 for throughput and a real-verification eval twin." + - "Pin thinking-on so the run matches the golden minimaxm3_eagle3_gqa thinking_on[3] = 2.78 acceptance target: dyn-default-thinking-mode enabled on the decode worker covers both the AgentX replay and the eval twin, with AIPERF_EXTRA_INPUTS thinking:true on the replay as the GB200-sibling belt-and-braces." + - "Set dyn-reasoning-parser and dyn-tool-call-parser alongside reasoning-parser, matching the GB200 MiniMax-M3 AgentX recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2663 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh with lower stream interval to collect correct client metrics" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2685 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Serve the TP8 arm with FULL_AND_PIECEWISE cudagraphs, capturing the mixed prefill/decode batches as well as the uniform decode batches. The DEP8 arm is unchanged." + - "Replace the TP8 concurrency list [1, 4, 8, 16, 24] with [1, 4, 8, 14], dropping the published concurrency 16 and 24 points and adding concurrency 14." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2707 + +- config-keys: + - glm5.1-fp8-b200-tilert + - glm5.1-fp8-b200-tilert-agentic + description: + - "Update 8k1kTileRT results, add AgentX results" + - "Replace the internal TileRT 0.1.5.post2+inferencex.1 queueing backport with the official 0.1.5.post3 PyPI wheel now that upstream has published it; remove the internal wheel builder and router patch" + - "Bump the shared TileRT dependency and router metadata to 0.1.5.post3 for both fixed-seq-length and AgentX configurations while retaining the AgentX 1800-second queue timeout" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the MI355X Kimi-K3 FP4 ATOM AgentX submission onto image kimi_k3_agentic_0821 and replace the published concurrency list [1, 4, 8, 10] with [1, 2, 4, 8, 12, 16, 32, 40, 56], retuning every point." + - "Shard the decode KV read across all 8 GPUs with decode-context-parallel-size 8 from concurrency 8 up. At 100k+ token agentic contexts decode is KV-bandwidth-bound; concurrency 1-4 leaves it off because a lightly loaded node gains nothing from the extra collectives." + - "Pin acceptance to the committed golden curve in golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml at both draft lengths: --spec-decode-acceptance-length 3.84 at 7 draft tokens for concurrency 1-4, and 3.00 at 3 draft tokens for concurrency 8 through 16. Evaluations drop the flag and use real acceptance. Concurrency 32 and up serve without a draft model." + - "Select max-num-seqs, max-num-batched-tokens, gpu-memory-utilization, ReplaySSM, and in-GPU state-checkpoint slots by concurrency: 32/8192/0.88 with ReplaySSM off at concurrency 1-4, 32 and 24 with a 4096-token prefill step and ReplaySSM on with 96 checkpoint slots at concurrency 8 and 12, and 32/64/80/72 with utilization 0.86-0.88 and ReplaySSM off from concurrency 16 up, where the Kimi Delta Attention recurrent state rides the CPU state-offload tier instead of in-GPU checkpoints." + - "Divide the aggregate TOTAL_CPU_DRAM_GB budget by TP for the per-rank LMCACHE_MAX_LOCAL_CPU_SIZE and OFFLOAD_STATE_CPU_SIZE. dram-utilization 0.268 lands 100 GB per rank of paged KV at concurrency 8 and 12; 0.343 lands 128 GB per rank from concurrency 16 up, split 96 GB paged KV plus 32 GB for the state tier." + - "Size the paged-KV tier at half of what the recipe was measured with (200 and 192 GB per rank). Every rank pins its own pool, and at the measured sizes the eight ranks finished pinning more than 600 seconds apart, which is exactly PyTorch's hardcoded NCCL process-group timeout: ranks reaching the barrier at the end of allocate_kv_cache first timed out waiting for rank 0 and the ModelRunner processes died before the server served a request. The measured CPU-tier hit rate is 0.0% at concurrency 8 and 12 and 0.3-2.4% above that, so the smaller pool costs little; 96 GB per rank still holds roughly 53M tokens. The state tier stays at 32 GB per rank." + - "Set LMCACHE_CHUNK_SIZE to 1024 rather than the 128-token page: with decode context parallelism the offload hash block is block-size times dcp, so any other chunk misaligns it." + - "Set OFFLOAD_GPU_STAGING_CHUNKS to 32 because the staging buffer defaults to 2 chunks (8 MiB) and one Kimi-K3 state entry is 54.78 MiB; a buffer too small to hold one entry makes the state tier decline to build after a single log line." + - "Keep AITER INT4 quick-reduce, AITER SITUV2 A4W4, FlyDSL stage-2 FP8, ptpc_fp8 online quantization excluding embeddings, lm_head, conv1d projections, experts, and the multimodal tower, an FP8 KV cache, a 128-token page, and prompt-end-anchor-only state checkpointing. Drop ATOM_MLA_MAX_SPLIT_PER_BATCH, which the retuned recipe does not set." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2716 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822." + - "Enable SGLANG_OPT_USE_AITER_BATCHED_GEMM=1, the aiter batched GEMM for the absorbed MLA projections, which defaults off." + - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." + - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." + - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and remove the TP8 no-offload arm [1, 2, 4, 8, 16]. This drops the published TP8 concurrency 1, 2, 4, 8 and 16 points and adds TP4 concurrency 1, 2, 4, 8 and 10." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2713 + +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d + - dsv4-fp8-h200-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Require dcgm-power telemetry on the GLM-5.2 H200 disaggregated AgentX recipe." + - "Enable dcgm-power telemetry on the DSV4 H200 aggregated AgentX recipe." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2684 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from lmsysorg/sglang:v0.5.17-cu130 to lmsysorg/sglang-staging:dev-cu13-pr-35880, which picks up sgl-project/sglang#35017 (--prefill-decode-interval). That tag is lmsysorg/sglang:dev-nightly-0820 (build commit 92eeed41d7) with sgl-project/sglang#35880 applied on top -- a cherry-pick of #34975 that fixes a HiCache load-back crash. Without it, DEP8 at conc 384/512 dies mid-run with 'AssertionError: node N pinned by load-back M, new anchor N' in unified_tree_core.py commit_load_back: a node can be pinned by two concurrent H->D load-backs, but load_back_pending_id held only one anchor. Eight schedulers exit at once and the surviving DP ranks then block forever in the MLP-sync collective, so the client sees in-flight requests never return with zero errors. The assertion is reachable only under --hicache-write-policy write_back, which this recipe uses. Swap to a released tag once #34975 merges and ships in a nightly." + - "Add --prefill-decode-interval 20: +28% output throughput at conc 128 (2,433-2,470 -> 3,127-3,161 tok/s), closing the gap to the vLLM recipe from 1.32x to 1.03x. Under spec-decoding + DP attention SGLang synchronises decode globally, so a rank with no prefill work runs an idle batch and the busiest rank sets the clock for all eight; the interval bounds that." + - "Engine changes are scoped to the DP-attention path except the decode CUDA-graph cap. The TP-only path keeps flashinfer_mxfp4, chunked-prefill 8192, mem-fraction 0.88 and swa-full-tokens-ratio 0.1, because every measurement here ran with DP attention enabled." + - "Size the TP-only decode CUDA graphs at 4*CONC (still capped at 64) instead of CONC. AgentX concurrency counts live session trees, not requests, and the recipe already allows max-running-requests of 2*CONC for subagent fan-out -- so capturing only up to CONC dropped every larger decode batch to eager execution on exactly the low-concurrency rows that exist to measure latency. The runtime clamps the captured list to the request-pool size, so this cannot over-capture." + - "On the DP path, switch --moe-runner-backend flashinfer_mxfp4 to --moe-a2a-backend megamoe plus --enable-deepseek-v4-fp4-indexer, and export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 and _USE_MXF4_KIND=1. Both env flags default to False, so the backend flag alone would select a different MegaMoE kernel than the one measured; DeepGEMM's DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded from them automatically." + - "Scale chunked-prefill-size with dp_size (8192 * TP) instead of pinning 65536. server_args.py divides this global budget by dp_size, so a fixed 65536 gives the tuned 8192/rank at DEP8 but 16384/rank at DEP4 -- which exceeds MegaMoE's per-rank token cap and raises at startup, and which measured slower when tried directly at DEP8. Also export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, since the 8192 default is an exact fit with no headroom." + - "On the DP path, replace --cuda-graph-max-bs with --cuda-graph-max-bs-decode 544 and add --enable-prefill-delayer. The generic flag aliases the same knob, and its CONC-capped value of 64 does not cover the padded MTP decode batch across DP ranks." + - "Scale DP-path mem-fraction-static down as concurrency rises: DEP8 uses 0.93 at conc 32/64/128, 0.9 at 256, 0.88 at 384 and 0.86 at 512/576. MegaMoE's transient workspace is allocated outside the static budget and needs a single ~7 GB contiguous block, so the headroom left by mem-fraction-static has to grow with batch size. Measured at conc 256: 0.835 (~42 GB free) completes, while 0.93 (~16 GB free) and 0.95 (~11 GB free) both hit a CUDA OOM on a single DP rank -- which then hangs the whole engine, because the surviving ranks block forever in the MLP-sync collective and the client sees in-flight requests never return with zero errors. Also lower swa-full-tokens-ratio from 0.1 to 0.075." + - "Restructure the search space from 42 points to 12: a TP-8 no-offload row at conc 1/4/8/16/32 and a DEP8+hicache row at conc 32/64/128/256/384/512/576. Both TP-4 rows and both DEP4 rows are dropped. DEP4 cannot serve this trace: sharding the model over half the node leaves ~627k KV tokens per rank at mem-fraction 0.93 (measured) against prompts reaching 950,812 tokens, so the longest requests cannot be held at all, while raising mem-fraction far enough to hold them leaves too little room for the mega-MoE workspace. The two constraints barely overlap, and every DEP4 point failed. This tracks the vLLM agentic lane on the same runner, which sweeps DEP4 with a DRAM tier at 48/64 and DEP8 with a DRAM tier at 128/256/384/512/576. Measured at conc 256 with a CPU tier, SGLang 4,168 vs vLLM 4,199 tok/s." + - "Raise dram-utilization from 0.80 to 0.95, matching the vLLM agentic lane on the same runner, and the TP>=8 hicache ratio from 2 to 3 so the CPU tier reflects the allocated budget. DSv4 HiCache takes a host/device token ratio, not bytes, so the recipe cannot apply TOTAL_CPU_DRAM_GB directly, and host capacity scales with both the ratio and device KV -- meaning it also grows with mem-fraction-static. Measured at ratio 2 / mem-fraction 0.835 the tier allocates 999 GB (124.88 GB per rank across 8 ranks), a third of the 2,849 GB the vLLM agentic lane consumes. ratio 4 at mem-fraction 0.93 overshot the other way, leaving 5.84 GB free on a 2,964 GB node and failing to allocate the V4 paged pool; ratio 3 keeps the tier near 2 TB with room for the paged pool, page cache, AIPerf and the router. The TP<8 ratio is left at its existing value of 8." + - "Costs TTFT (p50 2.2x, p90 3.3x); AgentX is closed-loop so this is already priced into the throughput figures" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2701 + + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang + - dsv4-fp4-gb200-dynamo-sglang-mtp + - dsv4-fp4-gb300-dynamo-sglang + - dsv4-fp4-gb300-dynamo-sglang-mtp + - qwen3.5-fp4-gb300-dynamo-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + - qwen3.5-fp8-gb300-dynamo-sglang + - qwen3.5-fp8-gb300-dynamo-sglang-mtp + description: + - "Enable dcgm-power telemetry on the 51 remaining eligible GB200 and GB300 recipes, covering all 59 eligible recipes." + - "Mark telemetry required on every eligible recipe now that both the GB200 and GB300 pools are available, and leave the 2 dedicated-infra recipes disabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2687 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add four proven GB300 AgentX P/D points: c4, c128, c256, and c512." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + append-only: true + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Tune the GB300 SGLang AgentX aggregate low-concurrency frontier with TP8 concurrency 1/4 and TP4 concurrency 8 profiles." + - "Update the aggregate and disaggregated configurations to SGLang nightly-dev-cu13-20260821-f825d729 and the Dynamo 1.4.0 release." + - "Use phase-specific decode CUDA graph limits and throughput-only synthetic MTP acceptance; eval-only runs retain real target verification." + - "Use an eight-hour DSV4 AgentX allocation and pass the configured GB300 Slurm account to image-validation steps." + - "Warm the aggregate low-latency profiles with the standard ten requests per trajectory lane instead of one, so they enter the profiling window from the same live state as every other AgentX recipe." + - "Reduce the concurrency-1536 disaggregated topology from six to five DEP8 prefill workers while retaining one DEP16 decode worker." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2644 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Replace the TP2/EP2 arm with TP2/EP1 so the 2-GPU point matches qwen3.5-fp4-b200-sglang-agentic-mtp." + - "Add HiCache host-DRAM KV tier arms at TP4 concurrency 40, 48, 56, and 64 and TP2 concurrency 20, 24, 28, and 32, using hicache ratio 1.5 with write_through, direct io, and page_first_direct layout." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2693 + + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + - dsv4-fp4-b200-dynamo-vllm + - dsv4-fp4-b300-dynamo-vllm + - dsv4-fp4-b300-dynamo-sglang + - dsv4-fp4-b200-dynamo-sglang-mtp + - dsv4-fp4-b200-dynamo-vllm-mtp + - dsv4-fp4-b300-dynamo-sglang-mtp + description: + - "Enable required dcgm-power telemetry on 37 eligible recipes: 21 B200 and 16 B300." + - "Required lanes promote exporter startup timeouts, launch failures, and endpoint resolution failures to blocking validation failures, so a recipe cannot publish a result whose power collection never started." + - "Route only enabled recipes through the immutable producer fork and preserve non-power launcher revisions." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2688 +- config-keys: + - qwen3.8next-fp8-h200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the day-zero Qwen3.8-Flash-Next FP8 AgentX recipe on H200 with SGLang native NEXTN MTP at TP8 and concurrency 1/4/8/12/16." + - "Use the FP8 checkpoint rather than NVFP4 because Hopper has no SM100 tensor cores, and keep the flashinfer attention backend for sm_90." + - "Correct the serve flags to the SGLang cookbook's verified low-latency single-node command for this model: TP4 with EP4 rather than TP8, memory fraction 0.85, chunked prefill 8192, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and the NEXTN speculative algorithm." + - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." + - "Use a float32 Mamba SSM state: flashinfer's gated_delta_rule_mtp verify kernel asserts float32 and aborts CUDA graph capture on the bfloat16 state the cookbook command specifies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2753 + +- config-keys: + - qwen3.8next-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B300 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16." + - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B300 sibling." + - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization." + - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." + - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2752 + +- config-keys: + - qwen3.8next-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B300 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16." + - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B300 sibling." + - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization." + - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." + - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2758 + +- config-keys: + - qwen3.8next-fp8-h100-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the day-zero Qwen3.8-Flash-Next FP8 AgentX recipe on H100 with SGLang native NEXTN MTP at TP8 with EP8 and concurrency 1/4/8/12/16." + - "Mirror the H200 arm, adjusted for the smaller part: TP8 with EP8 rather than TP4 with EP4, and memory fraction 0.75 rather than 0.85." + - "Use a float32 Mamba SSM state, as Hopper's flashinfer verify kernel requires, unlike the bfloat16 the Blackwell arms must use." + - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2756 + +- config-keys: + - glm5.1-fp8-b200-tilert-agentic + description: + - "Port the B200 GLM-5.1 FP8 TileRT AgentX MTP 1P1D configuration from its manual salloc/srun launcher to native srt-slurm orchestration." + - "Use srt-slurm's explicit vLLM-prefill/TileRT-decode topology and the official TileRT P/D router without Dynamo or the legacy submit.sh/run_node.sh control plane." + - "Convert TileRT decode weights once under a shared lock, publish the cache atomically, and keep prefill and decode in their role-specific vLLM 0.26.0 and TileRT 0.1.5 images." + - "Pass only the real vLLM prefill Prometheus endpoint to AIPerf; TileRT decode exposes health and token-transfer APIs but no /metrics route." + - "Filter AgentX traces at the same 202,752-token context limit used by both TileRT roles so oversized Weka trajectories are excluded before replay." + - "Pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, stacked on the AMD/native-router PR #7 and base runtime PR #1, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, backend-declared conversion GPU resources, pre-container NVIDIA driver-hook activation, and lossless Slurm container-environment exports." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2750 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt + - qwen3.5-fp4-gb300-dynamo-trt-mtp + description: + - "Refresh Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated TensorRT-LLM+Dynamo STP and MTP (3-token, with one 2-token point) 8k1k configurations with a new locally validated recipe sweep from srt-slurm." + - "Image bumped from nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 to 1.3.0rc24; kv-p2p-transfer switched from ucx to nixl for the STP config to match the new recipes' NIXL cache-transceiver backend (the MTP config was already nixl)." + - "Switch checkpoint from nvidia/Qwen3.5-397B-A17B-NVFP4 to nvidia/Qwen3.5-397B-A17B-NVFP4-V2 (https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4-V2), matching the checkpoint already used by the dynamo-sglang and dynamo-trt-agentic-disagg GB300 configs." + - "STP: 10 configs from 1P2D (conc 42) to 24P1D (conc 8192)." + - "MTP: 10 configs from 1P2D (conc 20) to 26P1D (conc 4301, 2-token draft); the 24P1D/conc 8192 tail point shares its recipe with the STP curve, where MTP no longer helps at saturation." + - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})." + - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. Extend the TP8 concurrency sweep to [4, 6, 8, 10] to cover the mid-range crossover region." + - "Raise HiCache ratio from 1.5 to 2.5 and switch write policy to write_through_selective (per PR #2679): the agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; the larger host tier sustains throughput through conc 12 (+12% tok/s/GPU vs conc 10). Trim the TP4 concurrency list to [1, 2, 4, 8, 10, 12] removing the untested conc 16 point." + - "Double MAX_RUNNING_REQUESTS to 2×CONC: MTP draft+verify transiently batches more tokens than CONC sessions; the extra headroom prevents scheduler stalls under burst. Cap CUDA_GRAPH_MAX_BS at 64 (SGLang auto-interpolates a batch-size list up to the cap)." + - "Trim TP8/EP=1 concurrency list from [4, 6, 8, 10] to [4, 10]: sweep validation shows c6 and c8 dominated by the TP4/EP4 HiCache arm at those concurrency points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2777 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Split the DeepSeek-V4-Pro FP4 ATOM AgentX MTP sweep into two attention bands per the ROCm/ATOM DeepSeek-V4-Agentic-InferenceX.md recipe: tensor-parallel attention at concurrency 1, 2, 4, 8, and 16, and DP attention at concurrency 48, 64, 96, 128, and 256 (previously a single tensor-parallel band at concurrency 1, 2, 4, 8, 16, 32, and 48)." + - "The DP-attention band adds --enable-dp-attention and --enable-tbo plus ATOM's DPA routing knobs (ATOM_DP_SESSION_AFFINITY, ATOM_DP_LB_REQ_EQUIV, ATOM_ENABLE_PREFILL_DELAYER, ATOM_PREFILL_DECODE_INTERVAL), the two-batch-overlap prerequisites GPU_MAX_HW_QUEUES and ATOM_NUMA_BIND, and the AIPerf session-affinity vars that keep each agentic conversation pinned to one DP rank; the tensor-parallel band is otherwise unchanged." + - "Session affinity is mandatory for the DP band: without it a session's turns scatter across DP ranks, the prefix KV written by one turn is unreachable by the next, and the multi-turn agentic workload collapses to cold prefill." + - "The attention band is selected by the config's dp-attn, propagated to the recipe as DP_ATTENTION, so the script's former DP_ATTENTION=false hard-fail guard is removed." + - "Bump the ATOM image from rocm/atom-dev:nightly_202608201032 to nightly_202608280858; FP8 KV/index caches, prefix caching, 32K state checkpoints, 16K batching/prefill chunks, FULL cudagraph mode, three-token MTP with golden AL 2.49, and max-num-seqs at twice concurrency are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2778 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the B200 SGLang AgentX curve with validated TP8/DP8/EP8 HiCache profiles at DP-attention concurrencies 64, 96, 128, and 160." + - "Use lmsysorg/sglang:nightly-dev-20260827-20621aa1 for the B200 SGLang AgentX sweep." + - "Scope cache-aware routing and --prefill-decode-interval to the DP-attention profiles: use interval 24 at concurrency 64, 96, and 128, and interval 20 at concurrency 160." + - "Use mem-fraction-static 0.88 for every DP-attention profile while retaining 0.90 for TP profiles. Full c96 and c128 sweeps at 0.90 exhausted HBM in the context-dependent FP4-indexer workspace; lowering the fraction adds about 3.57 GiB of headroom per GPU. Across every DP-attention profile, use a 49152-token global chunked-prefill budget, CUDA graph maximum batch size 32, total-requests DP balancing, one-second load snapshots, and correlation-ID session routing; set the cache-aware absolute balance threshold to 32 at concurrency 160." + - "Fit the HiCache tier to NScale host memory by using ratio 2.75 for replicated TP8 profiles and retaining ratio 8 for sharded DP8 profiles. Ratio 8 on TP8 requests about 351 GiB per rank and fails host-pool initialization on a 1.7 TiB NScale node; ratio 2.75 targets about 121 GiB per rank while leaving startup headroom." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2718 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Update the vLLM B200 Kimi-K3 AgentX configs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2776 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the vLLM ROCm image from nightly-cb8104839c141609d99f1254459ef3a4f1bd4263 to nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 and drop apply_k3_container_patches.sh (the patches are now baked into the image); add AITER_SITUV2_A8W4=1." + - "Replace the vllm-simple DRAM offload arm with an LMCache MP-connector arm (lmcache 0.5.5.dev60+rocm7.2), running one per-node MP server with --chunk-size 3072 (the minimum multiple of the hybrid KDA/MLA 1536/3072 KV group sizes) and --separate-object-groups; mq_timeout raised to 6000s for the large agentic prefixes." + - "Add decode context parallel (DCP_SIZE=8, --dcp-comm-backend a2a, TRITON_MLA attention backend) with GPU_MEM_UTIL lowered to 0.85, --max-num-batched-tokens 16384, and mla_prefill_backend=ROCM_AITER_FA." + - "Make speculative decoding concurrency-adaptive: concurrency 1 uses 6 draft tokens (synthetic acceptance 3.75), concurrency <=10 uses 2 tokens (2.51), and higher concurrencies disable MTP; MAX_NUM_SEQS is now 2*CONC with CUDA-graph capture sizes scaled to (1+SPEC_NUM_TOKENS)." + - "Raise dram-utilization from 0.50 to 0.60, extend the none-offload conc-list from [1, 4, 8] to [1, 4, 8, 10, 12], and set the lmcache arm conc-list to [4, 8, 10, 12]." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2787 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from vllm/vllm-openai-rocm:v0.27.1 to vllm/vllm-openai-rocm:nightly-6d4562c59b97b4e35d459ff9389e71b6fe4995de. This pin is what re-enables the AITER shuffled KV cache layout for MiniMax-M3 sparse PA, which was broken by the KV cache layout standardization in https://github.com/vllm-project/vllm/pull/51718. The recipe's TP4 VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 export is unchanged by this PR, so the restored layout behavior is attributable to the image pin rather than to the benchmark script, and the two must move together to reproduce these numbers." + - "Route dense-layer attention and EAGLE3 speculative decoding through ROCM_AITER_UNIFIED_ATTN instead of TRITON_ATTN, both in the server --attention-backend flag and in the speculative-config attention_backend for the throughput branch (synthetic acceptance) and the eval-only branch (real target verification)." + - "Drop the --hf-overrides that forced text_config use_index_cache true and index_topk_freq 4. The pinned nightly supplies the sparse-PA index-cache and top-k refresh behavior itself, so the override is redundant; because it rewrites the served model config, removing it is architecture-affecting rather than a measurement no-op." + - "Drop the TP4 LMCache DRAM KV-offloading sweep points at concurrency 32 and 40 (kv-offload-backend lmcache 0.5.3). The agentic-coding search space is now GPU-resident only, TP4 at concurrency 1 through 32 and TP2 at 1, 2, and 5, so this config no longer reports any DRAM-offload points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2781 + + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the MI355X SGLang DeepSeek-V4-Pro FP4 AgentX MTP image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829." + - "Export TORCH_BLAS_PREFER_HIPBLASLT=1 to prefer hipBLASLt for dense GEMMs." + - "Size chunked-prefill by tensor-parallel degree: 16384 tokens at TP8 and 8192 at TP4, erroring on any other TP." + - "Lower GPU_MAX_HW_QUEUES from 5 to 2 on the DP-attention path." + - "Add a TP8 EP1 no-offload spec-decoding mtp arm at concurrency 16." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2784 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260818 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829" + - "Route multi-GPU collectives through INT8-quantized ROCm quick all-reduce (ROCM_QUICK_REDUCE_QUANTIZATION=INT8), matching the published SGLang cookbook recipe for MXFP4 on MI355X (https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.5). This arm already omits --enable-aiter-allreduce-fusion, so the quick-reduce regime applies with no other launch change: the two paths are mutually exclusive and collectives only fall back to custom all-reduce, where the regime is read, when the fusion flag is off." + - "Halve the prefill budget, --max-prefill-tokens and --chunked-prefill-size from 32768 to 16384, matching the B200 sibling recipe qwen3.5_fp4_b200_sglang_mtp.sh." + - "Capture the decode CUDA graph to min(2*CONC, 128) instead of min(CONC, 64). The replay keeps --max-running-requests 2*CONC in flight, so a CONC-sized graph dropped every decode batch above CONC onto the eager path. The 128 cap follows the sibling MI355X AgentX recipe dsv4_fp4_mi355x_sglang_mtp.sh." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2737 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829" + - "Serve both arms with an fp8_e4m3 KV cache." + - "Route multi-GPU collectives through INT8-quantized ROCm quick all-reduce (ROCM_QUICK_REDUCE_QUANTIZATION=INT8) and drop --enable-aiter-allreduce-fusion, aligning both arms with the published SGLang cookbook recipe for MXFP4 on MI355X (https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.5). The two paths are mutually exclusive in SGLang: the AITER fused AR+RMSNorm path is gated on --enable-aiter-allreduce-fusion, and only with it off do collectives fall back to custom all-reduce where the quick-reduce regime applies. This supersedes the aiter allreduce fusion enabled for these two config keys in #1680, so throughput on this arm is not a like-for-like continuation of the previous MXFP4 series." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2640 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260828" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2763 + + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + - minimaxm3-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Replace the expired vLLM nightly image with the current pinned upstream nightly so MiniMax-M3 AgentX jobs can import their container instead of failing with Docker Hub HTTP 404." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic + scenario-type: + - agentic-coding + description: + - "Correct srt-slurm's external data-parallel process allocation so Kimi-K3 TP4/DP4 workers receive four GPUs per DP rank instead of one invalid rank per GPU." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + - minimaxm3-fp4-b200-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Accept the stock BFCL client's standard store=false chat-completion field instead of rejecting every TensorRT-LLM tool-use request with HTTP 400." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + - minimaxm3-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Keep AgentX dependency setup rootless so minimal vLLM images without git do not fail while attempting a privileged apt install." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Split heterogeneous vLLM layer-compact KV backing storage into per-layer offload regions so MiniMax-M3 cache layouts initialize without invalid tensor views." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Configure model-specific tool and reasoning parsers for the Kimi-K3 and MiniMax-M3 serving paths so opt-in verifier requests produce structured OpenAI tool calls." + - "Limit shared AgentX readiness and synthetic-acceptance rewrites to eval-only runs so default throughput recipe rendering remains unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2634 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update SGLang from lmsysorg/sglang-staging:dev-cu13-pr-35880 to lmsysorg/sglang:nightly-dev-cu13-20260827-20621aa1." + - "Remove DEP8 HiCache concurrency 64 and 128: 128 ran out of GPU memory at mem-fraction-static=0.93; 64 was unmeasured and used the same memory budget." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2759 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang + description: + - "Add a DeepSeek-V4-Pro FP4 Dynamo-SGLang STP configuration for B200 with ten 8k/1k disaggregated topologies." + - "Use Mooncake for KV transfer with raw synthetic benchmark inputs." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2560 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang + description: + - "Switch the DeepSeek-V4-Pro FP4 B200 Dynamo-SGLang STP recipes to NIXL KV transfer." + - "Refresh the SGLang image and Dynamo source revision, and align runtime environment flags with the updated image." + - "Use raw synthetic benchmark inputs across the ten disaggregated 8k/1k STP recipes." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2560 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang + description: + - "Enable the required DCGM power telemetry contract across all ten B200 disaggregated STP recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2560 + +- config-keys: + - qwen3.5-fp4-b200-sglang-mtp + description: + - "Use Triton linear attention with FlashInfer decode and prefill GDN backends for the TP2/EP2 Qwen3.5 FP4 B200 MTP points at concurrency 16, 32, and 64." + - "Keep all other topology and concurrency points on the existing backend defaults." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2790 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2770 + +- config-keys: + - minimaxm3-fp4-b200-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2771 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2772 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2773 + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt-agentx + scenario-type: + - agentic-coding + description: + - "Refresh to collect TensorRT-LLM server metrics." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2774 + +- config-keys: + - glm5.2-fp4-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Pin the GLM-5.2 B200 SGLang AgentX curve to lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729, which includes FlashInfer 0.6.18's BF16 TRTLLM MoE allocation fix." + - "Use a 169 GB/rank HiCache target pool at c12 and c16 while retaining ratio mode for lower concurrencies." + - "Isolate SGLang runtime caches per Slurm allocation to prevent concurrent sweep cells from sharing per-rank cache files." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2808 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the MI355X K3 vLLM recipe to vLLM ROCm nightly 7c5dc571 and LMCache 0.5.5rc3." + - "Add a TP8/DCP8 spec-mtp DRAM-offload arm at concurrency 44/48/52." + - "Retune the LMCache tier (12288-token chunk, 8 GPU workers) and the no-spec decode profile (gpu-memory-utilization 0.9, 8192 batched tokens); switch the DCP attention backend to ROCM_AITER_MLA." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2810 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add a GLM-5.2 concurrency-1 AgentX point using a 1-prefill/1-decode disaggregated topology with TP4 attention-DP prefill and TP8 decode." + - "Use MTP depth 5 with the committed golden acceptance target, tensor-parallel MoE on decode, and the CUTEDSL top-k path." + - "Align recipe KV offloading with the master config and keep the benchmark client colocated with the frontend orchestrator." + - "Add the new concurrency-1 topology to the existing disaggregated configuration key while retaining all published points." + - "Enable TensorRT-LLM iteration statistics and require KV-cache server metrics for the new concurrency-1 point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2720 + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt-agentx + scenario-type: + - agentic-coding + description: + - "Set OMP_NUM_THREADS=1 for the prefill and decode workers across the GB300 Dynamo-TensorRT-LLM AgentX recipe matrix." + - "Route AgentX replay and drain checks through the srt-slurm frontend endpoint when provided, with a localhost fallback." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2711 + +- config-keys: + - dsv4-fp4-b200-vllm + description: + - "Refresh the DeepSeek-V4-Pro FP4 B200 vLLM image and search space, pin the Nscale NVFP4 checkpoint, use the FlashInfer CuTeDSL MoE backend for expert-parallel runs with EPLB disabled, set GPU memory utilization to 0.95, and cap the model length at 12288." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2534 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Refresh the Kimi-K3 GB300 AgentX configurations with updated configs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2811 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the MI355X MiniMax-M3 AgentX image from vllm/vllm-openai-rocm:nightly-6d4562c59b97b4e35d459ff9389e71b6fe4995de to the latest immutable nightly, vllm/vllm-openai-rocm:nightly-8a728663c1c3eeace834a95f5654fa653cc1998c (digest sha256:0d07767c0b8471eaca0f1eca97899620dfb4ad33ff2f5226789afef1a877e9bc)." + - "Pick up the optimized BF16 MiniMax-M3 indexer and routed GEMM from vLLM PRs #54682 and #54845; use FULL_DECODE_ONLY CUDA graphs while breakable CUDA graphs remain disabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2825 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260902." + - "Add a DP-attention arm: tp 8, ep 1, dp-attn true, hicache host KV tier, concurrency [64, 96, 128, 160]. It runs behind sglang-router with consistent hashing on the AIPerf correlation id so multi-turn sessions stay on the DP rank holding their prefix." + - "Serve with --enable-deepseek-v4-fp4-indexer and set HSA_NO_SCRATCH_RECLAIM=0 on all arms." + - "DP-attention serving flags: --enable-two-batch-overlap, --enable-dp-attention-local-control-broadcast, --tokenizer-worker-num equal to TP, --stream-interval 20 and --prefill-decode-interval 10, alongside the existing --enable-prefill-delayer. Chunked prefill now scales the per-TP base by the DP degree instead of a fixed 8192." + - "Shared-experts fusion is now per-arm: kept enforced on the tensor-parallel arms and disabled under DP attention, matching the DP baselines." + - "GPU_MAX_HW_QUEUES is now set explicitly per-arm: 2 on the tensor-parallel arms and 5 under DP attention, the documented companion to two-batch overlap." + - "Lower mem-fraction-static from 0.89 to 0.86 on every arm: TP4, TP8 and TP8 with DP attention. swa-full-tokens-ratio becomes per-arm: 0.10 on the tensor-parallel arms as before, 0.15 under DP attention." + - "Drop concurrency 2 and 10 from the TP4 arm, leaving [1, 4, 8], and drop concurrency 16 from the TP8 hicache arm, leaving [32, 48]. Concurrency 16 remains on the TP8 no-offload arm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2800 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Repin the image from vllm/vllm-openai-rocm:nightly-311b3513af33bc29b4acb2fde2e9313e5e9966a0 to the 2026-09-05 ROCm nightly (nightly-e962733e08d10f7ca65dac4df99e116460b8b174, digest sha256:511755968434e26da590c3398f1e8a6399be4416226d5779ce92fb0655811a9c). Docker Hub last pushed it at 2026-09-05T05:27:09Z and the tag commit is vllm-project/vllm@e962733e." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2841 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the MI355X Kimi-K3 FP4 ATOM AgentX submission onto image kimi_k3_agentic_0907 and publish the full validated concurrency set [1, 2, 4, 8, 12, 14, 16, 32, 40, 56, 64], replacing [1, 2, 4, 8, 12, 32, 40, 56]. Concurrency 14, 16, and 64 are new points and every existing point is retuned." + - "Capture full CUDA graphs over the dense range [2 .. window x (1 + draft tokens)] with --level 3, --cudagraph-mode FULL, and an explicit --cudagraph-capture-sizes. The window is 2 x concurrency, which is what the AgentX client can put in flight per lane, and the (1 + draft tokens) factor covers the verify step of a DSpark round, which submits one row per draft token on top of the accepted token. Without the explicit list every speculative decode falls back to the eager path." + - "Run concurrency 14 on the concurrency 16 server verbatim, including the CUDA-graph width pinned at a window of 32; only the client concurrency differs. Deriving the width from 2 x concurrency there gives 28 and the server dies during graph warmup." + - "Select max-num-seqs, max-num-batched-tokens, gpu-memory-utilization, ReplaySSM, and draft depth by concurrency: 32/8192/0.88, ReplaySSM off, 7 draft tokens at concurrency 1-4; 32 and 24 with a 4096-token prefill step at 0.88 and ReplaySSM on with 3 draft tokens at concurrency 8 and 12; 32/8192/0.86 with ReplaySSM on and 3 draft tokens at concurrency 14 and 16; and 64/80/112/128 max-num-seqs at 8192/0.86 with ReplaySSM off and no draft model from concurrency 32 up." + - "Drop the CPU state-offload tier. The Kimi Delta Attention recurrent state is now rebuilt in place by ReplaySSM from the in-GPU checkpoint ring on every concurrency that keeps a draft model, so OFFLOAD_STATE, OFFLOAD_STATE_CPU_SIZE, OFFLOAD_STATE_STAGING_GROUPS, OFFLOAD_STATE_MIN_LOAD_TOKENS, and the --state-checkpoint-slots override are all gone and the whole per-rank CPU budget goes to the paged KV." + - "Size the paged-KV tier from two dram-utilization blocks, still divided by TP for the per-rank LMCACHE_MAX_LOCAL_CPU_SIZE: 0.343 lands 128 GB per rank for concurrency 8 through 40, and 0.513 lands 192 GB per rank for concurrency 56 and 64, where the wider in-flight window needs the deeper pool." + - "Pin the eight ranks across the two sockets explicitly with ATOM_NUMA_BIND=1, ATOM_NUMA_NODE=0,0,0,0,1,1,1,1, ATOM_AUTO_NUMA_BIND=0, and LMCACHE_NUMA_MODE=auto. Auto-binding is disabled precisely so the explicit mapping takes effect; a pinned pool on the wrong socket pays a cross-socket hop on every offload read." + - "Set AITER_REUSE_IDENTICAL_COMM_GROUPS=1 only at concurrency 56 and 64, and leave it off everywhere else." + - "Stage the Inferact/Kimi-K3-DSpark draft into the shared Hugging Face cache before the server starts, with the same five-attempt retry the MiniMax-M3 H200 launcher uses. The ATOM image carries no cache entry for the draft, and an uncached repo id makes every rank pull the same 7 GB checkpoint at once. The failure is silent: the log stops after loading the drafter, no error is printed, every GPU sits at 0% with the target weights already resident, and the pull was measured at ~0.7 MB/s on one cluster, about three hours." + - "Clear HTTP_PROXY, HTTPS_PROXY, http_proxy, and https_proxy at the top of the launcher. The 0907 image was built with podman on a host behind a local squid and the build environment leaked into the image config: all four are baked in as http://127.0.0.1:3128, with NO_PROXY covering only localhost. Nothing listens on that port inside the container on a benchmark node, so the uv bootstrap and aiperf install, the trace-corpus fetch, and both hf downloads would all reach a dead proxy before the server is started. The 0821 and 0903 images carried none of these." + - "Register the server under --served-model-name $MODEL. The AgentX client asks for the model by that name on the wire, and a mismatch with whatever MODEL_PATH resolves to returns HTTP 404 at warmup." + - "Set LMCACHE_CHUNK_SIZE to 1024 to stay aligned with the block-size 128 times dcp 8 offload hash block. Keep AITER INT4 quick-reduce, AITER SITUV2 A4W4, FlyDSL stage-2 FP8, ptpc_fp8 online quantization excluding embeddings, lm_head, conv1d projections, experts, and the multimodal tower, an FP8 KV cache, a 128-token page, --state-checkpoint-interval-tokens -1, prompt-end-anchor-only state checkpointing, OFFLOAD_PROFILE=1, and OFFLOAD_GPU_STAGING_CHUNKS=32. Keep OFFLOAD_KV_FOR_HYBRID=1 as well, which the ATOM recipe no longer lists: K3 is a hybrid, and if that gate is still off by default the paged KV never reaches the CPU tier at all, which reads as a tier that is on and idle rather than as an error. Setting it is a no-op if the default flipped or the knob went away." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2852 + +- config-keys: + - dsv4-fp4-mi355x-vllm + - dsv4-fp4-mi355x-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - "Bump the DeepSeek-V4-Pro FP4 MI355X 8k/1k STP and MTP images from nightly-09663abde0f50944a8d5ea30120666024b503faa to the 2026-09-01 ROCm nightly nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb (digest sha256:f0bdaf5217a09949842b45c1ea1f12260d3205ec81f143b320dfc2eb3ec95e55)." + - "Pick up the latest automatic ROCm DeepSeek-V4 optimizations, including fused mHC post/pre plus RMSNorm, gfx950 C4A top-k dispatch, fused C4 compressor GEMMs, fused SWA q/kv RMSNorm plus q FP8 quantization, and medium-batch cooperative top-k tuning." + - "Keep the existing VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, and --moe-backend aiter settings, and explicitly add VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 plus VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 to both STP and MTP paths. The current checkpoint's shared-expert path does not satisfy the latest vLLM fusion conditions, so that fusion flag self-disables while preserving recipe parity." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2792 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-mtp + - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update SGLang image to lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, digest sha256:19b8fa1223cc339c1eae7a5b703f1a8c2543b5b119155bf3d7efaef18f77f007, tag commit sgl-project/sglang@30705c00; Docker Hub last pushed 2026-09-07T01:43:42Z) for both H200 Qwen3.5 FP8 SGLang AgentX MTP recipes: qwen3.5-fp8-h200-sglang-agentic-mtp from lmsysorg/sglang:v0.5.16-cu130 (v0.5.16 release) and qwen3.5-fp8-h200-sglang-agentic-hicache-mtp from lmsysorg/sglang:nightly-dev-cu13-20260815-a5ba081f (2026-08-15 nightly). Both route to benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh, which is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. Concurrency grids unchanged. Same tag the B200 Qwen3.5 FP8/FP4 SGLang AgentX recipes moved to in #2861/#2862." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2868 + +- config-keys: + - minimaxm3-fp8-h200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update vLLM image from vllm/vllm-openai:v0.27.1 (v0.27.1 release) to vllm/vllm-openai:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 (2026-09-07 upstream nightly, digest sha256:254eebf919e8b7b0d530d97fccc606c36f380ff6724d64bc190951bec1aee838, tag commit vllm-project/vllm@d9105ea8; Docker Hub last pushed 2026-09-07T06:16:01Z), the same tag the B200 MiniMax-M3 AgentX recipe moved to in #2860 and the ROCm counterpart of the MI325X/MI300X bumps in #2872/#2873. benchmarks/single_node/agentic/minimaxm3_fp8_h200_mtp.sh is unchanged: TRITON_ATTN attention, fp8 KV, EAGLE3 with the Inferact MiniMax-M3 EAGLE3-GQA draft pinned to FLASH_ATTN and the committed golden synthetic acceptance length 2.78, Mooncake 0.3.11.post1 DRAM offload on the host-tier arm; the resident TP8 c1/c2/c4/c6/c8/c10 and Mooncake DRAM offload c12/c14 grid is unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2875 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update vLLM ROCm image from vllm/vllm-openai-rocm:v0.27.1 (v0.27.1 release) to vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 (2026-09-07 upstream ROCm nightly, digest sha256:74d4a95f3ae672ecddf9acb7296917def82d9eca51687fa2862ae72b03ff1907, tag commit vllm-project/vllm@d9105ea8; Docker Hub last pushed 2026-09-07T05:26:48Z). benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh is unchanged: TRITON_ATTN attention, fp8 KV, block-size 128, EAGLE3 speculative decoding with the Inferact MiniMax-M3 EAGLE3-GQA draft and the committed golden synthetic acceptance length, minimax_m3 tool-call and reasoning parsers; the search space is unchanged. The upstream commit-pinned ROCm nightly is the same vllm commit the B200 MiniMax-M3 AgentX recipe moved to in #2860. Note that vllm-openai-rocm commit-nightly tags have expired from Docker Hub within days in the past; node squash caches keep merged configs running, but a re-pin to a durable tag may be needed later." + - "Add --compilation-config cudagraph_mode=FULL_DECODE_ONLY to the serve command. The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE mode aborts at engine init with piecewise CUDA graphs unavailable (first sweep, run 34174124043, eval cell); full decode-only graphs are what the MI355X MiniMax-M3 sibling runs on its nightly (#2825)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2873 + +- config-keys: + - qwen3.5-fp8-b200-sglang + scenario-type: + - fixed-seq-len + description: + - "Bump the Qwen3.5 FP8 B200 SGLang image from lmsysorg/sglang:v0.5.14-cu130 to lmsysorg/sglang:v0.5.19-cu130." + - "Tune the 8k/1k serving recipe with --mamba-full-memory-ratio 0.37 and --linear-attn-prefill-backend flashinfer; raise max-prefill-tokens and chunked-prefill-size from 16384 to 32768 and mem-fraction-static from 0.8 to 0.86." + - "Expand the TP8 and TP4/EP1 sweep coverage through concurrency 640." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2866 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Replace the B200 DeepSeek-V4-Pro AgentX native EAGLE/MTP serving path with DSpark block size 6 on the DeepSeek-V4-Pro-0813 checkpoint." + - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP-attention instead of the deprecated MegaMoE environment variables." + - "Resolve draft-model jobs through the existing B200 SGLang speculative recipe and the staged DeepSeek-V4-Pro-0813 checkpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2821 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Upgrade the Kimi-K3 MI355X vLLM AgentX LMCache build from 0.5.5.dev89+rocm7.2 to 0.5.5.dev114+rocm7.2 from the nightly ROCm release index; the rolling release no longer publishes the intermediate 0.5.5.dev104+rocm7.2 wheel." + - "Preserve the image, runtime settings, and topology from the full sweep in Actions run 33773561410; run concurrency 1 without offload and LMCache concurrencies 4, 8, 10, 12, 14, 44, and 48, with 48 as the upper bound." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2889 + +- config-keys: + - dsv4-fp4-mi355x-vllm + - dsv4-fp4-mi355x-vllm-mtp + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - fixed-seq-len + - agentic-coding + description: + - "Repin all three DSv4 MI355X vLLM entries to vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 (Docker Hub pushed 2026-09-07T05:26:48Z): the 8K/1K STP and MTP entries from nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb (#2792) and AgentX from nightly-e962733e08d10f7ca65dac4df99e116460b8b174 (#2841). d9105ea8 is the first ROCm nightly that ships vllm-project/vllm#53161 (vllm-project/vllm@de69e821), which fuses the DeepSeek-V4 native-FP8 shared expert into the MXFP4 routed-expert AITER kernel (FHMoE). The tag commit is 7 commits ahead of de69e821; the two superseded pins are 266 and 28 commits behind it, so both predate FHMoE." + - "No new serving flags. All three scripts already export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 (#2792 for 8K/1K STP and MTP, #2841 for AgentX), but on every nightly before de69e821 the request was inert: vLLM's eligibility check rejected this mixed FP4+FP8 checkpoint and the fusion self-disabled at startup. This pin is what makes the existing flag take effect, so the comments above it in the two 8K/1K scripts and on the two 8K/1K entries in configs/amd-master.yaml now say what the fusion does and which upstream commit it needs, instead of why it self-disables." + - "Fusion eligibility is gated on gfx950, tensor parallelism 8, data parallelism 1, no expert parallelism and no EPLB, --moe-backend aiter, BF16 model dtype, and the DeepSeek-V4 block-128 FP8 shared-expert quantization config. Both 8K/1K entries and the AgentX TP8 arm satisfy it; the AgentX dp-attn conc-64 arm runs data parallelism 8 and keeps the unfused path." + - "Export OMP_NUM_THREADS=1 from all three scripts. vLLM clamps torch threads itself, but only after the weights are loaded: the 'Reducing Torch threads from 16 to 1 for serving' line is emitted after 'Model loading took', so the checkpoint is read with 16 torch threads per worker across 8 workers. Setting it externally caps threads from process start, which is what that log line recommends." + - "Weight loading currently costs 5760-5946 s (96-99 min) per job on this runner, measured across the c1, c4, c56 and c64 AgentX jobs of run 34087124574 - roughly half of each ~3 h job. It does not affect any measured number: loading completes before warmup and the profiling phase. The effect is also only observable on this runner. A local EXT4 box serves the checkpoint from page cache and loads in ~40 s with or without the cap, so it never exercises the cold-read path. CI reads cold from WEKAFS, where each of the 8 ranks streams the full 805 GiB to slice its own shards - roughly 6.4 TB of aggregate network reads per job - and that is where oversubscribed reader threads cost real time." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2851 + +- config-keys: + - dsv4-fp4-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." + - "Replace the unconditional --disable-shared-experts-fusion with --enforce-shared-experts-fusion, and move the disable to the EP > 1 branch only." + - "Add TORCH_BLAS_PREFER_HIPBLASLT=1 and SGLANG_OPT_USE_AITER_BATCHED_GEMM=1." + - "Set GPU_MAX_HW_QUEUES=5 on the DP-attention path, matching the spec-none sibling dsv4_fp4_mi355x_sglang.sh." + - "Add --enable-deepseek-v4-fp4-indexer." + - "Drop conc 8 from the TP8 no-offload arm, which now sweeps conc [1, 2, 4, 16, 32]." + - "Set SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE=1 on the DP-attention path." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2910 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto + scenario-type: + - agentic-coding + description: + - "Add five GB300 Qwen3.5 NVFP4 AgentX Pareto points on lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198." + - "Use PP4 prefill with DEP4 decode at concurrency 704 and 565, plus two distinct TP2 aggregate concurrency-44 recipes and one TP8 aggregate concurrency-7 recipe." + - "Keep full MTP and HiCache in every recipe; throughput uses the committed synthetic acceptance lengths while EVAL_ONLY keeps real MTP behavior." + - "Run the upstream SGLang image as shipped and pin the PP recipes plus router identity to ai-dynamo/dynamo main commit 61c37bc33c3e88da3facaa39fa2b5caf616740ba." + - "Match thinking-on Golden AL to SGLang speculative steps: 4.41 at C704 (5 steps), 3.97 at C565 (4 steps), and 4.8 for all three aggregate points (6 steps); select multi-node benchmark mode for the two-node TP8 point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2812 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the full GB200 Dynamo-vLLM AgentX configuration across the configured aggregate and disaggregated topology points." + - "Retain aggregate SimpleCPU-offload concurrency 20 and 30, and configure Mooncake host KV storage, NIXL over UCX, explicit thinking mode, and the committed MiniMax-M3 EAGLE3-GQA acceptance target." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2807 + +- config-keys: + - qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the GB200 Qwen3.5 NVFP4 Dynamo-SGLang AgentX matrix with an updated TP4/TP2-EP2 concurrency frontier." + - "Route the matrix through Dynamo over SGLang with a pinned srt-slurm revision, and restore the current writable Watchtower shared root and Slurm partition/account for GB200 workspaces, squash images, and AgentX caches." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2945 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Replace the GB300 DeepSeek-V4-Pro AgentX MTP serving path with DSpark block size 6 across aggregate and disaggregated recipes." + - "Use the DeepSeek-V4-Pro-0813 checkpoint, SGLang 20260829 CUDA 13 nightly, public Dynamo 1.5.0.dev20260902 wheels containing the upstream SGLang config-parser compatibility fix, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Retune the Pareto ladder to concurrency 1, 4, 8, 320, 640, 960, and 1280, while retaining the concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Increase the DEP16 concurrency ladder from 320, 640, 960, and 1280 to 480, 960, 1440, and 1920 after profiling showed request-starved prefill workers at the lower values." + - "Retain the independently tuned concurrency-1152 2P DEP8 prefill plus DEP12 decode efficiency point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Remove the unstable concurrency-1152 2P DEP8 prefill plus DEP12 decode point after repeated benchmark failures." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2623 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." + - "Keep TP-only chunked-prefill at 8192 for TP4 and 16384 for TP8. The DP-attention path uses an 8192 per-rank budget (engine-wide 8192*TP, 65536 at TP8) instead of scaling the TP8 16384 base." + - "DP-attention serving: drop --enable-two-batch-overlap; set mem-fraction-static to 0.92 (pure TP stays 0.86); keep --enforce-shared-experts-fusion and move --disable-shared-experts-fusion to the EP branch only; keep swa-full-tokens-ratio at 0.10; add --prefill-delayer-token-usage-low-watermark 0.7." + - "Remove the 256 cap on max-running-requests." + - "MTP draft length by concurrency: speculative-num-steps 3 (golden AL 2.49) below conc 256, and 1 (golden AL 1.79) at and above it." + - "Trim the search space to TP8 no-offload conc [1, 4, 16], TP8 hicache conc [32, 48], and TP8 DP-attention hicache conc [128, 256]." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2885 + +- config-keys: + - dsr1-fp8-mi325x-sglang + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-rocm700-mi30x to lmsysorg/sglang:v0.5.19-rocm700-mi30x" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2949 + +- config-keys: + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.14-cu130 to lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9)." + - "Model nvidia/Qwen3.5-397B-A17B-NVFP4-V2, TP4/EP1 and TP2/EP2 topologies, the 8k/1k workload, EAGLE (3,1,4) MTP settings and the launch script are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2950 + +- config-keys: + - qwen3.5-fp8-h100-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.14-cu130 to lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9); recipe script, TP8/EP8 topology and EAGLE/MTP settings unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2952 + +- config-keys: + - qwen3.5-fp4-b200-sglang + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.14-cu130 to lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9)." + - "Model nvidia/Qwen3.5-397B-A17B-NVFP4-V2, TP4/EP1 and TP2/EP1 topologies, the 8k/1k workload and the launch script are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2954 + +- config-keys: + - dsr1-fp8-h200-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, sglang commit 0bcd822377da7b5718e674eaf9c870d349424dd1); recipe script, TP8/EP1 topology and EAGLE/MTP settings unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2955 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + description: + - "Update the SGLang image from nightly-dev-cu13-20260829-89816a21 to the digest-pinned v0.5.19-cu130 release (lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9)." + - "Move the TP8 and TP4 low-latency recipes and the master router metadata from Dynamo 1.5.0.dev20260902 to 1.5.0.dev20260910, the nightly that carries the ServerArgs.get_model_config()/use_mla_backend() compatibility fix for SGLang #36972 and pins sglang==0.5.19." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2969 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 NVFP4 TensorRT-LLM AgentX on GB200 with Dynamo aggregated serving, TP4 plus EAGLE3-GQA with three draft tokens, seven concurrency-specific recipes, and 128 GiB per-rank host KV cache." + - "Add the GB200 launcher route and use NVIDIA/srt-slurm v1.0.91 for aggregated TensorRT-LLM workers." + - "Add TensorRT-LLM synthetic-acceptance injection for AgentX throughput runs while leaving eval-only runs on real acceptance." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2977 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + scenario-type: + - agentic-coding + description: + - "Update the MiniMax-M3 GB200 aggregate configuration to TP4 resident, TP4 lazy SimpleCPU KV offload, and two-node TP8 resident variants." + - "Pin vLLM nightly-9ea8f3ffc354901b740f0b31988900897b7221d7, Dynamo 1.5.0.dev20260908, and NVIDIA/srt-slurm d50ee7280c33d469df8708e363e23be2456e94fb." + - "Use Dynamo-native MiniMax parsing, EAGLE3 with FlashInfer, FULL_AND_PIECEWISE CUDA graphs, and wait for the OpenAI chat route before AgentX trace replay." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2925 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp + description: + - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, build commit sgl-project/sglang@30705c00) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, Docker Hub last pushed 2026-09-04T22:50:19Z)." + - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 + +- config-keys: + - dsv41flash-fp4-h200-vllm-agentic-dspark + description: + - "Add the H200 AgentX arm for DeepSeek-V4.1-Flash at TP8 with native five-token DSpark and adaptive real verification" + - "Engram tables offload to host DRAM through UVA while the KV cache stays GPU-resident; concurrency 1-128 on the uncapped 062126 trace corpus" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2963 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the GLM-5.2 NVFP4 B300 SGLang AgentX MTP TP8 HiCache curve from concurrency 16 to 48 with new points at 20, 24, 28, 32, 40, and 48." + - "Move the recipe from the retired cluster:b300-nv fleet to cluster:b300-dsxe so the sweep can be generated and scheduled again." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2829 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use the pinned SGLang nightly-dev-cu13-20260901-07c8f729 image with FlashInfer 0.6.18, which includes the BF16 TRTLLM MoE allocation fix for small-batch Blackwell execution. Model, TP8, HiCache, MTP settings and concurrency grid are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2829 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the GLM-5.2 NVFP4 B300 SGLang AgentX MTP TP8 HiCache curve from concurrency 16 to 48 with new points at 20, 24, 28, 32, 40, and 48." + - "Move the recipe from the retired cluster:b300-nv fleet to cluster:b300-dsxe so the sweep can be generated and scheduled again." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2994 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use the pinned SGLang nightly-dev-cu13-20260901-07c8f729 image with FlashInfer 0.6.18, which includes the BF16 TRTLLM MoE allocation fix for small-batch Blackwell execution. Model, TP8, HiCache, MTP settings and concurrency grid are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2994 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Add GB300 TP4 AgentX through concurrency 128 with Engram UVA DRAM offload and native five-token DSpark" + - "Use deepseekv41-flash-0909 with adaptive verification and available-port selection" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2961 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Align with the official single-node TP recipe: default model runner and scheduler batching" + - "Keep AgentX concurrency 1-128, UVA offload, DSpark, and concurrency-sized graph capture" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2961 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2961 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Allow 7200 seconds for GB300 DSv4.1 Flash engine startup; the one-hour frontend deadline interrupted cold graph capture" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2961 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Add B200 TP4 AgentX through concurrency 128 with Engram UVA DRAM offload and native five-token DSpark" + - "Use deepseekv41-flash-0909 with adaptive verification and available-port selection" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2960 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Align with the official single-node TP recipe: default model runner and scheduler batching" + - "Keep AgentX concurrency 1-128, UVA offload, DSpark, and concurrency-sized graph capture" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2960 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2960 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + description: + - "Add B300 TP4 AgentX through concurrency 128 with Engram UVA DRAM offload and native five-token DSpark" + - "Use deepseekv41-flash-0909 with adaptive verification and available-port selection" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2958 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + description: + - "Align with the official single-node TP recipe: default model runner and scheduler batching" + - "Keep AgentX concurrency 1-128, UVA offload, DSpark, and concurrency-sized graph capture" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2958 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + description: + - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2958 + +- config-keys: + - dsv41flash-fp4-gb200-vllm-agentic-dspark + description: + - "Add GB200 TP4 AgentX concurrency 1-128 with native DSpark and Engram UVA DRAM offload" + - "Use the official single-node recipe and full semianalysis_cc_traces_weka_062126 traces" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2971 + +- config-keys: + - dsv41flash-fp4-gb200-vllm-agentic-dspark + description: + - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2971 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + description: + - "Update the SGLang image from nightly-dev-cu13-20260829-89816a21 to the digest-pinned v0.5.19-cu130 release (lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9)." + - "Move the TP8 and TP4 low-latency recipes and the master router metadata from Dynamo 1.5.0.dev20260902 to 1.5.0.dev20260910, the nightly that carries the ServerArgs.get_model_config()/use_mla_backend() compatibility fix for SGLang #36972 and pins sglang==0.5.19." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2969 + +- config-keys: + - dsv41flash-fp4-h100-vllm-agentic-dspark + description: + - "Add the H100 AgentX arm for DeepSeek-V4.1-Flash at TP8 with native five-token DSpark and adaptive real verification, concurrency 1-4" + - "Cap max-num-batched-tokens at 4096 so the sparse attention indexer's [batched-tokens, max-model-len] buffer fits an 80 GB card at 1M context, and keep the uncapped 062126 trace corpus" + - "Measured at c1: 13.47 GiB KV per GPU, 7,022,899 tokens, 6.70x maximum concurrency at 1M tokens per request; the concurrency list stays under that ceiling" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2974 + +- config-keys: + - dsv41flash-fp4-h100-vllm-agentic-dspark + description: + - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2974 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260908." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2906 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the GB300 GLM-5.2 NVFP4 AgentX disaggregated Pareto curve: move the 5P+1D DEP16 point to concurrency 260, replace the 6P+1D DEP8 concurrency-227 point with 4P+1D plus a 128 GiB decode host KV cache, retire the 8P+1D DEP16 concurrency-259 point, add a 1P+1D concurrency-1 point, and remove the separate glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp key." + - "Bump all six recipes to tensorrt-llm/release:1.3.0rc26.dev202609040000, add TRTLLM_FUSED_DSA_METADATA=1 and TRTLLM_DSA_INDEXER_BF16=1, and run the AgentX client on a dedicated allocation node." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2939 + + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Reserve 144 CPU cores for the GB300 single-node serving task" + - "为 GB300 单节点服务任务预留 144 个 CPU 核心" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3017 + +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + - qwen3.5-fp8-h100-sglang + description: + - "Resolve the requested H100 SGLang container instead of a hardcoded older image, and stop benchmark/eval clients when their ready server or required worker exits." + - "H100 SGLang 使用所请求的容器而非硬编码旧镜像;已就绪的服务或必需工作进程退出后,停止其 benchmark/评测客户端。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3020 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + - kimik3-fp4-b300-vllm-agentic-dspark + - dsv4-fp4-b300-vllm-agentic-mtp + - qwen3.5-fp8-b300-sglang-agentic-mtp + - qwen3.5-fp4-b300-sglang-agentic-mtp + - qwen3.8next-fp4-b300-sglang-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Route ordinary B300 AgentX configurations through the registered DSXE runner after retirement of the B300 NV fleet; preserve model, image, serving flags, and concurrency grids. Dedicated power-ab configurations remain unchanged." + - "将普通 B300 AgentX 配置切换到已注册的 DSXE runner,保留模型、镜像、服务参数及并发配置;不修改独立 power-ab 配置。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2927 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Add draft DeepSeek-V4.1-Flash FP4 AgentX on MI355X: TP4, concurrency 1-32, native five-token DSpark with adaptive real verification" + - "Use upstream ROCm AITER recipe with GPU-resident KV on the official vllm/vllm-openai-rocm:deepseekv41-flash-0909 image; Engram compatibility and GPU validation pending" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Pin AgentX to semianalysis_cc_traces_weka_062126, preventing the 256k corpus override" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Replace unavailable Docker Hub image with public.ecr.aws/q9t5s3a7/vllm-release-repo:79a7108d9aea27ddab99ce1779290d300b17fc23-rocm" + - "Keep TP4, DSpark settings, and the full-context semianalysis_cc_traces_weka_062126 corpus" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Set INFMAX_CONTAINER_WORKSPACE=/ix for the MI355X DS V4.1 mount so AgentX dependencies and outputs resolve correctly" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Set VLLM_USE_BREAKABLE_CUDAGRAPH=1. Upstream does not torch-compile DeepseekV41ForCausalLM, so the default cudagraph_mode=FULL_AND_PIECEWISE aborted at engine init with piecewise CUDA graphs unavailable (run 34566727564, all concurrencies); the model targets the breakable cudagraph path" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." + - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证:它会在设备端裁剪验证请求,而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作,导致仅评测引擎拒绝启动(运行 34651830283,c32)。评测仍保留真实块拒绝采样;吞吐设置不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Enable required power for the seven existing B200 Kimi-K3 recipes, use the pinned AgentX producer and shared collector, fix the exporter import URI, and record DCP8 with offload disabled to match the serving commands." + - "启用 Kimi-K3 B200 实测功耗并修正 DCP 与 Offload 元数据。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3043 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." + - "保留请求失败状态及全部可用的固定序列诊断信息;发布基准产物前拒绝缺失并发点的批次。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + scenario-type: + - fixed-seq-len + description: + - "Add representative AMD coverage for fixed-sequence request diagnostics and multinode concurrency filename processing; shared processor changes also affect unlisted fixed-sequence recipes." + - "增加固定序列请求诊断与多节点并发文件名处理的代表性 AMD 覆盖;共享处理器变更也影响未列出的固定序列配置。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." + - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." + - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." + - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." + - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." + - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 + +- config-keys: + - glm5.1-fp8-b200-tilert + scenario-type: + - fixed-seq-len + description: + - Require native power only for TileRT 8k1k; keep native launches inside the owned + allocation and retain both role exit statuses. + - 仅对 TileRT 8k1k 要求原生功耗;原生启动始终使用本任务分配,并保留两个角色退出状态。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057 + +- config-keys: + - glm5.1-fp8-b200-tilert + description: + - "Preserve an explicit prepared GLM-5.1 model path and mount shared HF cache blobs in native TileRT containers." + - "保留显式指定的 GLM-5.1 预备模型路径,并在原生 TileRT 容器中挂载共享 HF 缓存 blobs。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057 + +- config-keys: + - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Collect shared measured GPU power for the existing two-node GB200 GLM-5.2 aggregate AgentX recipe with producer 80d7203e424f903c9017de4608ee2044afce9574, exact selected-concurrency windows, and post-job adapter validation. Preserve native status and invalid diagnostics before failure. Ordinary Slurm limits, serving settings, other AgentX recipes, and fixed-sequence producer pins are unchanged." + - "为现有双节点 GB200 GLM-5.2 聚合 AgentX 配方接入共享 GPU 实测功耗,按所选并发绑定测量窗口,并在任务结束后通过适配器校验。返回失败前保留原生状态及无效诊断;普通 Slurm 时限、服务配置、其他 AgentX 配方和固定序列 producer 版本保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2928 + +- config-keys: + - dsv41flash-fp4-h100-vllm-agentic-dspark + description: + - "Rerun the H100 DeepSeek V4.1 Flash AgentX curve at concurrency 1, 2, 4, 8, 16; retain TP8, GPU-resident KV, full traces, 1M context and DSpark5 synthetic AL 3.51" + - "重新运行 H100 DeepSeek V4.1 Flash AgentX 曲线,并发为 1、2、4、8、16;保留 TP8、GPU 驻留 KV、完整轨迹、1M 上下文及 DSpark5 合成 AL 3.51" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3070 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Enable the two GB300 aggregate recipes and their common launcher power route, with the pinned producer, strict result collection and retained diagnostics." + - "为 GB300 Kimi-K3 聚合部署启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Use port 19401 for the GB300 aggregate exporter; the two-node shared lifecycle check verified ownership, all eight GPU telemetry streams, and preservation of the system service on port 9401. Full Kimi-K3 qualification remains pending." + - "GB300 聚合部署改用已验证的 19401 功耗端口;完整 Kimi-K3 基准验证仍待完成。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 + +- config-keys: + - dsv41flash-fp4-h200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Rerun the existing H200 DeepSeek V4.1 Flash GPU-KV AgentX sweep at concurrency 1, 2, 4, 8, 16, 32, 64, 128 with the unchanged TP8, full-trace, 1M-context and DSpark5 AL 3.51 recipe" + - "使用现有 TP8、完整轨迹、1M 上下文和 DSpark5 AL 3.51 配方,重新运行 H200 DeepSeek V4.1 Flash GPU-KV AgentX sweep,并发为 1、2、4、8、16、32、64、128" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3077 + +- config-keys: + - kimik3-fp4-h200-vllm-agentic-latency + scenario-type: + - agentic-coding + description: + - "Enable required power for the H200 Kimi-K3 latency recipe and route their launch, concurrency injection, failure handling and result validation through the pinned producer and shared collector." + - "仅为 H200 Kimi-K3 latency 配方启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3044 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Replace unavailable vLLM build 5894fdf with 3696c77 for the existing Kimi-K3 B300 DSXE route. This establishes a new runtime baseline." + - "将现有 Kimi-K3 B300 DSXE 路径不可用的 vLLM 镜像 5894fdf 更换为 3696c77,建立新的运行时基线。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3040 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Mount and export the persistent writable model root for Kimi-K3 DSpark weights alongside the pre-staged target on B300 DSXE." + - "在 B300 DSXE 上为 Kimi-K3 DSpark 权重挂载并导出持久化可写模型目录,与预加载目标模型一同使用。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3040 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Serialize shared Kimi-K3 DSpark downloads and validate or resume the cached draft before each cell starts serving." + - "串行执行共享 Kimi-K3 DSpark 下载,在每个任务启动服务前校验或续传草稿缓存。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3040 + +- config-keys: + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-h100-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Require validated GPU power for the existing Qwen3.5 FP8 H100 STP and MTP 8k1k sweeps. Preserve runtime, topology, concurrency and eval selection." + - "为现有 Qwen3.5 FP8 H100 STP 和 MTP 8k1k sweep 强制校验 GPU 功耗,保留运行时、拓扑、并发和评估选择。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3083 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Switch --moe-backend from aiter_triton_mxfp4_bf16 to aiter, which selects the CK a8w4 experts (mfma_moe1_silu_mul_afp8_wfp4_bf16 / mfma_moe2_afp8_wfp4_bf16) instead of the W4A16 _moe_gemm_a16w4 Triton kernel, matching the DSV4-Pro MI355X recipe. Decode traces at 131k context show MoE GEMM falling from 2.72 to 2.30 ms/step at conc 1, 4.91 to 4.27 at conc 4 and 9.06 to 7.18 at conc 16, for whole-step gains of 4.2%, 5.3% and 8.7%. AgentX conc 1 improves mean ITL 4.01 to 3.91 ms and mean TTFT 681 to 639 ms; gsm8k is unchanged at 0.9719 strict-match." + - "将 --moe-backend 从 aiter_triton_mxfp4_bf16 改为 aiter,从而选择 CK a8w4 专家内核(mfma_moe1_silu_mul_afp8_wfp4_bf16 / mfma_moe2_afp8_wfp4_bf16),取代 W4A16 的 _moe_gemm_a16w4 Triton 内核,与 DSV4-Pro MI355X 配方保持一致。131k 上下文的解码 trace 显示 MoE GEMM 在并发 1 时从 2.72 降至 2.30 ms/step,并发 4 从 4.91 降至 4.27,并发 16 从 9.06 降至 7.18,整步分别加速 4.2%、5.3% 和 8.7%。AgentX 并发 1 的平均 ITL 从 4.01 降至 3.91 ms,平均 TTFT 从 681 降至 639 ms;gsm8k strict-match 保持 0.9719 不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3058 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Pin the recipe to vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657, which includes the AITER mHC delayed-pre path from vllm-project/vllm#56503; the exact image passed the concurrency 1-32 and eval-only concurrency 32 cells in run 34710937012 and is recorded by merged vllm-project/recipes#968. Set AITER_TRITON_LOG_LEVEL=ERROR to suppress the repeated gfx950 Gluon-fallback warning that accounted for 98% of the validation server log; restore WARNING when diagnosing new AITER Triton failures." + - "将配方固定到 vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657,其中包含 vllm-project/vllm#56503 引入的 AITER mHC delayed-pre 路径;该精确镜像已在运行 34710937012 中通过并发 1–32 及仅评测并发 32,并由已合并的 vllm-project/recipes#968 记录。设置 AITER_TRITON_LOG_LEVEL=ERROR 以抑制占验证服务日志 98% 的 gfx950 Gluon 回退重复警告;诊断新的 AITER Triton 故障时应恢复 WARNING。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3058 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Replace the local 2*CONC max_num_seqs limit with an explicit reproducibility cap of 128 (the effective vllm serve default on MI355X is 1024). The old cap sat below AgentX subagent fan-out and produced queue times up to 16.2 seconds at concurrency 1; 128 removed that scheduling queue. The derived CUDA-graph capture ceiling is 1024 for each configured concurrency." + - "将本地 2*CONC 的 max_num_seqs 上限改为显式固定的可复现值 128(MI355X 上 vllm serve 的实际默认值为 1024)。旧上限低于 AgentX 子代理扇出,并发 1 时排队时间最高达 16.2 秒;128 消除了该调度排队。所有已配置并发点由此得到的 CUDA graph 捕获上限均为 1024。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3058 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911." + - "Move the MI355X DeepSeek-V4-Pro AgentX SGLang serving path from MTP to DSpark block size 6 and switch model to the DeepSeek-V4-Pro-0813 checkpoint, whose bundled dspark_* head." + - "DP-attention serving adds --enable-dp-lm-head, which SGLang requires for DSpark under DP attention, and TP-only serving adds --prefill-decode-interval 10 to fix scheduler issue." + - "Use --cuda-graph-max-bs-decode instead of --cuda-graph-max-bs: the 20260911 image split the flag per phase and ships no legacy alias, so the old spelling aborts argument parsing as an ambiguous prefix." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3001 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the MI355X MiniMax-M3 AgentX image from vllm/vllm-openai-rocm:nightly-8a728663c1c3eeace834a95f5654fa653cc1998c to the latest immutable nightly, vllm/vllm-openai-rocm:nightly-2a02f6efe319c885e3ccbcecde402e0028f9ec1e" + - "Pick up the optimized FP8 MiniMax-M3 aiter indexer from vLLM PRs #52664" + - "hand-tuned graph capture range; increased max-num-batched-tokens to 65536" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2959 + +- config-keys: + - glm5.1-fp8-b200-tilert + scenario-type: + - fixed-seq-len + description: + - "Preserve explicit registries during native TileRT Enroot imports and select prepared shared weights and squash images for both C1 recipes." + - "原生 TileRT 导入镜像时保留显式仓库地址,并让两个 C1 配方使用已准备的共享权重与 squash 镜像。" + - "Allow 90 minutes for the 8k1k recipe and its full GSM8K evaluation." + - "为 8k1k 配方及其完整 GSM8K 评测预留 90 分钟。" + - "Avoid recursive TileRT eval dispatch and preserve evaluation failures after artifact staging." + - "避免 TileRT 评测分发无限递归,并在保存产物后保留评测失败状态。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3067 + +- config-keys: + - dsv41flash-fp4-gb200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for GB200 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 GB200 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3097 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Serve amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 instead of amd/Qwen3.5-397B-A17B-MXFP4 on the MI355X SGLang fixed-seq-len arms." + - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2793 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913." + - "Rename --cuda-graph-max-bs to --cuda-graph-max-bs-decode for sglang 0.5.19." + - "Prefill allreduce INT4; scheduler-recv-interval default 60." + - "Serve amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 instead of amd/Qwen3.5-397B-A17B-MXFP4." + - "Trim AgentX search space to TP4 none [1,4,8,12,16], TP2 none [1,4,8,12,16,20], TP2 HiCache [20,24,28,32,36,40]; drop the TP4 HiCache arm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3069 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the vLLM ROCm nightly to eed1f3d0c6043bd494424a22443ee198dd56f657, which includes vllm-project/vllm#54855, and disable incompatible AITER fused shared experts for the mixed MXFP4/FP8 checkpoint." + - "Extend the single-node DPA AgentX sweep from concurrency 64 to 64/128/192. Manual validation found c128 delivered the best throughput among c64/c128/c256; c192 fills the gap to locate the throughput knee before the c256 regression." + - "将 vLLM ROCm nightly 更新至包含 vllm-project/vllm#54855 的 eed1f3d0c6043bd494424a22443ee198dd56f657,并针对 MXFP4/FP8 混合权重关闭不兼容的 AITER shared-expert fusion。" + - "将单节点 DPA AgentX 扫描从并发 64 扩展为 64/128/192。手工验证显示,在 c64/c128/c256 三个点中 c128 吞吐量最高;新增 c192 用于补齐中间区间并定位 c256 性能回退前的吞吐拐点。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3071 + +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache + scenario-type: + - agentic-coding + description: + - "Add a 1P1D GLM-5.2 FP8 H200 AgentX configuration with six-step EAGLE MTP." + - "Enable prefill-only hierarchical KV cache offload to host DRAM with the layer_first memory layout; keep decode KV cache GPU-resident." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3021 + +- config-keys: + - qwen3.8next-fp4-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add a single-node Qwen3.8-Flash-Next NVFP4 AgentX configuration on B200 with SGLang native NEXTN MTP at TP1 and concurrency 1, 4, 8, and 16." + - "Follow the merged SGLang cookbook command with flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM state, and checkpoint-derived quantization." + - "Use the repository's committed thinking-on MTP=3 synthetic acceptance length of 2.32, while evaluation keeps real target-model verification." + - "Size maximum running requests and CUDA graph capture to twice concurrency; use a 0.90 static memory fraction and a 160-slot Mamba cache at concurrency 16." + - "Enable six tokenizer workers for the TP1 AgentX recipe and require a complete staged NVFP4 checkpoint before launch." + - "Route the qwen3.8next fp4 prefix in the B200 Nscale launcher to the staged Qwen3.8-Flash-Next NVFP4 checkpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751 + +- config-keys: + - glm5.2-fp4-b200-dynamo-sglang-agentic-agg + - glm5.2-fp4-b200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add a GLM-5.2 NVFP4 B200 Dynamo-SGLang AgentX curve on the NScale runner." + - "Cover Pareto aggregate TP8 points at concurrency 1, 4, and 8, disaggregated 1P(DEP8)4D(TP4) at concurrency 48, and disaggregated 1P(DEP8)1D(DEP8) at concurrency 64." + - "Use lmsysorg/sglang:nightly-dev-20260910-00840301 with its bundled Dynamo 1.5.0.dev20260909, EAGLE speculative decoding, and 110 GB HiCache write-back DRAM offload per prefill rank." + - "Allow up to 3600 seconds for NIXL UCX engine initialization in the 1P(DEP8)4D(TP4) configuration while registering its larger set of HiCache pools." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2995 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913. Enable the v2 fused top-k path for GLM-5.x on ROCm after sgl-project/sglang#36684 and #36851." + - "将 SGLang ROCm 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 更新到 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913;在 sgl-project/sglang#36684 和 #36851 合入后,为 ROCm 上的 GLM-5.x 启用 v2 fused top-k 路径。" + - "Remove concurrency 1 and 2 from the TP4/EP4 HiCache arm so those low-concurrency points run only on TP8/EP1 with KV offloading disabled; keep TP4/EP4 HiCache DRAM offloading for concurrency 4, 8, 10, and 12." + - "从 TP4/EP4 HiCache arm 中移除并发 1 和 2,使这些低并发点仅使用关闭 KV offloading 的 TP8/EP1;TP4/EP4 的并发 4、8、10 和 12 继续使用 HiCache DRAM offloading。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3081 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Replace the B300 DeepSeek-V4-Pro AgentX native EAGLE/MTP serving path with DSpark block size 6 on the DeepSeek-V4-Pro-0813 checkpoint." + - "Use one speculative step, seven draft tokens, and thinking-on golden synthetic acceptance length 3.77 for throughput; eval retains real DSpark verification." + - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP attention, with additional MegaMoE workspace headroom at concurrency 32, 64, and 128." + - "Route the workload through the B300 DSXE launcher, read weights from /data/models, and mount a writable Hugging Face/Xet cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2918 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Update the Kimi-K3 GB300 AgentX vLLM image from vllm/vllm-openai:nightly-dev-arm64-cu13-3696c77 to vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56." + - "Update Mooncake KV-offload backend metadata from version 0.3.12.post1 to 0.3.13.post1 and set MC_MAX_MR_SIZE=4294967296 in the worker environment." + - "Set --mamba-ssm-cache-dtype=bfloat16 and enable --enable-flashinfer-autotune in all six recipes, replacing --no-enable-flashinfer-autotune in the aggregate recipes." + - "Set --cp-kv-cache-interleave-size=1 on both prefill and decode workers in all four disaggregated recipes." + - "Remove VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 from both aggregate recipes." + - "Replace the no-spec aggregate recipe and config key with a DSpark-4 arm under the existing DSpark aggregate key, using Inferact/Kimi-K3-DSpark, block rejection for real acceptance, golden synthetic AL 3.36 for throughput, and concurrencies [48,52,56]. Preserve the DSpark-7 aggregate arm at [1,4] with AL 3.84." + - "Trim DSpark-4 disaggregated concurrencies to 1P1D [48,56], 1P2D [48,64], and 1P3D [48]. Preserve the DSpark-7 1P3D arm at concurrency 1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3105 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Expand B300 c1 graph capture from 8 to 64 tokens for DSpark5 subagent fan-out; measure P90/P99 latency with the unchanged full AgentX trace and AL 3.51" + - "将 B300 c1 的图捕获从 8 扩展至 64 tokens 以覆盖 DSpark5 subagent 并发;保持完整 AgentX 轨迹和 AL 3.51,测量 P90/P99 延迟" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3095 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for B300 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 B300 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3095 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for GB300 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 GB300 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3098 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Expand B300 c1 graph capture from 8 to 64 tokens for DSpark5 subagent fan-out; measure P90/P99 latency with the unchanged full AgentX trace and AL 3.51" + - "将 B300 c1 的图捕获从 8 扩展至 64 tokens 以覆盖 DSpark5 subagent 并发;保持完整 AgentX 轨迹和 AL 3.51,测量 P90/P99 延迟" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3116 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for B300 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 B300 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3116 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for B200 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 B200 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3096 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for B200 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 B200 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3096 + +- config-keys: + - dsv41flash-fp4-h200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Set a 64-token CUDA graph capture floor for H200 DSpark at c1/c2/c4; preserve c8+ sizing, full AgentX traces, AL 3.51, and Engram UVA" + - "将 H200 DSpark 的 c1/c2/c4 CUDA graph 最小捕获范围设为 64 tokens;保持 c8+ 大小、完整 AgentX 轨迹、AL 3.51 和 Engram UVA" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3099 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 MXFP4 ATOM AgentX points on MI355X: TP4 concurrency 1–32 and TP2 concurrency 1, 2, 5 with EAGLE3-GQA MTP." + - "Add TP4 concurrency 40 and 48 with LMCache DRAM offload and SLRU caching; speculative decoding is disabled at these points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2816 + +- config-keys: + - minimaxm3-fp8-h200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use FLASH_ATTN for the target model, select the Humming MXFP8 dense-linear backend, and enable Marlin atomic reduction for the MXFP8 MoE path." + - "Use 0.95 GPU memory utilization for resident serving while retaining 0.90 for Mooncake DRAM offload, and remove the Mooncake concurrency-14 point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3108 + +- config-keys: + - glm5.2-fp8-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the first GLM-5.2 FP8 AgentX (agentic-coding) recipe on MI355X: zai-org/GLM-5.2-FP8 on single-node SGLang with EAGLE off the checkpoint's built-in nextn head (num-steps 5, eagle-topk 1, 6 draft tokens = 5 speculative tokens, the same depth as the MXFP4 MI355X sibling), spec-decoding=mtp only per the AgentX policy, routed to benchmarks/single_node/agentic/glm5.2_fp8_mi355x_sglang_mtp.sh." + - "Port of the validated glm5.2_fp4_mi355x_sglang_mtp.sh. Serve flags are unchanged: --kv-cache-dtype fp8_e4m3, tilelang DSA prefill/decode backends, SGLANG_OPT_USE_TOPK_V2=false (the top-k v2 JIT cannot build for gfx950), glm47 tool-call and glm45 reasoning parsers, chunked-prefill 32768, --mem-fraction-static 0.85, max-running-requests 2*CONC (cap 256), cuda-graph-max-bs min(2*CONC, 64), HiCache write_through_selective/direct/page_first_direct at ratio 1.5. Quantization is auto-detected from the checkpoint's quantization_config (quant_method=fp8, 128x128 weight blocks), as the ROCm GLM-5.2 FP8 MI325X sibling does; the MXFP4 script's Mooncake L3 branch is not carried." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL 3.61 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, K=5, run 28058352479), which was measured on the FP8 checkpoint itself; EVAL_ONLY runs keep real target verification and double the SWE-bench step budget to 150 like the other GLM-5.2 recipes." + - "Image lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 (2026-09-07 ROCm 7.2 mi35x nightly, digest sha256:1a31d1ff5f6b4afee1866d72c0610445c629fa3b0c056f372f843ef839dbd8be; Docker Hub last pushed 2026-09-07T14:35:05Z), the newest tag in the rocm720-mi35x flavor the other MI355X SGLang recipes use. --dsa-prefill-backend/--dsa-decode-backend tilelang, SGLANG_OPT_USE_TOPK_V2 and SGLANG_SIMULATE_ACC_TOKEN_MODE are all present in v0.5.19." + - "Memory: the ~756 GB block-FP8 checkpoint (141 shards) is ~94.5 GB/rank at TP8 on 288 GB MI355X (MXFP4: ~47 GB/rank), leaving a ~150 GB/rank fp8 KV pool inside the 0.85 static share (MXFP4 TP8: ~182.7 GB/rank). TP8-only on day zero: TP4 would leave ~56 GB/rank for KV, DSA indexer, and EAGLE verification batches, and the MI325X FP8 sibling is TP8-only for the same reason. HiCache ratio 1.5 pins ~1.8 TB across TP8, inside the ~3.0 TB of host DRAM on cluster:mi355x-amds." + - "Search space, steps of at least 2 with a hard stop at 16: TP8/EP1 GPU-resident KV at conc [1, 2, 4, 10] (mirrors the MXFP4 sibling's TP8 arm exactly) and TP8/EP1 HiCache DRAM offload at conc [8, 12, 16]; dram-utilization 0.8." + - "runners/launch_mi355x-amds.sh mounts the NFS hf-hub cache (/it-share/hf-hub-cache) for zai-org/GLM-5.2-FP8, following the MiniMax-M3 precedent, so the 756 GB checkpoint is pulled once for the cluster instead of once per node-local NVMe cache. No script-name routing change was needed: the launcher already resolves __mi355x__mtp.sh." + - "Re-pin from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260908 (2026-09-08 ROCm 7.2 mi35x daily, digest sha256:976bdf01d2e41ed5354741f99afc2cf0164e4f25607a1263e2f5c7519c60fa9f; Docker Hub last pushed 2026-09-08T14:36:46Z). The 2026-09-07 build carries an unguarded kv_index_translator.translate_dcp_read_ids call on the DSA fp8 KV read path that the EAGLE draft backend never binds, so GLM-5.2 MTP cells crash intermittently with AttributeError after minutes to an hour of clean serving (run 34173459478: HiCache c12 after 74 min, c16 and the eval; the four GPU-resident cells and HiCache c8 passed). sgl-project/sglang#38318 (merged 2026-09-07T20:03Z) adds the None guard and is in the 2026-09-08 daily." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2867 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Update vLLM image from vllm/vllm-openai:deepseekv41-flash-0909 to vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 (2026-09-15 upstream nightly, vllm-project/vllm@cd10ed6f); recipe script, DSpark settings, and the TP4 c1–128 grid are unchanged" + - "将 vLLM 镜像从 vllm/vllm-openai:deepseekv41-flash-0909 更新为 vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3(2026-09-15 上游 nightly,vllm-project/vllm@cd10ed6f);配方脚本、DSpark 设置及 TP4 c1–128 网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3136 + +- config-keys: + - dsv41flash-fp4-h100-vllm-agentic-dspark + description: + - "Update vLLM image from vllm/vllm-openai:deepseekv41-flash-0909 to vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 (2026-09-15 upstream nightly, vllm-project/vllm@cd10ed6f); recipe script, DSpark settings, and the TP8 c1–4 grid are unchanged" + - "将 vLLM 镜像从 vllm/vllm-openai:deepseekv41-flash-0909 更新为 vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3(2026-09-15 上游 nightly,vllm-project/vllm@cd10ed6f);配方脚本、DSpark 设置及 TP8 c1–4 网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3135 + +- config-keys: + - dsv41flash-fp4-gb200-vllm-agentic-dspark + description: + - "Update vLLM image from vllm/vllm-openai:deepseekv41-flash-0909 to vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 (2026-09-15 upstream nightly, vllm-project/vllm@cd10ed6f); recipe script, DSpark settings, and the TP4 c1–128 grid are unchanged" + - "将 vLLM 镜像从 vllm/vllm-openai:deepseekv41-flash-0909 更新为 vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3(2026-09-15 上游 nightly,vllm-project/vllm@cd10ed6f);配方脚本、DSpark 设置及 TP4 c1–128 网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3138 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Update vLLM image from vllm/vllm-openai:deepseekv41-flash-0909 to vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 (2026-09-15 upstream nightly, vllm-project/vllm@cd10ed6f); recipe script, DSpark settings, and the TP4 c1–128 grid are unchanged" + - "将 vLLM 镜像从 vllm/vllm-openai:deepseekv41-flash-0909 更新为 vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3(2026-09-15 上游 nightly,vllm-project/vllm@cd10ed6f);配方脚本、DSpark 设置及 TP4 c1–128 网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3139 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update the MI355X disaggregated AgentX recipe to DeepSeek-V4-Pro-0813 on lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911 with the FP4 sparse-attention indexer." + - "Keep the supported EAGLE three-step, top-k-one, four-token verification path and use the committed Pro-0813 thinking-on golden AL of 3.01 for throughput, replacing the original V4 value of 2.49. Evaluation retains real target verification." + - "Use checkpoint-specific Pro-0813 golden AL values for draft lengths one through three and reject unsupported lengths rather than falling through to the original V4 curve." + - "Run the full seven-point sweep: TP4 prefill/decode at concurrency 4, TP8 at 16 without offload, TP8 with HiCache at 32 and 48, and TP8/DP8 with UMBP linker at 128, 192, and 256." + - "Gate DP environment settings by role, scale request limits and graph ranges with concurrency, and extend UMBP readiness and barrier timeouts. Remove commented-out environment assignments." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2823 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Select CUDA graph batch-size flags from the installed SGLang argument parser. Preserve --cuda-graph-bs on older images and use explicit phase flags only when the legacy option is absent." + - "Restore compatibility for existing AMD disaggregated recipes on older SGLang images without changing the Pro-0813 image's graph sizes, disabled prefill graphs, golden AL 3.01, topology, or precision. Fail before launch if neither supported spelling is available." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2823 + +- config-keys: + - dsv4-fp8-h200-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Refresh DeepSeek-V4-Pro-0813 FP8 H200 Dynamo+SGLang AgentX aggregate serving with DSpark K=6." + - "Image: lmsysorg/sglang:nightly-dev-20260914-4358a161" + - "Update simulated acceptance length from 2.49 to 3.77 and replace the removed cuda-graph-max-bs flag with cuda-graph-max-bs-decode." + - "Preserve the eight-hour DSV4 AgentX allocation through the H200 launcher's default time limit." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3100 + +- config-keys: + - dsv41flash-fp4-h200-vllm-agentic-dspark + description: + - "Update vLLM image from vllm/vllm-openai:deepseekv41-flash-0909 to vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 (2026-09-15 upstream nightly, vllm-project/vllm@cd10ed6f); recipe script, DSpark settings, and the TP8 c1–128 grid are unchanged" + - "将 vLLM 镜像从 vllm/vllm-openai:deepseekv41-flash-0909 更新为 vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3(2026-09-15 上游 nightly,vllm-project/vllm@cd10ed6f);配方脚本、DSpark 设置及 TP8 c1–128 网格保持不变" + - "H200 checkpoint loading: on nightly-cd10ed6f the lazy safetensors path read the 475 GiB checkpoint from the VIRTIOFS HF cache at ~170 s/shard and the Rust frontend hit its 3600 s readiness deadline at 19/48 shards (run 35012494184, c1 and the c128 eval), versus ~12.5 s/shard on deepseekv41-flash-0909 (run 34504985992). runners/launch_h200-dgxc-slurm.sh now exports VLLM_SAFETENSORS_LOAD_STRATEGY=prefetch, which the shared script forwards as --safetensors-load-strategy so shards are streamed into page cache by parallel readers, and VLLM_ENGINE_READY_TIMEOUT_S=7200 like the GB300 launcher. Serve flags and the grid are otherwise unchanged; other SKUs do not set the variable." + - "H200 检查点加载:在 nightly-cd10ed6f 上,惰性 safetensors 路径从 VIRTIOFS HF 缓存读取 475 GiB 检查点仅约 170 秒/分片,Rust 前端在 3600 秒就绪期限内只加载了 19/48 分片(运行 35012494184,c1 与 c128 评测),而 deepseekv41-flash-0909 约为 12.5 秒/分片(运行 34504985992)。runners/launch_h200-dgxc-slurm.sh 现导出 VLLM_SAFETENSORS_LOAD_STRATEGY=prefetch(共享脚本转发为 --safetensors-load-strategy,由并行读取器先将分片流式载入页缓存)以及 VLLM_ENGINE_READY_TIMEOUT_S=7200(与 GB300 启动器一致)。其余服务参数与网格不变;其他 SKU 不设置该变量。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3134 + +- config-keys: + - glm5.2-fp8-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the first GLM-5.2 FP8 AgentX (agentic-coding) recipe on B300: zai-org/GLM-5.2-FP8 on single-node SGLang with EAGLE off the checkpoint's built-in nextn head (num-steps 3, eagle-topk 1, 4 draft tokens = 3 speculative tokens), spec-decoding=mtp only per the AgentX policy, routed to benchmarks/single_node/agentic/glm5.2_fp8_b300_sglang_mtp.sh on cluster:b300-dsxe." + - "Port of the FP8 B200 sibling glm5.2_fp8_b200_sglang_mtp.sh (#2863) with the B300 deltas the NVFP4 B300 sibling carries: --mem-fraction-static 0.85 on the 288 GB part and a fixed 270 GB/rank HiCache target pool at every concurrency (#2651) in place of the B200 recipe's ratio mode at c1-c8. Serve flags are otherwise identical: TP8 low-latency arm with --kv-cache-dtype fp8_e4m3, --bf16-gemm-backend cutedsl, --max-prefill-tokens 8192, chunked-prefill 8192, --quantization fp8 (the checkpoint ships quant_method=fp8 with 128x128 weight blocks), glm47 tool-call and glm45 reasoning parsers, cuda-graph-max-bs min(2*CONC, 64), HiCache write_back/direct/page_first_direct." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, K=3, run 28058352479), which was measured on the FP8 checkpoint itself; EVAL_ONLY runs keep real target verification and double the SWE-bench step budget to 150 like the other GLM-5.2 recipes." + - "Memory: the ~756 GB block-FP8 checkpoint (141 shards) is ~94.5 GB/GPU at TP8 on 288 GB B300s (NVFP4: ~58 GB/GPU), leaving an fp8 KV pool of roughly 150 GB/GPU inside the 0.85 static share against ~187 GB/GPU on NVFP4 B300 and ~55 GB/GPU on FP8 B200. TP8-only for memory and comparability. HiCache 270 GB/rank target plus the ~38.73 GB/rank DSA indexer host pool uses about 2.47 TB across TP8, inside the ~3.18 TB (80% of 3,977,095 MiB) cluster:b300-dsxe exposes." + - "Search space mirrors glm5.2-fp4-b300-sglang-agentic-mtp exactly so the two precision curves on B300 are comparable: one TP8 + HiCache arm at conc [1, 4, 8, 12, 16], dram-utilization 0.80." + - "Image lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b (2026-09-08 cu13 dev nightly, digest sha256:9a352a35c973a2357372e85f3bcb5388b6b3c46c1329165987260f3b089647dc, sgl-project/sglang@20ca564b), the first nightly carrying sgl-project/sglang#38318, the None guard for the EAGLE DSA fp8 KV read path that crashed the 2026-09-07 build intermittently; same tag as the FP8 B200 sibling in #2863. SGLANG_SIMULATE_ACC_TOKEN_MODE requires v0.5.16 or newer." + - "zai-org/GLM-5.2-FP8 is now pre-staged on cluster:b300-dsxe like the other DSXE checkpoints, so runners/launch_b300-dsxe.sh adds GLM-5.2-FP8 to STAGED_MODELS (plus glm-5.2-fp8 / zai-org/GLM-5.2-FP8 aliases) and exports MODEL_PATH=/scratch/models/GLM-5.2-FP8; the script no longer downloads the checkpoint and instead verifies the staged tree is complete (tokenizer, shard index, every shard) and fails fast if it is not. Supersedes #2884, whose only sweep died on an enroot squash-import I/O error before serving." + - "新增 B300 上首个 GLM-5.2 FP8 AgentX(agentic-coding)配方:zai-org/GLM-5.2-FP8,单节点 SGLang,EAGLE 使用检查点自带 nextn 头(3 步 / topk 1 / 4 草稿 token),吞吐按黄金 AL 2.99 固定接受率,评测保留真实验证;TP8 + 270 GB/rank HiCache,并发 [1, 4, 8, 12, 16],mem-fraction-static 0.85。检查点已在 cluster:b300-dsxe 预先暂存,启动器将 GLM-5.2-FP8 加入 STAGED_MODELS,脚本不再下载,仅校验暂存目录完整。取代 #2884。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3146 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-mtp + - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp + description: + - "Update SGLang image for the resident arm from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 and the HiCache arm from lmsysorg/sglang:v0.5.19-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 (2026-09-14 upstream cu13 nightly, sgl-project/sglang@4358a161). The 2026-09-15 nightly (nightly-dev-cu13-20260915-8874c51a) cannot start any HiCache arm: sgl-project/sglang#35233 (merged 2026-09-14T22:49Z) made the host pool import get_device_accessible_ptr from a kernel wheel that image does not ship, so every kv-offloading dram point died at startup (runs 35016956476, 35016965515); sgl-project/sglang#39516 fixes it for the 2026-09-16 build, and 09-14 is the newest nightly that predates the regression. NEXTN MTP settings, HiCache arms, and the grid are unchanged" + - "将 SGLang 镜像(驻留分支自 lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00,HiCache 分支自 lmsysorg/sglang:v0.5.19-cu130)更新为 lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161(2026-09-14 上游 cu13 nightly,sgl-project/sglang@4358a161)。2026-09-15 nightly(nightly-dev-cu13-20260915-8874c51a)的所有 HiCache 分支无法启动:sgl-project/sglang#35233(2026-09-14T22:49Z 合入)令主机池从镜像未附带的内核 wheel 导入 get_device_accessible_ptr,所有 kv-offloading dram 点在启动时失败(运行 35016956476、35016965515);sgl-project/sglang#39516 已修复并将随 2026-09-16 构建发布,09-14 是早于该回归的最新 nightly。NEXTN MTP 设置、HiCache 分支及网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3149 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915." + - "Switch HiCache defaults to --hicache-io-backend kernel and --hicache-mem-layout page_first (from direct / page_first_direct). Ratio 1.5 and write_through are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3118 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0911, tracking recipes/Agentic-Kimi-K3.md at ROCm/ATOM#2131 as retuned by ROCm/ATOM#2208." + - "Publish the concurrency set [1, 4, 14, 16, 48, 56, 72], replacing [1, 2, 4, 8, 12, 14, 16, 32, 40, 56, 64]." + - "Set gpu-memory-utilization to 0.90 and max-num-batched-tokens to 8192 on every point." + - "Set max-num-seqs to 32 through concurrency 16 and to 2 x concurrency above it, giving 96, 112, and 144 at concurrency 48, 56, and 72." + - "Set AITER_REUSE_IDENTICAL_COMM_GROUPS=1 at concurrency 56 and 72, and 0 from concurrency 1 through 48." + - "Set ATOM_GDN_SSM_DTYPE=fp16, which recipes/Agentic-Kimi-K3.md at ROCm/ATOM#2131 carries and this recipe had never ported. The ATOM default fp32 doubles the SSM half of the per-sequence state-checkpoint image, from 29039616 to 56171520 bytes at zero draft tokens, and that reservation comes out of the KV pool: up to 2209 fewer KV blocks, 7.8 percent, at concurrency 72." + - "Start the CUDA-graph capture ladder at batch size 1 instead of 2. ATOM rounds a decode batch up to the smallest captured size, so a single-sequence step previously replayed the batch-size-2 graph and forwarded twice the rows it needed. Batch size 1 is 92 percent of decode steps at concurrency 1 and 55 percent at concurrency 4, and under 2 percent from concurrency 14 up." + - "Exclude the DSpark draft model from the ptpc_fp8 online quantization. ptpc_fp8 is a global online-quant target and ATOM runs the online-quant streamer for speculative draft loads as well, so it re-quantized the BF16 Inferact/Kimi-K3-DSpark checkpoint: the five draft decoder layers' self_attn fused_qkv_a_proj, q_b_proj, kv_b_proj and o_proj, their mlp gate_up_proj and down_proj, and the 35840x7168 context_proj aux-fusion linear, which alone is larger than one draft decoder layer. The draft now serves at its native BF16. The draft tree carries no root prefix, so patterns anchored on 'layers.' reach it without touching the target's 'model.layers.' modules." + - "Carry the two paged-KV dram-utilization blocks onto the new concurrency set: 0.343 lands 128 GB per rank for concurrency 14 through 48, and 0.513 lands 192 GB per rank for concurrency 56 and 72." + - "Draft depth and synthetic acceptance (7 draft tokens at AL 3.84 for concurrency 1 and 4, 3 at AL 3.00 for concurrency 14 and 16, no draft from 48 up), ReplaySSM placement, dcp-size 8 from concurrency 14 up, the FP8 KV cache, the 128-token page, --state-checkpoint-interval-tokens -1 with anchor-only state checkpointing, ptpc_fp8 online quantization, LMCACHE_CHUNK_SIZE 1024, the NUMA pinning, the draft pre-staging retry, --served-model-name $MODEL, and the AgentX client are unchanged from #2852." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3207 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Retune the GLM-5.2 FP4 MI355X ATOM AgentX recipe to the validated standalone recipe (ROCm/ATOM PR 2247). Image -> rocm/atom-dev:nightly_202609151450 (ATOM e03b7f17 / AITER 797cce25). Per-concurrency MTP draft depth with the golden acceptance length via --spec-decode-acceptance-length: K5/3.61 for C1-C8, K4/3.33 for C10-C40, K3/2.99 for C48. Add a TP4+DCP4 arm (dcp-size 4) at conc [16, 24, 32, 40, 48] to shard the decode KV. online-quant is global ptpc_fp8 with a 7-module exclude that keeps every MoE expert (layers 0-77, native MXFP4) and the whole MTP head (layer 78, native BF16) out of online quant, so only the dense attention/MLP Linears are re-quantized to ptpc_fp8. Add --gpu-memory-utilization 0.95, --enable_prefix_caching, ATOM_USE_FLYDSL_GATHER_KV_B_PROJ=0, and LMCACHE_NUMA_MODE=auto. dram-utilization 0.14 -> 0.171 yields 256 GiB/rank LMCache (min(3095781,2861022)MiB*0.171*4/8). TP8 GPU-only arm unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3205 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260914." + - "Raise --prefill-decode-interval from 10 to 20 on both the TP-only and DP-attention serving paths, so the scheduler runs more decode steps between prefill admissions." + - "Switch the sglang-router policy for the DP-attention arms from consistent_hashing to cache_aware, and set the cache-aware absolute balance threshold to 32, down from the router default of 64, at concurrency above 160." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3120 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Update vLLM image from vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 to vllm/vllm-openai:nightly-af1c01499b289be555c475669ba50a88e96d846e (2026-09-16 upstream nightly, vllm-project/vllm@af1c0149, manifest digest sha256:f3b756ede21083a9415baac77b38998b89c738d2a9c1441c22de2d49cacf5282); the 59 upstream commits touch the DeepSeek-V4.1 path (Mega-mHC via the vLLM DeepGEMM fork, DSpark collapse before SP all-gather, KV-only context insertion, renderer and XGrammar tool-schema fixes). v0.29.0 predates the deepseek_v41 package this recipe needs. Recipe script, DSpark settings and the TP4 c1-128 grid are unchanged" + - "将 vLLM 镜像从 vllm/vllm-openai:nightly-cd10ed6f9f6b37a8ace9cf380007e66fe12ec0c3 更新为 vllm/vllm-openai:nightly-af1c01499b289be555c475669ba50a88e96d846e(2026-09-16 上游 nightly,vllm-project/vllm@af1c0149,manifest digest sha256:f3b756ede21083a9415baac77b38998b89c738d2a9c1441c22de2d49cacf5282);上游 59 个提交涉及 DeepSeek-V4.1 路径(基于 vLLM DeepGEMM 分支的 Mega-mHC、DSpark 在 SP all-gather 前折叠、仅 KV 的上下文插入、渲染器与 XGrammar 工具 schema 修复)。v0.29.0 早于该配方所需的 deepseek_v41 包。配方脚本、DSpark 设置及 TP4 c1-128 网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3176 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update B200 DeepSeek-V4-Pro AgentX to lmsysorg/sglang:v0.5.19-cu130; recipe settings unchanged." + - "将 B200 DeepSeek-V4-Pro AgentX 镜像更新为 lmsysorg/sglang:v0.5.19-cu130;配方参数不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3012 + +- config-keys: + - dsr1-fp4-b200-sglang + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.16-cu130 (build commit sgl-project/sglang@fdebc938f7f4d16fe6b9f55dcd9a767cf0899ea1, CUDA 13.0.1, FlashInfer 0.6.14, sgl-kernel 0.4.5) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, CUDA 13.0.3, FlashInfer 0.6.18, sgl-kernel 0.4.6.post1)." + - "benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200.sh is unchanged: nvidia/DeepSeek-R1-0528-FP4-V2 with modelopt_fp4, trtllm_mla attention, flashinfer_trtllm MoE, fp8_e4m3 KV cache, TP4/EP1 concurrency 1-32 and TP4/EP4 DP-attention concurrency 64-256 on the 8k1k workload." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2989 + +- config-keys: + - qwen3.5-fp8-h100-sglang-agentic-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.16-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 (2026-09-14 upstream cu13 nightly, sgl-project/sglang@4358a161). The 2026-09-15 nightly (nightly-dev-cu13-20260915-8874c51a) cannot start any HiCache arm: sgl-project/sglang#35233 (merged 2026-09-14T22:49Z) made the host pool import get_device_accessible_ptr from a kernel wheel that image does not ship, so every kv-offloading dram point died at startup (runs 35016956476, 35016965515); sgl-project/sglang#39516 fixes it for the 2026-09-16 build, and 09-14 is the newest nightly that predates the regression. NEXTN MTP settings, HiCache arms, and the grid are unchanged" + - "将 SGLang 镜像从 lmsysorg/sglang:v0.5.16-cu130更新为 lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161(2026-09-14 上游 cu13 nightly,sgl-project/sglang@4358a161)。2026-09-15 nightly(nightly-dev-cu13-20260915-8874c51a)的所有 HiCache 分支无法启动:sgl-project/sglang#35233(2026-09-14T22:49Z 合入)令主机池从镜像未附带的内核 wheel 导入 get_device_accessible_ptr,所有 kv-offloading dram 点在启动时失败(运行 35016956476、35016965515);sgl-project/sglang#39516 已修复并将随 2026-09-16 构建发布,09-14 是早于该回归的最新 nightly。NEXTN MTP 设置、HiCache 分支及网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3150 + +- config-keys: + - qwen3.5-fp8-b200-sglang-agentic-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.16-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 (2026-09-14 upstream cu13 nightly, sgl-project/sglang@4358a161); rename --cuda-graph-max-bs to --cuda-graph-max-bs-decode, which exists on both this and the 09-15 nightly, because recent nightlies split the flag into decode/prefill variants and reject the old prefix as ambiguous (B200 sibling, run 35013444957). The 2026-09-15 nightly (nightly-dev-cu13-20260915-8874c51a) cannot start any HiCache arm: sgl-project/sglang#35233 (merged 2026-09-14T22:49Z) made the host pool import get_device_accessible_ptr from a kernel wheel that image does not ship, so every kv-offloading dram point died at startup (runs 35016956476, 35016965515); sgl-project/sglang#39516 fixes it for the 2026-09-16 build, and 09-14 is the newest nightly that predates the regression. NEXTN MTP settings, HiCache arms, and the grid are unchanged" + - "将 SGLang 镜像从 lmsysorg/sglang:v0.5.16-cu130更新为 lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161(2026-09-14 上游 cu13 nightly,sgl-project/sglang@4358a161);将 --cuda-graph-max-bs 改为 --cuda-graph-max-bs-decode(09-14 与 09-15 nightly 均支持),因近期 nightly 将该参数拆分为 decode/prefill 两个变体并拒绝旧写法(B200 同级配方运行 35013444957)。2026-09-15 nightly(nightly-dev-cu13-20260915-8874c51a)的所有 HiCache 分支无法启动:sgl-project/sglang#35233(2026-09-14T22:49Z 合入)令主机池从镜像未附带的内核 wheel 导入 get_device_accessible_ptr,所有 kv-offloading dram 点在启动时失败(运行 35016956476、35016965515);sgl-project/sglang#39516 已修复并将随 2026-09-16 构建发布,09-14 是早于该回归的最新 nightly。NEXTN MTP 设置、HiCache 分支及网格保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3142 + +- config-keys: + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.14-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260915-8874c51a (2026-09-15 upstream cu13 nightly, sgl-project/sglang@8874c51a); raise --max-prefill-tokens and --chunked-prefill-size from 16384 to 32768 and add --mamba-full-memory-ratio 0.37 and --linear-attn-prefill-backend flashinfer; the TP8/TP4 EAGLE grid is unchanged" + - "将 SGLang 镜像从 lmsysorg/sglang:v0.5.14-cu130 更新为 lmsysorg/sglang:nightly-dev-cu13-20260915-8874c51a(2026-09-15 上游 cu13 nightly,sgl-project/sglang@8874c51a);将 --max-prefill-tokens 与 --chunked-prefill-size 从 16384 提高到 32768,并新增 --mamba-full-memory-ratio 0.37 和 --linear-attn-prefill-backend flashinfer;TP8/TP4 EAGLE 网格保持不变" + - "Rename --cuda-graph-max-bs to --cuda-graph-max-bs-decode: the 2026-09-15 SGLang nightly splits the flag into decode/prefill variants and rejects the old spelling as ambiguous (run 35013444957, canary c4)" + - "将 --cuda-graph-max-bs 改为 --cuda-graph-max-bs-decode:2026-09-15 SGLang nightly 将该参数拆分为 decode/prefill 两个变体,旧写法因歧义被拒绝(运行 35013444957,canary c4)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3144 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Use a B300-specific vLLM AgentX script with explicit FULL_AND_PIECEWISE CUDA graph capture sizes, concurrency-tiered batched-token limits, and an explicit maximum sequence count." + - "Add a TP2 search-space variant at concurrency 2-128 while retaining the TP4 concurrency 1-128 variant." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3109 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + description: + - "Switch the DSpark draft attention backend from TRITON_MLA to ROCM_AITER_MLA; measured +28.9% interactivity p90 and +14.5% per-GPU at conc 1 with k/AL held fixed" + - "Use the published Inferact/Kimi-K3-DSpark draft unmodified: vLLM #55966 landed upstream, so ROCM_AITER_MLA now accepts the stock checkpoint's dflash_config: null and the earlier local causal rewrite is dropped. The recipe carries no model patch and no vLLM patch" + - "Raise the conc-1 speculative envelope from k=6/AL 3.75 to k=7/AL 3.84, matching the NV B300 cell (+3.6% interactivity p90)" + - "Descend the speculative envelope with concurrency instead of a flat k=3 across conc 4-14: conc 4 k=5/AL 3.62, conc 8 and 10 k=4/AL 3.36, conc 12 and 14 k=3/AL 3.00. Each k is paired with its own acceptance length. Conc 4 measured 111.23 interactivity p90 (ITL p90 8.99 ms), 162.61 output tok/s, 2801.95 tok/s per GPU" + - "Scope the CUDA graph pairing by concurrency: conc 1 and 4 run FULL_AND_PIECEWISE with VLLM_USE_BREAKABLE_CUDAGRAPH=1, the pairing both measurements below were taken under. Every other cell keeps FULL with breakable off. Breakable costs KV pool in proportion to capture size 2*CONC*(1+k) -- 11.5% at conc 4 (capture 48) and a third at conc 48 (capture 96, deadlocks at 86% usage) -- so unmeasured cells default to the cheaper setting" + - "Move conc 4 to the kv-offloading: none arm, matching how it is measured and how CI runs it (kimik3_tp8_conc4_kvnone)" + - "Replace the LMCache 0.5.5.dev114+rocm7.2 KV-offload backend with vllm-simple (SimpleCPUOffloadConnector) on every offload arm; conc 48 measured 11,083 tok/s per GPU vs 10,947 for the LMCache baseline, with no external cache server" + - "Bump the image to vllm/vllm-openai-rocm:nightly-rocm100-af1c0149 (vLLM 0.29.1rc1.dev187), the first nightly carrying vLLM #55966. Conc 1 on it measured 146.41 interactivity p90 (ITL p90 6.83 ms), flat against the previous 2671fed reference's 146.14" + - "Add conc 70 to the dcp-size 8 arm" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3124 + +- config-keys: + - glm5.2-fp8-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the first GLM-5.2 FP8 AgentX (agentic-coding) recipe on B200: zai-org/GLM-5.2-FP8 on single-node SGLang with EAGLE off the checkpoint's built-in nextn head (num-steps 3, eagle-topk 1, 4 draft tokens = 3 speculative tokens), spec-decoding=mtp only per the AgentX policy, routed to benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh." + - "Port of the validated glm5.2_fp4_b200_sglang_mtp.sh with two functional deltas: HF_MODEL_ID zai-org/GLM-5.2-FP8 and --quantization fp8 (the checkpoint ships quant_method=fp8 with 128x128 weight blocks and dynamic e4m3 activations) in place of modelopt_fp4. Serve flags are otherwise identical: TP8 low-latency arm with --kv-cache-dtype fp8_e4m3, --bf16-gemm-backend cutedsl, --max-prefill-tokens 8192, chunked-prefill 8192, --mem-fraction-static 0.83, glm47 tool-call and glm45 reasoning parsers, cuda-graph-max-bs min(2*CONC, 64), HiCache write_back/direct/page_first_direct with ratio 0.75 at c1-c8 and a 169 GB/rank target pool at c12/c16." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, K=3, run 28058352479), which was measured on the FP8 checkpoint itself; EVAL_ONLY runs keep real target verification." + - "Image lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, digest sha256:19b8fa1223cc339c1eae7a5b703f1a8c2543b5b119155bf3d7efaef18f77f007, tag commit sgl-project/sglang@30705c00; Docker Hub last pushed 2026-09-07T01:43:42Z). The same tag the B200 Qwen3.5 FP8/FP4 SGLang AgentX recipes moved to in #2861/#2862; SGLANG_SIMULATE_ACC_TOKEN_MODE requires v0.5.16 or newer." + - "Memory: the ~756 GB block-FP8 checkpoint (141 shards) is ~94.5 GB/GPU at TP8 on 180 GB B200s (NVFP4: ~58 GB/GPU), leaving an fp8 KV pool of roughly 55 GB/GPU inside the 0.83 static share against ~91 GB/GPU on NVFP4. TP8-only for memory and comparability; HiCache absorbs the smaller HBM pool as host spill. MEM_FRACTION_STATIC and HICACHE_* stay env-overridable for on-node tuning." + - "Search space mirrors glm5.2-fp4-b200-sglang-agentic-mtp exactly so the two precision curves are comparable: one TP8 + HiCache arm at conc [1, 4, 8, 12, 16], dram-utilization 0.80." + - "Adds glm5.2/fp8 routing to runners/launch_b200-nscale-compat.sh (MODEL_PATH default /scratch/models/GLM-5.2-FP8, SRT_SLURM_MODEL_PREFIX glm5.2-fp8), which previously hard-failed with 'Unsupported model prefix/precision' for this model. The script's completeness-checked, flock-serialized hf download stages the checkpoint on the day-zero run if it is not already on the cluster." + - "Re-pin from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 to lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b (2026-09-08 cu13 dev nightly, digest sha256:9a352a35c973a2357372e85f3bcb5388b6b3c46c1329165987260f3b089647dc; Docker Hub last pushed 2026-09-08T01:40:59Z, tag commit sgl-project/sglang@20ca564b). The 2026-09-07 build carries an unguarded kv_index_translator.translate_dcp_read_ids call on the DSA fp8 KV read path that the EAGLE draft backend never binds, so GLM-5.2 MTP runs crash intermittently with AttributeError (observed on the MI355X FP8 sibling in run 34173459478 after 74 minutes of serving). sgl-project/sglang#38318 (merged 2026-09-07T20:03Z) adds the None guard and is six commits behind 20ca564b." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2863 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the ATOM image from rocm/atom-dev:nightly_202608280858 to rocm/atom-dev:nightly_202609161445, the first ROCm/ATOM line carrying native RCCL DEP and the DSpark K6 kernels." + - "Switch both throughput bands from EAGLE MTP (spec-decoding: mtp) to DSpark (spec-decoding: draft_model) on the DeepSeek-V4-Pro-0813 checkpoint, pinned to revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 and verified before GPU startup by check_dsv4_dspark_checkpoint.py (66 shards plus the DSpark Markov/confidence heads)." + - "Raise NUM_SPEC_TOKENS from 3 to 6 (DSpark K6: six draft tokens plus one target token, a q7 verify window) and the forced acceptance length SPEC_DECODE_AL from 2.49 to 3.77, the committed golden value from golden_al_distribution/dsv4-pro-0813-dspark.yaml (thinking_on, K6). Both throughput bands pin this AL; EVAL_ONLY runs verify real DSpark drafts." + - "Move the concurrency 48/64/96/128/256 band from DPA with TBO and the default MoE transport to native RCCL DEP (TP8/DPA8/EP8: --enable-dp-attention --all2all-backend rccl --dp-load-balance least_tokens --moe-backend standard, TBO disabled), while the concurrency 1/2/4/8/16 band stays TP-only (TP8/EP1). Session affinity is preserved via the AIPerf correlation-id session id so each conversation pins to one DP rank, and STATE_CHECKPOINT_INTERVAL_TOKENS stays 8192." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2912 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark + scenario-type: + - agentic-coding + description: + - "Switch all four arms from EAGLE (spec-decoding: mtp) to DSpark (spec-decoding: draft_model) at DECODE_MTP_SIZE=3, and bump the image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911 to -20260913, the first tag carrying the DSpark optimizations. CLIENT_IMAGE stays at 20260907 (load generator only). Topology, concurrency list, HiCache/UMBP settings, and precision are unchanged." + - "DeepSeek-V4-Pro-AgentX in models.yaml gains dspark_flags (--speculative-algorithm DSPARK --speculative-eagle-topk 1) alongside mtp_flags rather than replacing it, so recipes remaining on spec-decoding: mtp keep their EAGLE arm unchanged; dp_flags gains --enable-dp-lm-head, required by SGLang for DSpark under DP attention and inert for EAGLE. The -0813 checkpoint bundles the draft head, so no separate draft checkpoint is staged." + - "server_sglang.sh builds DSpark flags as --speculative-dspark-block-size DECODE_MTP_SIZE --speculative-num-steps 1 --speculative-num-draft-tokens (DECODE_MTP_SIZE + 1). DECODE_MTP_SIZE is the draft length for both algorithms but EAGLE spends it as sequential draft passes while DSpark emits one block, so num-steps is pinned to 1. The verify window is identical for both, so the MORI decode dispatch scaling by (DECODE_MTP_SIZE + 1) is unchanged. A model set to draft_model without dspark_flags now fails hard instead of silently serving EAGLE." + - "Golden AL is unchanged: the AgentX curve is keyed on checkpoint, thinking mode, and draft length, not on the algorithm, so DeepSeek-V4-Pro-0813 at draft length 3 keeps AL 3.01 from golden_al_distribution/dsv4-pro-0813-dspark.yaml (thinking_on)." + - "将四条臂全部从 EAGLE(spec-decoding: mtp)切换为 DSpark(spec-decoding: draft_model),DECODE_MTP_SIZE=3;镜像由 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911 升级到 -20260913(首个带 DSpark 优化的标签),CLIENT_IMAGE 保持 20260907(仅负载生成器)。拓扑、并发列表、HiCache/UMBP 设置与精度均不变。models.yaml 中 DeepSeek-V4-Pro-AgentX 新增 dspark_flags(与 mtp_flags 并列而非替换,因此仍用 mtp 的配方 EAGLE 分支不受影响),dp_flags 增加 --enable-dp-lm-head。server_sglang.sh 按 SPEC_DECODING == draft_model 分流:DSpark 一次前向出一整块,故 num-steps 钉死为 1,草稿长度转为 --speculative-dspark-block-size;验证窗口(长度+1)与 EAGLE 相同,所以 MORI 解码 dispatch 的 ×(DECODE_MTP_SIZE+1) 缩放无需改动。黄金 AL 不变:按检查点而非算法选表,DeepSeek-V4-Pro-0813 在草稿长度 3 仍为 AL 3.01。" + - "Rename the config key from dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp to dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark to reflect the DSpark algorithm and the UMBP-linker arm; earlier entries for this recipe are recorded under the old key. No other field changes with the rename." + - "配置键由 dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp 改名为 dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark,以反映 DSpark 算法与 UMBP-linker 分支;该配方此前的条目仍记录在旧键名下。改名不伴随任何其他字段变化。" + - "Collect completed logs from every Slurm node so both prefill and decode artifacts are uploaded; forward workflow-owned AgentX fast-mode and power inputs, and supply the existing router port to evaluation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3188 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-mtp + description: + - "Pin the Qwen3.5 FP8 H200 SGLang AgentX image to lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, replacing the 2026-09-14 nightly; recipe settings and the c4/8/12/16 grid are unchanged." + - "将 Qwen3.5 FP8 H200 SGLang AgentX 镜像从 2026-09-14 nightly 更新为按摘要固定的 lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9;配方设置及 c4/8/12/16 网格保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3198 + +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Add aggregate TP8 GLM-5.2 FP8 H200 AgentX recipes with EAGLE MTP and HiCache at concurrencies 2, 4, 8, 12, and 16." + - "Use one recipe per concurrency and cap max-running-requests and CUDA graph batch size at twice the trajectory concurrency." + - "Set mem-fraction-static to 0.75 to leave additional GPU workspace headroom for MTP." + - "Use SGLang nightly-dev-20260914-4358a161 with Dynamo 1.5.0.dev20260913 and retry truncated srt-slurm setup dependency downloads." + - "Update telemetry to the current srt-slurm schema, preserving the one-second collection interval and using a schema-compliant 12-second collector join timeout." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3112 + +- config-keys: + - dsv41flash-fp4-mi325x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add the MI325X (gfx942) vLLM AgentX arm for DeepSeek-V4.1-Flash as a copy of the validated MI355X recipe on vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657: TP8 rather than TP4 so a 256 GB card holds its share of the 511 GB checkpoint, the GPU-resident Engram tables and a 1M-context KV pool; native five-token DSpark with thinking-on golden AL 3.51 for throughput and real block verification (adaptive verification off on ROCm) for evals; VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, VLLM_USE_BREAKABLE_CUDAGRAPH=1, --moe-backend aiter, --max-num-batched-tokens 16384, --max-num-seqs 128, concurrency 1-32" + - "runners/launch_mi325x-amds.sh now mounts the checkout at /ix for this checkpoint and rewrites RESULT_DIR so AgentX runtime directories stay out of /workspace" + - "新增 DeepSeek-V4.1-Flash 在 MI325X(gfx942)上的 vLLM AgentX 配方,复制自已验证的 MI355X 配方并使用同一镜像 vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657:采用 TP8 而非 TP4,使 256 GB 显卡可容纳 511 GB 检查点的分片、常驻 GPU 的 Engram 表以及 1M 上下文 KV 池;原生五 token DSpark,吞吐使用 thinking 开启的黄金 AL 3.51,eval 保留真实块验证(ROCm 上关闭自适应验证);VLLM_ROCM_USE_AITER=1、VLLM_ROCM_USE_AITER_MOE=1、VLLM_USE_BREAKABLE_CUDAGRAPH=1、--moe-backend aiter、--max-num-batched-tokens 16384、--max-num-seqs 128,并发 1-32" + - "runners/launch_mi325x-amds.sh 现为该检查点将仓库挂载到 /ix 并重写 RESULT_DIR,使 AgentX 运行目录不落在 /workspace 下" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3247 + +- config-keys: + - dsv41flash-fp4-mi325x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Drop --moe-backend aiter and let vLLM select the MXFP4 MoE kernel: on gfx942 the explicit aiter name resolved to the Triton W4A16 kernel (AITER_TRITON_MXFP4_BF16) and every worker segfaulted during breakable CUDA-graph capture at concurrency 16 (run 35305045778); auto selection prefers the CK W4A16 kernel (AITER_MXFP4_BF16) that the retired DeepSeek-V4-Pro gfx942 arm ran on" + - "runners/launch_mi325x-amds.sh drains the allocated node and retries once on a fresh allocation that excludes it when pyxis cannot start the container (enroot-nsenter user-namespace denial, as on the MI300X fleet in run 35305037528)" + - "去掉 --moe-backend aiter,改由 vLLM 自动选择 MXFP4 MoE 内核:在 gfx942 上显式指定 aiter 会解析为 Triton W4A16 内核(AITER_TRITON_MXFP4_BF16),并发 16 时所有 worker 在 breakable CUDA graph 捕获期间段错误(运行 35305045778);自动选择优先使用已退役的 DeepSeek-V4-Pro gfx942 配方所用的 CK W4A16 内核(AITER_MXFP4_BF16)" + - "当 pyxis 无法启动容器时(enroot-nsenter 用户命名空间被拒,如运行 35305037528 中的 MI300X 节点),runners/launch_mi325x-amds.sh 会 drain 所分配的节点,并在排除该节点的新分配上重试一次" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3247 + +- config-keys: + - dsv41flash-fp4-mi325x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Capture only full decode CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=0 with cudagraph_mode FULL_DECODE_ONLY, as the MiniMax-M3 gfx942 arm does) and restore --moe-backend aiter: on gfx942 every worker segfaulted during piecewise graph capture with both the Triton W4A16 MoE kernel (run 35305045778) and the auto-selected unfused Triton kernel (run 35306398350), so the capture mode rather than the MoE kernel is the failing piece; prefill runs eagerly" + - "仅捕获完整的 decode CUDA graph(VLLM_USE_BREAKABLE_CUDAGRAPH=0 并设置 cudagraph_mode FULL_DECODE_ONLY,与 MiniMax-M3 gfx942 配方一致)并恢复 --moe-backend aiter:在 gfx942 上,无论使用 Triton W4A16 MoE 内核(运行 35305045778)还是自动选择的未融合 Triton 内核(运行 35306398350),所有 worker 都在 piecewise graph 捕获期间段错误,说明问题在于捕获模式而非 MoE 内核;prefill 以 eager 方式运行" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3247 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Disable SGLANG_OPT_USE_TOPK_V2 for the precompiled HIP Top-K path and update the SGLang ROCm 7.2.0 image from 20260913 to 20260916 for sgl-project/sglang#39155 and #39631; versus published run 34783501425, final run 35159075984 improves c8 P90 interactivity from 84.53 to 86.81 tok/s/user (+2.7%) and throughput per GPU from 10,389.25 to 12,695.57 tok/s (+22.2%)." + - "关闭 SGLANG_OPT_USE_TOPK_V2 以使用预编译 HIP Top-K 路径,并将 SGLang ROCm 7.2.0 镜像从 20260913 更新到 20260916,以包含 sgl-project/sglang#39155 和 #39631;相比已发布运行 34783501425,最终运行 35159075984 的并发 8 P90 interactivity 从 84.53 提升到 86.81 tok/s/user(+2.7%),单卡吞吐从 10,389.25 提升到 12,695.57 tok/s(+22.2%)。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3117 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang + description: + - "Update the Qwen3.5 FP8 GB300 Dynamo-SGLang 8k1k STP SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 to lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 (2026-09-18 upstream cu13 dev nightly, sgl-project/sglang@20518d85, amd64 + arm64)" + - "In the 3 srt-slurm recipes of this key, set model.container to the same image, rename cuda-graph-max-bs to cuda-graph-max-bs-decode for the same reason, and move the Dynamo install from source commit 46520ca59afe992fb5ef61b3197b2316f8df9b2b to the public 1.5.0.dev20260917 wheel (the master router version follows), the newest Dynamo nightly, since the June source build predates the SGLang ServerArgs compatibility fixes that Dynamo 1.5.0.dev20260910 and later carry; recipe topologies, concurrencies and serving flags are otherwise unchanged" + - "将 Qwen3.5 FP8 GB300 Dynamo-SGLang 8k1k STP 的 SGLang 镜像从 lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 更新为 lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85(2026-09-18 上游 cu13 dev nightly,sgl-project/sglang@20518d85,amd64 + arm64)" + - "在该 key 的 3 个 srt-slurm 配方中,将 model.container 设为同一镜像,出于同样原因将 cuda-graph-max-bs 改为 cuda-graph-max-bs-decode,并将 Dynamo 安装从源码提交 46520ca59afe992fb5ef61b3197b2316f8df9b2b 改为公开的 1.5.0.dev20260917 wheel(master 中的 router 版本随之更新),这是最新的 Dynamo nightly,因为 6 月的源码构建早于 Dynamo 1.5.0.dev20260910 及之后版本所包含的 SGLang ServerArgs 兼容性修复;配方拓扑、并发与服务参数保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3254 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang + description: + - "Rename mamba-scheduler-strategy to mamba-radix-cache-strategy (value no_buffer unchanged) in the 2 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 rejects the old spelling as an unrecognized argument (run 35317753866, every prefill and decode worker exited at argument parsing), and its help lists --mamba-radix-cache-strategy {auto,no_buffer,extra_buffer,extra_buffer_lazy} as the replacement" + - "在该 key 中设置了该参数的 2 个 srt-slurm 配方里,将 mamba-scheduler-strategy 改名为 mamba-radix-cache-strategy(取值 no_buffer 不变):SGLang nightly-dev-cu13-20260918-20518d85 将旧写法视为未识别参数(运行 35317753866 中所有 prefill 与 decode worker 在参数解析阶段退出),其帮助信息列出 --mamba-radix-cache-strategy {auto,no_buffer,extra_buffer,extra_buffer_lazy} 为替代" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3254 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang + description: + - "Replace the decode-role prefill-round-robin-balance: true with load-balance-method: follow_bootstrap_room in the 2 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 no longer has --prefill-round-robin-balance (run 35319100186, every decode worker of the DEP16 arm exited at argument parsing) and its --load-balance-method choices are auto, round_robin, follow_bootstrap_room, total_requests and total_tokens; the prefill roles keep load-balance-method: round_robin" + - "在该 key 中设置了该参数的 2 个 srt-slurm 配方里,将 decode 角色的 prefill-round-robin-balance: true 替换为 load-balance-method: follow_bootstrap_room:SGLang nightly-dev-cu13-20260918-20518d85 已没有 --prefill-round-robin-balance(运行 35319100186 中 DEP16 分支的所有 decode worker 在参数解析阶段退出),其 --load-balance-method 可选值为 auto、round_robin、follow_bootstrap_room、total_requests 与 total_tokens;prefill 角色保持 load-balance-method: round_robin" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3254 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add the H200 SGLang AgentX arm for DeepSeek-V4.1-Flash from the SGLang cookbook: preview image lmsysorg/sglang:dev-dsv41 (digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860), native five-token DSpark (--speculative-algorithm DSPARK --speculative-dspark-block-size 5), thinking on with reasoning effort high, mem-fraction-static 0.8, GPU-resident KV cache, concurrency 1-128" + - "TP8/EP8 from the cookbook's verified H200 cell, which pins --attention-backend dsv4 and --moe-runner-backend flashinfer_mxfp4. Throughput uses the thinking-on golden synthetic AL 3.51 (SGLANG_SIMULATE_ACC_LEN, match-expected, real-draft-token); accuracy evals keep real DSpark verification." + - "新增 DeepSeek-V4.1-Flash 在 H200 上的 SGLang AgentX 配方,遵循 SGLang cookbook:预览镜像 lmsysorg/sglang:dev-dsv41(digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860),原生五 token DSpark(--speculative-algorithm DSPARK --speculative-dspark-block-size 5),thinking 开启、reasoning effort 为 high,mem-fraction-static 0.8,KV cache 常驻 GPU,并发 1-128" + - "采用 cookbook 已验证 H200 配置的 TP8/EP8,并显式指定 --attention-backend dsv4 与 --moe-runner-backend flashinfer_mxfp4。吞吐测试使用 thinking 开启的黄金合成 AL 3.51(SGLANG_SIMULATE_ACC_LEN、match-expected、real-draft-token);准确率 eval 保留真实 DSpark 验证。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3239 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Cut HBM pressure during 1M-context AgentX prefill after the first sweep (run 35304458924: c128 died at startup with torch.OutOfMemoryError (12 GiB allocation, 5 GiB free of 139.8 GiB, 127.6 GiB already held by PyTorch)): move the Engram tables to one shared host copy (SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1, the SGLang analogue of the vLLM arm's Engram CPU offload, output bitwise unchanged), lower --mem-fraction-static from 0.8 to 0.75, bound --chunked-prefill-size at 8192 instead of the default 16384, fix the decode CUDA graph batch at the cookbook's 64, and cap --max-running-requests at 128" + - "首次 sweep 后(run 35304458924: c128 died at startup with torch.OutOfMemoryError (12 GiB allocation, 5 GiB free of 139.8 GiB, 127.6 GiB already held by PyTorch))降低 1M 上下文 AgentX prefill 的 HBM 压力:将 Engram 表移至共享主机副本(SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1,对应 vLLM 配方的 Engram CPU offload,输出逐位一致),--mem-fraction-static 从 0.8 降至 0.75,--chunked-prefill-size 限制为 8192(默认 16384),decode CUDA graph batch 固定为 cookbook 的 64,--max-running-requests 上限设为 128" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3239 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Cap --max-running-requests at the 64-request decode graph batch and lower --mem-fraction-static from 0.75 to 0.70, after run 35306704553: the eval-only arm OOMed in the eager DSpark verify path once 128 GSM8K requests were running (6.4 GiB allocation with 2 GiB free of 139.8 GiB), while the throughput points at 64 running requests or fewer stayed up; a DSpark verify batch above the captured tier runs eagerly and allocates its attention workspace per step, so the pool now stays inside the graph tier" + - "将 --max-running-requests 上限设为 64(即 decode graph batch),并将 --mem-fraction-static 从 0.75 降至 0.70;依据:运行 35306704553:eval-only 配方在 128 个 GSM8K 请求同时运行时于 eager DSpark verify 路径 OOM(139.8 GiB 中仅剩 2 GiB 时申请 6.4 GiB),而运行请求不超过 64 的吞吐点保持正常。超出已捕获层级的 DSpark verify batch 会以 eager 方式运行并逐步分配 attention 工作区,因此请求池现保持在 graph 层级之内" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3239 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Halve --chunked-prefill-size from 8192 to 4096, matching the B200 and GB200 arms: in run 35308550355 c1 passed but c32 OOMed in the sparse indexer's prefill top-k (_low_ratio_index_topk_torch, 5 GiB allocation with 3.4 GiB free of 139.8 GiB) with 29 long AgentX prompts in flight at mem-fraction 0.70" + - "将 --chunked-prefill-size 从 8192 减半至 4096,与 B200、GB200 配方一致:运行 35308550355 中 c1 通过,但 c32 在 29 个长 AgentX 提示同时在途、mem-fraction 0.70 时于稀疏 indexer 的 prefill top-k(_low_ratio_index_topk_torch)OOM(139.8 GiB 中仅剩 3.4 GiB 时申请 5 GiB)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3239 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Honor the workflow's SALLOC_TIME_LIMIT (480 minutes) in the H200 DGX Cloud launcher instead of its hardcoded 180-minute salloc, as the Blackwell launchers already do, and trim the conc-list from [1, 2, 4, 8, 16, 32, 64, 128] to [1, 2, 4, 8, 16]: in run 35312220802 c8 passed in 2h52m but c64 hit the 3-hour Slurm limit with 182 of its 707 warmup requests returned; the server log shows the arm prefill-bound (7689 chunks of 4096 tokens at a median 4.2k input tok/s, 63 running and 93 queued requests, KV pool of 34M tokens far from full), so AgentX's roughly 11 warmup sessions per unit of concurrency need about 10 hours at c64 and 5 hours at c32 before the 60-minute profile, while c16 fits the 8-hour budget; the vLLM H200 arm keeps its c64 and c128 points" + - "H200 DGX Cloud 启动器改为使用工作流的 SALLOC_TIME_LIMIT(480 分钟),而非硬编码的 180 分钟 salloc,与 Blackwell 启动器一致,并将 conc-list 从 [1, 2, 4, 8, 16, 32, 64, 128] 缩减为 [1, 2, 4, 8, 16]:运行 35312220802 中 c8 用 2 小时 52 分通过,但 c64 在 707 个 warmup 请求仅返回 182 个时触及 3 小时 Slurm 上限;服务日志显示该配方受 prefill 限制(7689 个 4096 token 分块,输入吞吐中位数约 4.2k tok/s,63 个运行中、93 个排队请求,34M token 的 KV 池远未占满),因此 AgentX 每单位并发约 11 个 warmup 会话在 c64 需约 10 小时、c32 需约 5 小时才能进入 60 分钟 profile,而 c16 可在 8 小时预算内完成;vLLM H200 配方保留 c64 与 c128 点" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3239 + +- config-keys: + - dsv41flash-fp4-h100-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add the H100 SGLang AgentX arm for DeepSeek-V4.1-Flash from the SGLang cookbook: preview image lmsysorg/sglang:dev-dsv41 (digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860), native five-token DSpark (--speculative-algorithm DSPARK --speculative-dspark-block-size 5), thinking on with reasoning effort high, mem-fraction-static 0.8, GPU-resident KV cache, concurrency 1-8" + - "TP8/EP8 with the Hopper backends from the cookbook's verified H200 cell; H100 is not in the cookbook's hardware table, so the script moves the Engram tables to one shared host copy (SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1) and caps --chunked-prefill-size at 4096 to fit 80 GB. Throughput uses the thinking-on golden synthetic AL 3.51 (SGLANG_SIMULATE_ACC_LEN, match-expected, real-draft-token); accuracy evals keep real DSpark verification." + - "新增 DeepSeek-V4.1-Flash 在 H100 上的 SGLang AgentX 配方,遵循 SGLang cookbook:预览镜像 lmsysorg/sglang:dev-dsv41(digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860),原生五 token DSpark(--speculative-algorithm DSPARK --speculative-dspark-block-size 5),thinking 开启、reasoning effort 为 high,mem-fraction-static 0.8,KV cache 常驻 GPU,并发 1-8" + - "采用 cookbook 已验证 H200 配置的 TP8/EP8 与 Hopper 后端;H100 不在 cookbook 硬件表中,脚本将 Engram 表移至共享主机副本(SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1)并将 --chunked-prefill-size 上限设为 4096 以适配 80 GB。吞吐测试使用 thinking 开启的黄金合成 AL 3.51(SGLANG_SIMULATE_ACC_LEN、match-expected、real-draft-token);准确率 eval 保留真实 DSpark 验证。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3240 + +- config-keys: + - dsv41flash-fp4-h100-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Lower --mem-fraction-static from 0.8 to 0.7 and --chunked-prefill-size from 4096 to 2048 after run 35304509605: c1-c4 served, but c8 OOMed in eager extend (1 GiB allocations with ~600 MB free of 80 GB) once five requests were live with a 570k-token prompt pending; the change leaves 24 GB of headroom instead of 16 GB and halves the per-chunk hyper-connection and indexer working set" + - "在运行 35304509605 后将 --mem-fraction-static 从 0.8 降至 0.7,--chunked-prefill-size 从 4096 降至 2048:c1-c4 可服务,但 c8 在五个请求同时在线、且有 570k token 提示待处理时于 eager extend 中 OOM(80 GB 中仅剩约 600 MB 时申请 1 GiB);调整后余量从 16 GB 增至 24 GB,并将每分块的 hyper-connection 与 indexer 工作集减半" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3240 + +- config-keys: + - dsv41flash-fp4-h100-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Raise --chunked-prefill-size back from 2048 to 4096 while keeping --mem-fraction-static 0.7: in run 35307250127 c1 and c2 passed but c4 failed AIPerf's 95% latency-coverage check (TTFT 87.6%, inter-token latency 88.4% over the 3600 s window) because prefill of the 100k-600k-token prompts ran near 1,000 tok/s at the smaller chunk; 0.7 leaves 24 GB of headroom for the doubled per-chunk working set, against the 16 GB that OOMed c8 at 0.8/4096" + - "将 --chunked-prefill-size 从 2048 恢复为 4096,--mem-fraction-static 保持 0.7:运行 35307250127 中 c1、c2 通过,但 c4 未达到 AIPerf 95% 的延迟覆盖率要求(3600 s 窗口内 TTFT 87.6%、token 间延迟 88.4%),原因是较小分块下 100k-600k token 提示的 prefill 仅约 1,000 tok/s;0.7 为翻倍的每分块工作集保留 24 GB 余量,而此前 0.8/4096 下 c8 仅有 16 GB 并发生 OOM" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3240 + +- config-keys: + - dsv41flash-fp4-h100-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Honor the workflow's SALLOC_TIME_LIMIT (480 minutes) in the H100 DGX Cloud launcher instead of its hardcoded 180-minute salloc, as the B200, B300, GB200 and GB300 launchers already do: in run 35317886085 c8 finished its 87 warmup requests after 1h55m (the server log shows prefill-bound serving, median 2.8k input tok/s with 4096-token chunks and 210 tok/s decode at 6 running requests) and was 4 minutes from completing the 60-minute profiling phase when the Slurm step hit the 3-hour limit; the hpc-gpu-1 partition has no MaxTime" + - "H100 DGX Cloud 启动器改为使用工作流的 SALLOC_TIME_LIMIT(480 分钟),而非硬编码的 180 分钟 salloc,与 B200、B300、GB200、GB300 启动器一致:运行 35317886085 中 c8 在 1 小时 55 分后完成 87 个 warmup 请求(服务日志显示受 prefill 限制:4096 token 分块下输入吞吐中位数约 2.8k tok/s,6 个并发请求时解码 210 tok/s),距完成 60 分钟 profiling 阶段仅差 4 分钟时 Slurm step 触及 3 小时上限;hpc-gpu-1 分区无 MaxTime 限制" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3240 + +- config-keys: + - dsv41flash-fp4-b200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add the B200 SGLang AgentX arm for DeepSeek-V4.1-Flash from the SGLang cookbook: preview image lmsysorg/sglang:dev-dsv41 (digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860), native five-token DSpark (--speculative-algorithm DSPARK --speculative-dspark-block-size 5), thinking on with reasoning effort high, mem-fraction-static 0.8, GPU-resident KV cache, concurrency 1-128" + - "TP4/EP4 from the cookbook's verified Blackwell low-latency cell with automatically resolved backends. Throughput uses the thinking-on golden synthetic AL 3.51 (SGLANG_SIMULATE_ACC_LEN, match-expected, real-draft-token); accuracy evals keep real DSpark verification. The launcher runners/launch_b200-nscale-compat.sh now routes dsv41flash for framework sglang as well as vllm." + - "新增 DeepSeek-V4.1-Flash 在 B200 上的 SGLang AgentX 配方,遵循 SGLang cookbook:预览镜像 lmsysorg/sglang:dev-dsv41(digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860),原生五 token DSpark(--speculative-algorithm DSPARK --speculative-dspark-block-size 5),thinking 开启、reasoning effort 为 high,mem-fraction-static 0.8,KV cache 常驻 GPU,并发 1-128" + - "采用 cookbook 已验证 Blackwell 低延迟配置的 TP4/EP4,后端自动解析。吞吐测试使用 thinking 开启的黄金合成 AL 3.51(SGLANG_SIMULATE_ACC_LEN、match-expected、real-draft-token);准确率 eval 保留真实 DSpark 验证。 launcher runners/launch_b200-nscale-compat.sh 现同时为 framework sglang 与 vllm 路由 dsv41flash。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3241 + +- config-keys: + - dsv41flash-fp4-b200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Cut HBM pressure during 1M-context AgentX prefill after the first sweep (run 35304517453: c4 came up, then the server exited on the first two warmup prompts): move the Engram tables to one shared host copy (SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1, the SGLang analogue of the vLLM arm's Engram CPU offload, output bitwise unchanged), lower --mem-fraction-static from 0.8 to 0.75, bound --chunked-prefill-size at 8192 instead of the default 16384, fix the decode CUDA graph batch at the cookbook's 64, and cap --max-running-requests at 128" + - "首次 sweep 后(run 35304517453: c4 came up, then the server exited on the first two warmup prompts)降低 1M 上下文 AgentX prefill 的 HBM 压力:将 Engram 表移至共享主机副本(SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1,对应 vLLM 配方的 Engram CPU offload,输出逐位一致),--mem-fraction-static 从 0.8 降至 0.75,--chunked-prefill-size 限制为 8192(默认 16384),decode CUDA graph batch 固定为 cookbook 的 64,--max-running-requests 上限设为 128" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3241 + +- config-keys: + - dsv41flash-fp4-b200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Cap --max-running-requests at the 64-request decode graph batch and lower --mem-fraction-static from 0.75 to 0.70 and halve --chunked-prefill-size to 4096 on this TP4 card, which holds 82 GiB of weights per GPU, after run 35306710037: c32 came up and the server exited with 27 warmup prompts in flight and 64 running requests allowed, after c1-c16 style points had been serving; a DSpark verify batch above the captured tier runs eagerly and allocates its attention workspace per step, so the pool now stays inside the graph tier" + - "将 --max-running-requests 上限设为 64(即 decode graph batch),并将 --mem-fraction-static 从 0.75 降至 0.70,并在这块每 GPU 承载 82 GiB 权重的 TP4 显卡上将 --chunked-prefill-size 减半至 4096;依据:运行 35306710037:c32 启动后,在 27 个 warmup 提示同时在途、允许 64 个运行请求时服务退出。超出已捕获层级的 DSpark verify batch 会以 eager 方式运行并逐步分配 attention 工作区,因此请求池现保持在 graph 层级之内" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3241 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add the GB200 SGLang AgentX arm for DeepSeek-V4.1-Flash from the SGLang cookbook: preview image lmsysorg/sglang:dev-dsv41 (digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860), native five-token DSpark (--speculative-algorithm DSPARK --speculative-dspark-block-size 5), thinking on with reasoning effort high, mem-fraction-static 0.8, GPU-resident KV cache, concurrency 1-128" + - "TP4/EP4 from the cookbook's verified Blackwell low-latency cell with automatically resolved backends. Throughput uses the thinking-on golden synthetic AL 3.51 (SGLANG_SIMULATE_ACC_LEN, match-expected, real-draft-token); accuracy evals keep real DSpark verification. The launcher runners/launch_gb200-nv.sh now routes dsv41flash for framework sglang as well as vllm." + - "新增 DeepSeek-V4.1-Flash 在 GB200 上的 SGLang AgentX 配方,遵循 SGLang cookbook:预览镜像 lmsysorg/sglang:dev-dsv41(digest sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860),原生五 token DSpark(--speculative-algorithm DSPARK --speculative-dspark-block-size 5),thinking 开启、reasoning effort 为 high,mem-fraction-static 0.8,KV cache 常驻 GPU,并发 1-128" + - "采用 cookbook 已验证 Blackwell 低延迟配置的 TP4/EP4,后端自动解析。吞吐测试使用 thinking 开启的黄金合成 AL 3.51(SGLANG_SIMULATE_ACC_LEN、match-expected、real-draft-token);准确率 eval 保留真实 DSpark 验证。 launcher runners/launch_gb200-nv.sh 现同时为 framework sglang 与 vllm 路由 dsv41flash。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3243 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Cut HBM pressure during 1M-context AgentX prefill after the first sweep (run 35304536578: c2 came up, then the server exited on the first warmup prompt): move the Engram tables to one shared host copy (SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1, the SGLang analogue of the vLLM arm's Engram CPU offload, output bitwise unchanged), lower --mem-fraction-static from 0.8 to 0.75, bound --chunked-prefill-size at 8192 instead of the default 16384, fix the decode CUDA graph batch at the cookbook's 64, and cap --max-running-requests at 128" + - "首次 sweep 后(run 35304536578: c2 came up, then the server exited on the first warmup prompt)降低 1M 上下文 AgentX prefill 的 HBM 压力:将 Engram 表移至共享主机副本(SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1,对应 vLLM 配方的 Engram CPU offload,输出逐位一致),--mem-fraction-static 从 0.8 降至 0.75,--chunked-prefill-size 限制为 8192(默认 16384),decode CUDA graph batch 固定为 cookbook 的 64,--max-running-requests 上限设为 128" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3243 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Cap --max-running-requests at the 64-request decode graph batch and lower --mem-fraction-static from 0.75 to 0.70 and halve --chunked-prefill-size to 4096 on this TP4 card, which holds 73 GiB of weights per GPU, after run 35306711846: c16 OOMed in the sparse indexer prefill top-k (_low_ratio_index_topk_extend, 3.8-7 GiB allocations with 3 GiB free of 186 GiB) with 13 warmup prompts in flight at chunk 8192 and mem-fraction 0.75; a DSpark verify batch above the captured tier runs eagerly and allocates its attention workspace per step, so the pool now stays inside the graph tier" + - "将 --max-running-requests 上限设为 64(即 decode graph batch),并将 --mem-fraction-static 从 0.75 降至 0.70,并在这块每 GPU 承载 73 GiB 权重的 TP4 显卡上将 --chunked-prefill-size 减半至 4096;依据:运行 35306711846:c16 在 13 个 warmup 提示同时在途、分块 8192、mem-fraction 0.75 时于稀疏 indexer 的 prefill top-k(_low_ratio_index_topk_extend)OOM(186 GiB 中仅剩 3 GiB 时申请 3.8-7 GiB)。超出已捕获层级的 DSpark verify batch 会以 eager 方式运行并逐步分配 attention 工作区,因此请求池现保持在 graph 层级之内" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3243 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add the MI300X (gfx942) vLLM AgentX arm for DeepSeek-V4.1-Flash as a copy of the validated MI355X recipe on vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657: TP8 rather than TP4 so a 192 GB card holds its share of the 511 GB checkpoint, the GPU-resident Engram tables and a 1M-context KV pool; native five-token DSpark with thinking-on golden AL 3.51 for throughput and real block verification (adaptive verification off on ROCm) for evals; VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, VLLM_USE_BREAKABLE_CUDAGRAPH=1, --moe-backend aiter, --max-num-batched-tokens 8192, --max-num-seqs 128, concurrency 1-32" + - "runners/launch_mi300x-amd.sh now mounts the checkout at /ix for this checkpoint and rewrites RESULT_DIR so AgentX runtime directories stay out of /workspace, and allocates 480 minutes instead of 180 for it so a cold checkpoint download plus a one-hour arm fits" + - "新增 DeepSeek-V4.1-Flash 在 MI300X(gfx942)上的 vLLM AgentX 配方,复制自已验证的 MI355X 配方并使用同一镜像 vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657:采用 TP8 而非 TP4,使 192 GB 显卡可容纳 511 GB 检查点的分片、常驻 GPU 的 Engram 表以及 1M 上下文 KV 池;原生五 token DSpark,吞吐使用 thinking 开启的黄金 AL 3.51,eval 保留真实块验证(ROCm 上关闭自适应验证);VLLM_ROCM_USE_AITER=1、VLLM_ROCM_USE_AITER_MOE=1、VLLM_USE_BREAKABLE_CUDAGRAPH=1、--moe-backend aiter、--max-num-batched-tokens 8192、--max-num-seqs 128,并发 1-32" + - "runners/launch_mi300x-amd.sh 现为该检查点将仓库挂载到 /ix 并重写 RESULT_DIR,使 AgentX 运行目录不落在 /workspace 下,并为其分配 480 分钟(原 180 分钟),以容纳冷下载检查点加一小时的基准运行" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3246 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Drop --moe-backend aiter and let vLLM select the MXFP4 MoE kernel: on gfx942 the explicit aiter name resolved to the Triton W4A16 kernel (AITER_TRITON_MXFP4_BF16) and every worker segfaulted during breakable CUDA-graph capture on the MI325X sibling (run 35305045778); auto selection prefers the CK W4A16 kernel (AITER_MXFP4_BF16) that the retired DeepSeek-V4-Pro gfx942 arm ran on" + - "runners/launch_mi300x-amd.sh drains the allocated node and retries once on a fresh allocation that excludes it when pyxis cannot start the container (enroot-nsenter user-namespace denial on smci300x-ccs-aus-e06-40 and e07-22 in run 35305037528)" + - "去掉 --moe-backend aiter,改由 vLLM 自动选择 MXFP4 MoE 内核:在 gfx942 上显式指定 aiter 会解析为 Triton W4A16 内核(AITER_TRITON_MXFP4_BF16),MI325X 同级配方在 breakable CUDA graph 捕获期间所有 worker 段错误(运行 35305045778);自动选择优先使用已退役的 DeepSeek-V4-Pro gfx942 配方所用的 CK W4A16 内核(AITER_MXFP4_BF16)" + - "当 pyxis 无法启动容器时(运行 35305037528 中 smci300x-ccs-aus-e06-40 与 e07-22 的 enroot-nsenter 用户命名空间被拒),runners/launch_mi300x-amd.sh 会 drain 所分配的节点,并在排除该节点的新分配上重试一次" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3246 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Capture only full decode CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=0 with cudagraph_mode FULL_DECODE_ONLY, as the MiniMax-M3 gfx942 arm does) and restore --moe-backend aiter: on gfx942 every worker segfaulted during piecewise graph capture with both the Triton W4A16 MoE kernel (run 35305045778) and the auto-selected unfused Triton kernel (run 35306398350), so the capture mode rather than the MoE kernel is the failing piece; prefill runs eagerly" + - "仅捕获完整的 decode CUDA graph(VLLM_USE_BREAKABLE_CUDAGRAPH=0 并设置 cudagraph_mode FULL_DECODE_ONLY,与 MiniMax-M3 gfx942 配方一致)并恢复 --moe-backend aiter:在 gfx942 上,无论使用 Triton W4A16 MoE 内核(运行 35305045778)还是自动选择的未融合 Triton 内核(运行 35306398350),所有 worker 都在 piecewise graph 捕获期间段错误,说明问题在于捕获模式而非 MoE 内核;prefill 以 eager 方式运行" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3246 + +- config-keys: + - dsv41flash-fp4-h100-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add concurrency 20, 24 and 28 to the H100 TP8 AgentX arm to fill in the saturation curve between the existing c16 point and the range where 1M-token trajectories start to preempt; recipe script, image and DSpark settings are unchanged" + - "为 H100 TP8 AgentX 配方新增并发 20、24、28,以填补现有 c16 点与 1M token 轨迹开始抢占区间之间的饱和曲线;配方脚本、镜像与 DSpark 设置不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3248 + +- config-keys: + - qwen3.5-fp8-b200-sglang + description: + - "Update the Qwen3.5 FP8 B200 SGLang 8k1k STP SGLang image from lmsysorg/sglang:v0.5.19-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 (2026-09-18 upstream cu13 dev nightly, sgl-project/sglang@20518d85, amd64 + arm64)" + - "Rename --cuda-graph-max-bs to --cuda-graph-max-bs-decode: nightlies since 2026-09-15 split the flag into decode/prefill variants and reject the old spelling as ambiguous; the value (CONC) and every other serving flag are unchanged" + - "将 Qwen3.5 FP8 B200 SGLang 8k1k STP 的 SGLang 镜像从 lmsysorg/sglang:v0.5.19-cu130 更新为 lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85(2026-09-18 上游 cu13 dev nightly,sgl-project/sglang@20518d85,amd64 + arm64)" + - "将 --cuda-graph-max-bs 改为 --cuda-graph-max-bs-decode:自 2026-09-15 起的 nightly 将该参数拆分为 decode/prefill 两个变体并拒绝旧写法;取值(CONC)与其余服务参数不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3250 + +- config-keys: + - qwen3.5-fp8-b200-sglang-mtp + description: + - "Update the Qwen3.5 FP8 B200 SGLang 8k1k MTP SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260915-8874c51a to lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 (2026-09-18 upstream cu13 dev nightly, sgl-project/sglang@20518d85, amd64 + arm64)" + - "将 Qwen3.5 FP8 B200 SGLang 8k1k MTP 的 SGLang 镜像从 lmsysorg/sglang:nightly-dev-cu13-20260915-8874c51a 更新为 lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85(2026-09-18 上游 cu13 dev nightly,sgl-project/sglang@20518d85,amd64 + arm64)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3251 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang + description: + - "Update the Qwen3.5 FP8 GB200 Dynamo-SGLang 8k1k STP SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc to lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 (2026-09-18 upstream cu13 dev nightly, sgl-project/sglang@20518d85, amd64 + arm64)" + - "In the 3 srt-slurm recipes of this key, set model.container to the same image, rename cuda-graph-max-bs to cuda-graph-max-bs-decode for the same reason, and move the Dynamo install from source commit 46520ca59afe992fb5ef61b3197b2316f8df9b2b to the public 1.5.0.dev20260917 wheel (the master router version follows), the newest Dynamo nightly, since the June source build predates the SGLang ServerArgs compatibility fixes that Dynamo 1.5.0.dev20260910 and later carry; recipe topologies, concurrencies and serving flags are otherwise unchanged" + - "将 Qwen3.5 FP8 GB200 Dynamo-SGLang 8k1k STP 的 SGLang 镜像从 lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc 更新为 lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85(2026-09-18 上游 cu13 dev nightly,sgl-project/sglang@20518d85,amd64 + arm64)" + - "在该 key 的 3 个 srt-slurm 配方中,将 model.container 设为同一镜像,出于同样原因将 cuda-graph-max-bs 改为 cuda-graph-max-bs-decode,并将 Dynamo 安装从源码提交 46520ca59afe992fb5ef61b3197b2316f8df9b2b 改为公开的 1.5.0.dev20260917 wheel(master 中的 router 版本随之更新),这是最新的 Dynamo nightly,因为 6 月的源码构建早于 Dynamo 1.5.0.dev20260910 及之后版本所包含的 SGLang ServerArgs 兼容性修复;配方拓扑、并发与服务参数保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3252 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang + description: + - "Rename mamba-scheduler-strategy to mamba-radix-cache-strategy (value no_buffer unchanged) in the 2 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 rejects the old spelling as an unrecognized argument (run 35317753866, every prefill and decode worker exited at argument parsing), and its help lists --mamba-radix-cache-strategy {auto,no_buffer,extra_buffer,extra_buffer_lazy} as the replacement" + - "在该 key 中设置了该参数的 2 个 srt-slurm 配方里,将 mamba-scheduler-strategy 改名为 mamba-radix-cache-strategy(取值 no_buffer 不变):SGLang nightly-dev-cu13-20260918-20518d85 将旧写法视为未识别参数(运行 35317753866 中所有 prefill 与 decode worker 在参数解析阶段退出),其帮助信息列出 --mamba-radix-cache-strategy {auto,no_buffer,extra_buffer,extra_buffer_lazy} 为替代" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3252 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang + description: + - "Replace the decode-role prefill-round-robin-balance: true with load-balance-method: follow_bootstrap_room in the 2 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 no longer has --prefill-round-robin-balance (run 35319100186, every decode worker of the DEP16 arm exited at argument parsing) and its --load-balance-method choices are auto, round_robin, follow_bootstrap_room, total_requests and total_tokens; the prefill roles keep load-balance-method: round_robin" + - "在该 key 中设置了该参数的 2 个 srt-slurm 配方里,将 decode 角色的 prefill-round-robin-balance: true 替换为 load-balance-method: follow_bootstrap_room:SGLang nightly-dev-cu13-20260918-20518d85 已没有 --prefill-round-robin-balance(运行 35319100186 中 DEP16 分支的所有 decode worker 在参数解析阶段退出),其 --load-balance-method 可选值为 auto、round_robin、follow_bootstrap_room、total_requests 与 total_tokens;prefill 角色保持 load-balance-method: round_robin" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3252 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + description: + - "Update the Qwen3.5 FP8 GB200 Dynamo-SGLang 8k1k MTP SGLang image from lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 to lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 (2026-09-18 upstream cu13 dev nightly, sgl-project/sglang@20518d85, amd64 + arm64)" + - "In the 7 srt-slurm recipes of this key, set model.container to the same image, rename cuda-graph-max-bs to cuda-graph-max-bs-decode for the same reason, and move the Dynamo install from source commit 46520ca59afe992fb5ef61b3197b2316f8df9b2b to the public 1.5.0.dev20260917 wheel (the master router version follows), the newest Dynamo nightly, since the June source build predates the SGLang ServerArgs compatibility fixes that Dynamo 1.5.0.dev20260910 and later carry; recipe topologies, concurrencies and serving flags are otherwise unchanged" + - "将 Qwen3.5 FP8 GB200 Dynamo-SGLang 8k1k MTP 的 SGLang 镜像从 lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 更新为 lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85(2026-09-18 上游 cu13 dev nightly,sgl-project/sglang@20518d85,amd64 + arm64)" + - "在该 key 的 7 个 srt-slurm 配方中,将 model.container 设为同一镜像,出于同样原因将 cuda-graph-max-bs 改为 cuda-graph-max-bs-decode,并将 Dynamo 安装从源码提交 46520ca59afe992fb5ef61b3197b2316f8df9b2b 改为公开的 1.5.0.dev20260917 wheel(master 中的 router 版本随之更新),这是最新的 Dynamo nightly,因为 6 月的源码构建早于 Dynamo 1.5.0.dev20260910 及之后版本所包含的 SGLang ServerArgs 兼容性修复;配方拓扑、并发与服务参数保持不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3253 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + description: + - "Rename mamba-scheduler-strategy to mamba-radix-cache-strategy (value no_buffer unchanged) in the 7 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 rejects the old spelling as an unrecognized argument (run 35317753866, every prefill and decode worker exited at argument parsing), and its help lists --mamba-radix-cache-strategy {auto,no_buffer,extra_buffer,extra_buffer_lazy} as the replacement" + - "在该 key 中设置了该参数的 7 个 srt-slurm 配方里,将 mamba-scheduler-strategy 改名为 mamba-radix-cache-strategy(取值 no_buffer 不变):SGLang nightly-dev-cu13-20260918-20518d85 将旧写法视为未识别参数(运行 35317753866 中所有 prefill 与 decode worker 在参数解析阶段退出),其帮助信息列出 --mamba-radix-cache-strategy {auto,no_buffer,extra_buffer,extra_buffer_lazy} 为替代" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3253 + +- config-keys: + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + description: + - "Replace the decode-role prefill-round-robin-balance: true with load-balance-method: follow_bootstrap_room in the 5 srt-slurm recipes of this key that set it: SGLang nightly-dev-cu13-20260918-20518d85 no longer has --prefill-round-robin-balance (run 35319100186, every decode worker of the DEP16 arm exited at argument parsing) and its --load-balance-method choices are auto, round_robin, follow_bootstrap_room, total_requests and total_tokens; the prefill roles keep load-balance-method: round_robin" + - "在该 key 中设置了该参数的 5 个 srt-slurm 配方里,将 decode 角色的 prefill-round-robin-balance: true 替换为 load-balance-method: follow_bootstrap_room:SGLang nightly-dev-cu13-20260918-20518d85 已没有 --prefill-round-robin-balance(运行 35319100186 中 DEP16 分支的所有 decode worker 在参数解析阶段退出),其 --load-balance-method 可选值为 auto、round_robin、follow_bootstrap_room、total_requests 与 total_tokens;prefill 角色保持 load-balance-method: round_robin" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3253 + +- config-keys: + - kimik3-fp4-h200-vllm-agentic-simple + scenario-type: + - agentic-coding + description: + - "Enable required measured power for the complete H200 Kimi-K3 simple recipe through the shared recipe-gated launcher." + - "为 H200 Kimi-K3 simple 完整配方启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3054 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Enable required telemetry on the four GB300 disaggregated recipes using the launcher and collector introduced below this draft in the stack. Preserve the original bilingual backfill and validation guidance." + - "为 GB300 Kimi-K3 分离部署启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3047 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB300 disaggregated DCGM exporter to port 19401 to avoid the system listener on 9401. Preserve prefill/decode telemetry grouping; full disaggregated hardware qualification remains pending." + - "GB300 分离部署的 DCGM exporter 改用 19401,避免与 9401 系统服务冲突;保留 prefill/decode 分组,完整硬件验证仍待完成。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3047 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2 + scenario-type: + - agentic-coding + description: + - "Enable required telemetry on seven GB200 aggregate recipes, preserve failed-job artifacts, and reserve CUDA Graph memory in the five TP8/PP2 recipes." + - "为七个 GB200 Kimi-K3 聚合配方启用必需实测功耗并保留失败产物;五个 TP8/PP2 配方为 CUDA Graph 预留显存。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3045 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Restore multinode AgentX power on the pinned upstream srt-slurm: the launcher window adapter now needs only SRT_MEASUREMENT_WINDOW_DIR, which srt-slurm exports to custom benchmark commands, and derives the result root and concurrency locally instead of reading three fork-only variables. Applies to every srt-slurm AgentX recipe with telemetry enabled; recipes, images and grids are unchanged." + - "在固定的上游 srt-slurm 上恢复多节点 AgentX 功耗采集:启动器窗口适配器只需 srt-slurm 导出给自定义基准命令的 SRT_MEASUREMENT_WINDOW_DIR,并在本地推导结果根目录与并发数,不再读取三个仅分支提供的变量。适用于所有启用遥测的 srt-slurm AgentX 配方;配方、镜像与网格保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3206 + +- config-keys: + - kimik3-fp4-h200-vllm-agentic-balanced + scenario-type: + - agentic-coding + description: + - "Enable required measured power for the complete H200 Kimi-K3 balanced recipe through the shared recipe-gated launcher." + - "为 H200 Kimi-K3 balanced 完整配方启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3053 + +- config-keys: + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - Add B200 Qwen3.5-397B-A17B FP8 AgentX disaggregation with colocated TP4 prefill + and TP4 decode on eight GPUs at concurrency 8, 16, 24, 32, 48 and 64. + - Use Dynamo/SGLang nightly 20260918-20518d85 with NEXTN MTP and Mooncake NVLink transfers. + - Enable 104 GB per-rank prefill HiCache at C8-C32; use 215 GB with first-use + write-through and decode static memory 0.88 at C48/C64. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3269 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + - glm5.2-fp8-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the GLM-5.2 MI355X SGLang AgentX images to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917 (2026-09-17 ROCm 7.2.0 mi35x daily, digest sha256:21c1cc9a…, the newest rocm720-mi35x tag; the 2026-09-18 builds only ship rocm724/rocm10 flavors): glm5.2-fp4-mi355x-sglang-agentic-mtp from 20260916 and glm5.2-fp8-mi355x-sglang-agentic-mtp from 20260908" + - "In glm5.2_fp8_mi355x_sglang_mtp.sh pass the decode graph batch as --cuda-graph-max-bs-decode instead of the removed --cuda-graph-max-bs alias, matching the MXFP4 sibling; serve flags, search spaces and golden acceptance are otherwise unchanged" + - "将 GLM-5.2 MI355X SGLang AgentX 镜像更新至 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917(2026-09-17 ROCm 7.2.0 mi35x 日构建,digest sha256:21c1cc9a…,rocm720-mi35x 系列最新 tag;2026-09-18 构建仅提供 rocm724/rocm10 版本):glm5.2-fp4-mi355x-sglang-agentic-mtp 自 20260916,glm5.2-fp8-mi355x-sglang-agentic-mtp 自 20260908" + - "在 glm5.2_fp8_mi355x_sglang_mtp.sh 中改用 --cuda-graph-max-bs-decode 传递 decode graph batch,替换已移除的 --cuda-graph-max-bs 别名,与 MXFP4 姊妹配方一致;其余服务参数、搜索空间与 golden acceptance 不变" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3277 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a TP=2 search space alongside TP=4 for MI355X DSv4.1-Flash AgentX, and extend both to concurrency 128. TP=2 is feasible now that the Engram tables can live in host memory on ROCm rather than costing 47.2 GiB of device memory per rank, which previously forced four GPUs per server." + - "Offload the Engram tables only where the KV pool is short, not everywhere. vllm-project/vllm#57491 widened the two ROCm is_cuda() gates to is_cuda_alike(), so this image resolves an Engram config on gfx950 and the recipe sets the value explicitly instead of relying on the default. TP=2 always offloads: the tables cost 94.4 GiB per rank there against a 288 GiB card. TP=4 keeps them resident through c64, exactly as the validated concurrency 1-32 run measured, and offloads only at c128, where resident would leave 14.83M KV tokens, 116K per concurrent request, below the 122K at which TP=2 c64 collapsed. Offloaded lookups go to pinned host memory over UVA, so that cost is paid only where it buys KV room." + - "Trade prefill chunk size for KV room only above c32: --max-num-batched-tokens drops from the upstream 16384 to 8192 at TP=2 c64 and TP=4 c128, and to 4096 at TP=2 c128. The sparse-attention indexer and its companion per-rank buffers scale at roughly 4.4 MiB per batched token. KV tokens per concurrent request, which is what decides whether the prefix cache holds, rises from 116K to 388K at TP=4 c128, from 122K to 386K at TP=2 c64 and from 23.6K to 260K at TP=2 c128. Concurrency 1-32 keeps the upstream chunk untouched, where TP=2 measured 245K per request and TP=4 463K. B300 runs 8192 at TP=4 and the Blackwell TP=2 arms run 4096 (#3320, #3321)." + - "Cap --max-num-seqs at the graph-capture shape wherever the chunk falls below six times the MI355X API-server default of 1024 sequences, which is TP=2 c128 alone. DSpark verifies 1+5 tokens per sequence, so a decode batch of max_num_seqs needs six times that many token slots; at 4096 against 1024 sequences only four remain and the engram projection faults during profiling with HSA_STATUS_ERROR_EXCEPTION at M=1024, N=129280, K=256, reproduced on two separate GPU pairs. The same probes show why c128 never completed before: at the upstream 16384 with capture at 2048, TP=2 c128 held just 7.72 GiB, 3.02M KV tokens, 23.6K per request." + - "Sweep the full concurrency range 1 through 128 on both TP arms, sixteen points. The changes above leave concurrency 1-32 on the upstream settings, so the recipe there is unchanged, but its only measured points come from run 34710937012 on the superseded nightly-eed1f3d0 pin. Re-measuring them puts the whole MI355X AgentX curve on one image, and TP=2 has no measured point below c64 at all." + - "Move the image from vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657 on the plain ROCm nightly channel to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 on the ROCm 10.0 nightly channel, published 2026-09-21T05:50:03Z with digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1. kimik3-fp4-mi355x-vllm-agentic-mtp already runs that channel on this cluster. This sweep is the qualification for the new image; the previous green run is not evidence for it." + - "Drop the fixed max_num_seqs of 128 and use vLLM's MI355X API-server default of 1024, since a fixed 128 caps in-flight sequences at the outer concurrency once the sweep reaches c128. The CUDA graph ceiling stays at 1024 through c64 and rises to 2048 at c128, covering twice the outer concurrency. Supersedes PR #3111." + - "Pass --no-swa-bounded-replay. vllm-project/vllm#56227 added SWA bounded replay, default on, between the eed1f3d0 pin and this one. The window clamp it relies on landed in the FlashInfer and FlashMLA kernels, while the ROCm sparse SWA path only gained the replay_start kwarg, so on gfx950 the replayed tokens' slots are padded but the prefill kernels never stop at replay_start. Every TP=2 and TP=4 point of run 35567570539 died with HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay start. Prefix caching itself stays on; drop this once ROCm clamps too." + - "为 MI355X DSv4.1-Flash AgentX 在 TP=4 之外新增 TP=2 搜索空间,并将两者的并发扩展到 128。ROCm 上 Engram 表现在可以驻留主机内存,不再占用每 rank 47.2 GiB 设备内存(此前这迫使每个服务器使用四张 GPU),因此 TP=2 可行。" + - "仅在 KV 池吃紧处下放 Engram 表,而非全面下放。vllm-project/vllm#57491 将 ROCm 上的两处 is_cuda() 判断放宽为 is_cuda_alike(),因此该镜像在 gfx950 上会解析 Engram 配置,recipe 显式设置该值而不再依赖默认行为。TP=2 始终下放:此时表每 rank 需 94.4 GiB,而单卡仅 288 GiB。TP=4 在 c64 及以下保持常驻,与已验证的并发 1-32 运行完全一致,仅在 c128 下放:此时若保持常驻只有 14.83M KV token,即每并发请求 116K,低于 TP=2 c64 崩溃时的 122K。下放后的查表经 UVA 访问锁页主机内存,因此只在能换来 KV 空间处才付出该代价。" + - "仅在并发高于 c32 时用 prefill 分块大小换取 KV 空间:--max-num-batched-tokens 从上游的 16384 降至 TP=2 c64 与 TP=4 c128 的 8192,以及 TP=2 c128 的 4096。稀疏注意力 indexer 及其配套的每 rank 缓冲区按每个批量 token 约 4.4 MiB 增长。决定前缀缓存能否维持的每并发请求 KV token 数随之提升:TP=4 c128 从 116K 升至 388K,TP=2 c64 从 122K 升至 386K,TP=2 c128 从 23.6K 升至 260K。并发 1-32 完全保留上游分块大小,实测 TP=2 每请求 245K、TP=4 463K。B300 在 TP=4 使用 8192,Blackwell 的 TP=2 搜索空间使用 4096(#3320、#3321)。" + - "当分块大小低于 MI355X API server 默认 1024 序列所需的六倍时,将 --max-num-seqs 限制为 CUDA graph 捕获的规模,实际只影响 TP=2 c128。DSpark 每个序列验证 1+5 个 token,因此 max_num_seqs 规模的 decode 批次需要六倍的 token 槽位;4096 对 1024 序列仅剩四个槽位,profiling 阶段 engram 投影会在 M=1024、N=129280、K=256 处以 HSA_STATUS_ERROR_EXCEPTION 崩溃,并在两组不同 GPU 上复现。同一批探测也解释了此前 c128 始终无法完成的原因:在上游的 16384 且捕获上限为 2048 时,TP=2 c128 仅有 7.72 GiB、3.02M KV token,即每请求 23.6K。" + - "在两个 TP 分支上完整扫描并发 1 至 128,共十六个点。上述改动在并发 1-32 区间保留上游设置,recipe 本身没有变化,但该区间仅有的实测点来自运行 34710937012,使用的是已被取代的 nightly-eed1f3d0 镜像。重新测量这些点可使整条 MI355X AgentX 曲线落在同一镜像上;此外 TP=2 在 c64 以下完全没有实测点。" + - "镜像从 plain ROCm nightly 渠道的 vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657 切换到 ROCm 10.0 nightly 渠道的 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984,该镜像于 2026-09-21T05:50:03Z 发布,digest 为 sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1。本集群上的 kimik3-fp4-mi355x-vllm-agentic-mtp 已在使用该渠道。本次 sweep 即为新镜像的验证;此前的绿色运行不能作为其证据。" + - "移除固定的 max_num_seqs 128,改用 vLLM 在 MI355X API server 上的默认值 1024:扫描到 c128 时,固定 128 会把在途序列数限制在外层并发上。CUDA graph 上限在 c64 及以下保持 1024,c128 提升至 2048,以覆盖两倍外层并发。本 PR 取代 PR #3111。" + - "传入 --no-swa-bounded-replay。vllm-project/vllm#56227 在 eed1f3d0 与本次固定的镜像之间引入了默认开启的 SWA bounded replay。它所依赖的窗口钳制只落在 FlashInfer 与 FlashMLA 内核中,而 ROCm 稀疏 SWA 路径仅新增了 replay_start 参数,因此在 gfx950 上被重放 token 的 slot 会被填充,但 prefill 内核并不会在 replay_start 处停下。运行 35567570539 的所有 TP=2 与 TP=4 数据点都在首次带 replay start 的前缀命中时以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃。前缀缓存本身保持开启;待 ROCm 同样实现钳制后即可移除。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3326 + +- config-keys: + - dsv41flash-fp4-h200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a TP4 arm to the H200 vLLM DeepSeek-V4.1-Flash AgentX recipe alongside the existing TP8 arm, keeping the Engram tables in pinned host DRAM via --engram-config cpu_offload; ~99 GiB of weights per 141 GB GPU leaves ~15 GiB of KV per rank (~17M tokens), about three times the pool that served c1-c128 on B200 TP2" + - "On sub-TP8 Hopper arms dsv41flash_fp4_vllm_mtp.sh caps --max-num-batched-tokens at 4096 (the indexer's [batched-tokens, 1M] fp8 buffer is 32 GiB at the upstream 16384), bounds --max-num-seqs to twice the concurrency (16-256) and stops CUDA graph capture at 512 tokens; the gate is TP<8 on GPUs under 150 GB, so B200/GB200 TP4 and every TP8 arm are unchanged" + - "为 H200 vLLM DeepSeek-V4.1-Flash AgentX 配方在现有 TP8 臂旁新增 TP4 臂,Engram 表继续通过 --engram-config cpu_offload 放在固定页主机 DRAM;每张 141 GB GPU 约 99 GiB 权重,每 rank 约 15 GiB KV(约 1700 万 token),约为 B200 TP2 跑通 c1-c128 所用 KV 池的三倍" + - "在低于 TP8 的 Hopper 臂上,dsv41flash_fp4_vllm_mtp.sh 将 --max-num-batched-tokens 上限设为 4096(上游 16384 时 indexer 的 [batched-tokens, 1M] fp8 缓冲区达 32 GiB),--max-num-seqs 限定为并发的两倍(16-256),CUDA graph 捕获上限 512 token;判定条件为 TP<8 且 GPU 显存低于 150 GB,因此 B200/GB200 TP4 与所有 TP8 臂不受影响" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3310 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Add B200 TP2 DeepSeek-V4.1-Flash with --engram-config '{\"cpu_offload\":true}' for AgentX, at concurrency 1, 2, 4, 8, 16, 32, 64, 128; retain the existing TP4 AgentX arm and pinned vLLM image." + - "Keep DSpark5, golden AL 3.51 for throughput, real block rejection with adaptive verification for evals, and GPU-resident KV." + - "新增 B200 TP2 DeepSeek-V4.1-Flash AgentX 配方,启用 Engram CPU 卸载,并发为 1、2、4、8、16、32、64、128;保留现有 TP4 AgentX 分支和固定 vLLM 镜像。吞吐使用 DSpark5 和黄金 AL 3.51,eval 使用真实块拒绝采样和自适应验证,KV 保留在 GPU 上。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "TP2 only: cap --max-num-batched-tokens at 4096, bound --max-num-seqs to 2*CONC (max 256) and stop CUDA graph capture at 512 tokens in the shared AgentX script, after run 35180394796 served TP2 c1-c16 but c32, c64 and c128 died in memory profiling with -10.8 GiB of KV budget (145 GiB of weights per GPU plus the 32 GiB indexer buffer at 16384 batched tokens); TP4 and TP8 arms keep the upstream defaults" + - "仅 TP2:在共享 AgentX 脚本中将 --max-num-batched-tokens 上限设为 4096,--max-num-seqs 限制为 2*CONC(最多 256),CUDA graph 捕获止于 512 token;运行 35180394796 中 TP2 c1-c16 可服务,但 c32、c64、c128 在显存分析阶段因 KV 预算为 -10.8 GiB 而失败(每 GPU 145 GiB 权重加 16384 批处理 token 时 32 GiB 的 indexer 缓冲区);TP4 与 TP8 配方保持上游默认" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Floor the TP2 --max-num-seqs at 16 (it stays 2x the concurrency above that, capped at 256): in run 35320655804 c1, c2 and c4 never started because FlashInfer's autotune dummy run batches max-num-seqs requests through the DSpark draft head and with 2-8 requests selected an invalid MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4), while c8 with 16 seqs and every larger point served" + - "将 TP2 的 --max-num-seqs 下限设为 16(高于该值时仍为并发数的 2 倍,上限 256):运行 35320655804 中 c1、c2、c4 未能启动,原因是 FlashInfer 的 autotune dummy run 会让 max-num-seqs 个请求经过 DSpark draft head,在 2-8 个请求时选中了无效的 MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4),而 c8(16 seqs)及更大的点均正常服务" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3216 + +- config-keys: + - dsv41flash-fp4-gb200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a TP2 arm to the GB200 vLLM DeepSeek-V4.1-Flash AgentX recipe alongside the existing TP4 arm, keeping the Engram tables in pinned host DRAM via --engram-config cpu_offload; weights rise to ~145 GiB on each 256 GiB GPU" + - "Apply the B200 TP2 caps in dsv41flash_fp4_vllm_mtp.sh to every TP2 arm: --max-num-batched-tokens 4096 (the sparse-attention indexer's [batched-tokens, 1M] fp8 buffer is 32 GiB at the upstream 16384), --max-num-seqs at twice the concurrency (16-256) and CUDA graph capture stopped at 512, which leaves ~49 GiB of KV per GPU; TP4 and TP8 arms keep the upstream defaults" + - "为 GB200 vLLM DeepSeek-V4.1-Flash AgentX 配方在现有 TP4 臂旁新增 TP2 臂,Engram 表继续通过 --engram-config cpu_offload 放在固定页主机 DRAM;每张 256 GiB GPU 的权重升至约 145 GiB" + - "在 dsv41flash_fp4_vllm_mtp.sh 中将 B200 TP2 的上限推广到所有 TP2 臂:--max-num-batched-tokens 4096(上游 16384 时 indexer 的 [batched-tokens, 1M] fp8 缓冲区达 32 GiB)、--max-num-seqs 为并发的两倍(16-256)、CUDA graph 捕获上限 512,为每张 GPU 留出约 49 GiB KV;TP4 与 TP8 臂沿用上游默认值" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3320 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a TP2 arm to the GB300 vLLM DeepSeek-V4.1-Flash AgentX recipe alongside the existing TP4 arm, keeping the Engram tables in pinned host DRAM via --engram-config cpu_offload; weights rise to ~175 GiB on each 277 GiB GPU" + - "Apply the B200 TP2 caps in dsv41flash_fp4_vllm_mtp.sh to every TP2 arm: --max-num-batched-tokens 4096 (the sparse-attention indexer's [batched-tokens, 1M] fp8 buffer is 32 GiB at the upstream 16384), --max-num-seqs at twice the concurrency (16-256) and CUDA graph capture stopped at 512, which leaves ~36 GiB of KV per GPU; TP4 and TP8 arms keep the upstream defaults" + - "为 GB300 vLLM DeepSeek-V4.1-Flash AgentX 配方在现有 TP4 臂旁新增 TP2 臂,Engram 表继续通过 --engram-config cpu_offload 放在固定页主机 DRAM;每张 277 GiB GPU 的权重升至约 175 GiB" + - "在 dsv41flash_fp4_vllm_mtp.sh 中将 B200 TP2 的上限推广到所有 TP2 臂:--max-num-batched-tokens 4096(上游 16384 时 indexer 的 [batched-tokens, 1M] fp8 缓冲区达 32 GiB)、--max-num-seqs 为并发的两倍(16-256)、CUDA graph 捕获上限 512,为每张 GPU 留出约 36 GiB KV;TP4 与 TP8 臂沿用上游默认值" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3321 + +- config-keys: + - qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - 'Add B300 Qwen3.5 FP8 AgentX 1P1D recipes: TP4 at C4/C12/C24, TP2+EP2 at C32/C40/C44/C48/C56, and prefill TP4+EP4 + at C32.' + - Use prefill HiCache, ReplaySSM at C56, and tuned decode capacity. + - Use SGLang nightly 20260918-20518d85 and Dynamo/SRT Slurm with the official FP8 checkpoint’s embedded NEXTN head. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3268 + +- config-keys: + - qwen3.5-fp8-h200-sglang-agentic-hicache-mtp + description: + - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9 (2026-09-16 upstream cu13 nightly, sgl-project/sglang@c9a8fba9, index digest sha256:509c2742b9441fe9182ae3dd7dac8840ba48e03d2d44b7b24e80d428f117fd85, amd64 digest sha256:19a1b266fac457d4ef92cba90b464119e42bdbee680c081df0a73f31461c03ca; Docker Hub last pushed 2026-09-16T01:06:04Z). This is the first cu13 nightly carrying sgl-project/sglang#39516, the CUDA fallback for the HiCache registered host-pool ImportError that sgl-project/sglang#35233 introduced against the pinned sglang-kernel 0.4.7 wheel and that broke every kv-offloading dram point on the 2026-09-15 build. The image moves sglang-kernel 0.4.6.post1 to 0.4.7 and sgl-deep-gemm 0.1.7 to 0.2.0; CUDA 13.0.3, NCCL 2.30.7 and FlashInfer 0.6.18 are unchanged, and server_args.py is identical across the 100 upstream commits. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps with golden AL 3.39, flashinfer attention with allreduce fusion, fp8 quantization with fp8_e4m3 KV, HiCache kernel IO / page_first layout / write_through_selective; the TP8/EP1 DRAM HiCache c2-c24 grid is unchanged." + - "将 SGLang 镜像从 lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161 更新为 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9(2026-09-16 上游 cu13 nightly,sgl-project/sglang@c9a8fba9,索引摘要 sha256:509c2742b9441fe9182ae3dd7dac8840ba48e03d2d44b7b24e80d428f117fd85)。这是首个包含 sgl-project/sglang#39516 的 cu13 nightly,该修复为 sgl-project/sglang#35233 在固定 sglang-kernel 0.4.7 wheel 上引入的 HiCache 已注册主机池 ImportError 提供 CUDA 回退,该问题曾导致 2026-09-15 构建的所有 kv-offloading dram 点无法启动。镜像将 sglang-kernel 由 0.4.6.post1 升至 0.4.7、sgl-deep-gemm 由 0.1.7 升至 0.2.0;CUDA 13.0.3、NCCL 2.30.7 与 FlashInfer 0.6.18 不变,100 个上游提交中 server_args.py 亦未改动。配方脚本、MTP 设置、HiCache 分支及 TP8/EP1 DRAM HiCache c2-c24 网格保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3200 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + description: + - "Update the vLLM ROCm image from nightly-d9105ea8 to v0.29.0; benchmark configuration unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3063 + +- config-keys: + - glm5.2-fp8-mi325x-sglang-agentic-mtp + description: + - "Update the SGLang ROCm image from v0.5.16-rocm720-mi30x to v0.5.19-rocm720-mi30x; benchmark configuration unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2980 + +- config-keys: + - dsv41flash-fp4-mi325x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add TP4 and TP2 arms with Engram CPU offload to the MI325X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the 183 GiB of tables GPU-resident. With the tables in pinned host memory the resident weights are ~81 GiB per rank at TP4 and ~145 GiB per rank at TP2, against 256 GB per card, so TP2 keeps the full concurrency 1-32 list here" + - "Cap the scheduler at TP2 only: halve the sparse-attention indexer buffer to 8192 batched tokens (16 GiB rather than 32 GiB of a [batched-tokens, 1M] fp8 logits allocation), scale max-num-seqs to 2*CONC clamped to [16, 256] and cap cudagraph capture at 512. Same cap set the Blackwell TP2 arms validated on B200, GB200 and GB300" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the ROCm 10.0 nightly channel build of the same commit the MI355X arm moved to in SemiAnalysisAI/InferenceX#3326 (digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1, published 2026-09-21T05:50:03Z); it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" + - "为 MI325X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 与 TP2 臂,同时保留 183 GiB Engram 表驻留 GPU 的现有 TP8 臂;表移至固定页主机内存后,TP4 每 rank 常驻权重约 81 GiB,TP2 约 145 GiB,而每张卡为 256 GB,因此 TP2 在此保留完整的并发 1-32 列表" + - "仅在 TP2 收紧调度器:将稀疏注意力 indexer 缓冲区减半至 8192 batched tokens([batched-tokens, 1M] fp8 logits 分配由 32 GiB 降至 16 GiB),max-num-seqs 取 2*CONC 并夹在 [16, 256],cudagraph capture 上限 512。该组参数与 Blackwell TP2 臂在 B200、GB200、GB300 上验证过的一致" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984(与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建);该镜像包含 vllm-project/vllm#57491,将两处 is_cuda() 判定放宽为 is_cuda_alike(),使 engram_config 在 gfx942 上可解析;由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启,配方按 TP 显式设置该值" + - "新增 --no-swa-bounded-replay:vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启,其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失,曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃;gfx942 使用同一路径。prefix caching 保持开启" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3337 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Add B200 Dynamo+SGLang AgentX TP8 aggregate HiCache recipes at concurrency 1, 4, and 8." + - "Use lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 with bundled DSpark K=6, full-node CPU/DRAM allocation, and per-point request/CUDA-graph limits." + - "Map the exact DeepSeek-V4-Pro-0813 recipe alias to the pre-staged NScale checkpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3257 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add TP4 and TP2 arms with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the 183 GiB of tables GPU-resident. With the tables in pinned host memory the resident weights are ~81 GiB per rank at TP4 and ~145 GiB per rank at TP2, against 192 GB per card" + - "Cap the scheduler at TP2 only: halve the sparse-attention indexer buffer to 4096 batched tokens (8 GiB rather than 16 GiB of a [batched-tokens, 1M] fp8 logits allocation), scale max-num-seqs to 2*CONC clamped to [16, 256] and cap cudagraph capture at 512. Same cap set the Blackwell TP2 arms validated on B200, GB200 and GB300. TP2 concurrency stops at 8 because ~145 GiB of weights leaves the smallest KV pool of any arm here" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the ROCm 10.0 nightly channel build of the same commit the MI355X arm moved to in SemiAnalysisAI/InferenceX#3326 (digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1, published 2026-09-21T05:50:03Z); it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" + - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 与 TP2 臂,同时保留 183 GiB Engram 表驻留 GPU 的现有 TP8 臂;表移至固定页主机内存后,TP4 每 rank 常驻权重约 81 GiB,TP2 约 145 GiB,而每张卡为 192 GB" + - "仅在 TP2 收紧调度器:将稀疏注意力 indexer 缓冲区减半至 4096 batched tokens([batched-tokens, 1M] fp8 logits 分配由 16 GiB 降至 8 GiB),max-num-seqs 取 2*CONC 并夹在 [16, 256],cudagraph capture 上限 512。该组参数与 Blackwell TP2 臂在 B200、GB200、GB300 上验证过的一致。TP2 并发止于 8,因为约 145 GiB 权重使其 KV 池为本配方中最小" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984(与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建);该镜像包含 vllm-project/vllm#57491,将两处 is_cuda() 判定放宽为 is_cuda_alike(),使 engram_config 在 gfx942 上可解析;由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启,配方按 TP 显式设置该值" + - "新增 --no-swa-bounded-replay:vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启,其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失,曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃;gfx942 使用同一路径。prefix caching 保持开启" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Drop the TP2 arm from the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe: measured infeasible. In Actions run 35671005506 the TP2 concurrency-1 point reported \"Available KV cache memory: -13.51 GiB\" and the engine refused to start with \"No available memory for the cache blocks\", even with the sparse-attention indexer buffer already halved to 4096 batched tokens. At TP2 the Engram-offloaded resident weights are ~145 GiB per rank, which a 192 GB card cannot hold alongside a usable 1M-context KV pool. Raising gpu-memory-utilization to 0.95 recovers ~9 GiB and halving the buffer again ~4 GiB, which still leaves effectively no KV, so the arm is removed rather than tuned. MI325X runs the same TP2 configuration on 256 GB per card" + - "The TP8 and TP4 arms are unaffected and stay. TP4 passed at concurrency 1 and 32 in the same run with the tables offloaded, at ~81 GiB per rank" + - "从 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方中移除 TP2 臂:经实测不可行。在 Actions 运行 35671005506 中,TP2 并发 1 数据点报告 \"Available KV cache memory: -13.51 GiB\",引擎以 \"No available memory for the cache blocks\" 拒绝启动,且此时稀疏注意力 indexer 缓冲区已减半至 4096 batched tokens。TP2 下经 Engram 卸载后每 rank 常驻权重约 145 GiB,192 GB 的卡无法在容纳该权重的同时保留可用的 1M 上下文 KV 池。将 gpu-memory-utilization 提高到 0.95 可回收约 9 GiB,再次将缓冲区减半可回收约 4 GiB,仍几乎不剩 KV,因此选择移除该臂而非调参。MI325X 以每卡 256 GB 运行相同的 TP2 配置" + - "TP8 与 TP4 臂不受影响并保留。TP4 在同一运行中以每 rank 约 81 GiB 在并发 1 与 32 通过" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 + +- config-keys: + - dsv41flash-fp4-b200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Pin B200 DeepSeek-V4.1 Flash to SGLang nightly-dev-cu13-20260922-582389ce by digest and preserve shipped DSpark precision with the committed five-token golden acceptance curve" + - "Use per-rank host Engram shards, interval 16, and concurrency-scaled SWA prefix capacity: TP4 C1-128 uses static fraction 0.80, chunk 4096, and 128-4096 tails; TP2 C1-8 uses static fraction 0.92, chunk 2048, and 128 tails per session" + - "Test TP2 loading with the unmodified pinned stock loader and expandable CUDA allocator segments alone; preserve tensor payloads, layouts, and computation" + - "Resolve local model snapshots, retain robust AgentX HTTP timeouts, and normalize digest-pinned images for the installed Enroot parser with immediate import failure handling" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3346 + +- config-keys: + - dsv41flash-fp4-b300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - Add B300 SGLang DeepSeek-V4.1-Flash AgentX TP4/EP4 with the newest official CUDA 13 nightly (20260922-582389ce), pinned amd64 digest, native DSpark5, measured thinking-on AL 3.51 for throughput and real verification for evals. + - Keep automatic backends, host Engram tables, GPU-resident KV, bounded prefill and decode graphs; route the B300 SGLang runtime through /ix. + - Use per-rank anonymous host Engram tables to allow transparent huge pages without changing host settings; preserve native FP8 table payload and automatic backend selection. + - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." + - "Add DSpark TP2/EP2 alongside TP4/EP4 to explore the vLLM parallelism frontier, keeping native host Engram tables and GPU KV; interleave 16 decode batches between prefill chunks to test long-context responsiveness." + - "Submit this SGLang AgentX lane through a Slurm batch allocation and launch its steps from the compute node, avoiding failed interactive allocation notifications on the runner login host while preserving GPU reservations and terminal-status checks." + - "Test retained SWA tails at 64 times concurrency, bounded to 128–4096; use static memory 0.80 for TP2 C16 and 0.85 at C32 and above after measuring its smaller KV budget and 43.59 GiB of post-graph headroom at C64, otherwise retain 0.70. Prefetch checkpoints for shared Lustre loading. Test prefill/decode interval 4 at TP2 C32 and above against the interval-16 baseline; keep other points at 16." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3342 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Raise ATOM GPU memory utilization from 0.90 to 0.95 and declare a fine-grained CUDA graph ladder (capture sizes 1-64) sized for the observed AgentX running batch, which is 22-41% of CONC; ModelRunner trims it to min(2*CONC, 8192). A controlled TP4 C24 A/B improved throughput by 9.2% and interactivity by 50.7% with unchanged estimated graph memory." + - "Enable ATOM_M3_INDEXER_CP for TP4 C15/C20/C24/C28/C32/C40/C48, where MiniMax-M3's four index query heads can context-parallelize the replicated single index-K head; the TP-size gate (tp_size == sparse_num_index_heads == 4) keeps it disabled for the TP2 C20 offload point. Same-commit A/B pairs at TP4 C15/C24/C28 improved interactivity by 3.7%/12.3%/16.7% and ISL-normalized throughput by 0.67%/0.24%/2.7%; C20 is interpolated between measured C15 and C24. Restore EAGLE3 K3 with golden AL 2.78 at C40/C48 instead of disabling speculative decoding." + - "Collapse the per-band capacity knobs into a single case statement, add the measured TP2 LMCache DRAM-offload curve at C20/C25/C30 on the chunk-256 ATOM_SLRU tier, size LMCACHE_MAX_LOCAL_CPU_SIZE as TOTAL_CPU_DRAM_GB/TP, and derive LMCACHE_LOOKUP_SERVER_WORKER_IDS from TP so every rank is covered. Remove the TP2 C5 point and the unreachable hybrid/state-offload (STATE_OFFLOAD/OFFLOAD_STATE) and C56 branches." + - "Spread offload ranks across NUMA nodes by selecting GPUs 0,4 for TP2 and 0,1,4,5 for TP4 when the caller has not pinned devices, cutting TP2 host-memory pinning from 45 minutes to 21 seconds at unchanged throughput. Move to rocm/atom-dev:nightly_202609171455 and record LMCache 0.5.5rc3+rocm7.2.4 for both offload arms." + - "Set dram-utilization to 0.687 so the recorded budget matches the real per-rank pinning: LMCACHE_MAX_LOCAL_CPU_SIZE = TOTAL_CPU_DRAM_GB/TP resolves to ~256 GB per rank on both offload arms (512 GB total at TP2, 1,024 GB at TP4), independent of TP. This replaces the earlier 0.20 metadata that under-described the allocation." + - "Verified against ATOM commit b6e2a39433d73b586c366ca90e661f098eb463e3 that --online_quant_config ptpc_fp8 does not change the Inferact/MiniMax-M3-EAGLE3-GQA draft precision: atom/models/eagle3_llama.py builds every draft linear (qkv_proj, o_proj, gate_up_proj, down_proj, fc) with no quant_config, so should_stream_online_quant short-circuits to False and OnlineQuantStreamer.maybe_create finds no streaming candidates. The EAGLE3 draft loads and runs at its native bf16 checkpoint precision; only the target MiniMax-M3 weights are online-quantized to ptpc_fp8, and the exclude_layer list applies to the target alone." + - "将 MiniMax-M3 ATOM AgentX 的 GPU 显存利用率提高到 0.95,并按真实运行 batch(约为 CONC 的 22-41%)声明细粒度 CUDA graph(capture 档 1-64,运行时裁剪为 min(2*CONC, 8192));TP4 在 C15/C20/C24/C28/C32/C40/C48 启用 indexer-only CP,TP2 C20 由 TP-size gate 保持关闭,C40/C48 恢复 EAGLE3。将分档容量参数合并为单一 case,新增 TP2 C20/C25/C30 的 LMCache SLRU offload 曲线,LMCACHE_MAX_LOCAL_CPU_SIZE 取 TOTAL_CPU_DRAM_GB/TP,lookup worker 覆盖全部 rank,并将 offload GPU 跨 NUMA 节点放置(TP2 用 0,4;TP4 用 0,1,4,5),把 TP2 主机内存 pinning 从 45 分钟缩短到 21 秒;更新到 20260917 rocm7 ATOM 镜像 nightly_202609171455 与 LMCache 0.5.5rc3。dram-utilization 改为 0.687,使记录预算与每 rank 约 256 GB 的真实固定一致(与 TP 无关),取代此前 0.20 的元数据。已在 ATOM commit b6e2a394 验证 online_quant(ptpc_fp8)不改变 EAGLE3-GQA 草稿模型精度:eagle3_llama.py 的各 draft 线性层不带 quant_config,should_stream_online_quant 直接返回 False,草稿以原生 bf16 运行,仅目标 MiniMax-M3 权重在线量化为 ptpc_fp8。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3189 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Track the updated GLM-5.2 standalone recipe (ROCm/ATOM PR 2345) on the MI355X ATOM AgentX config." + - "Bump the image from rocm/atom-dev:nightly_202609151450 to rocm/atom-dev:nightly_202609211553, which carries ROCm/ATOM PR 2345." + - "Drop the LMCache DRAM offload from the small-concurrency TP4 arm: conc [2, 4, 8, 10] moves from kv-offloading dram with the lmcache 0.4.5 backend to kv-offloading none, so those points serve off the native GPU prefix cache alone and no longer start an LMCache CPU tier. The TP4+DCP4 arm at conc [16, 24, 32, 40, 48] keeps LMCache at 256 GiB per rank (dram-utilization 0.171 unchanged), and the TP8 arm stays GPU-resident as before." + - "Add --index_cache_dtype fp4 and --block-size 64 to the ATOM server launch on every point. Neither flag was set before, so both took the ATOM default." + - "Drop ATOM_USE_FLYDSL_GATHER_KV_B_PROJ=0, which pinned gather_kv_b_proj to the Triton path on the DCP prefill-context and MTP verify paths; the updated recipe no longer sets it." + - "The MTP block data precision is not touched by --online_quant_config: the exclude_layer list adds \"model.layers.78.*\" so layer 78 (the GLM-5.2 MTP head) is skipped by online quantization and stays in native BF16, as it ships unquantized in amd/GLM-5.2-MXFP4. The expert excludes only reach layers 0-77, so without this exclude the whole MTP block would be online-quantized to ptpc_fp8 while the target's experts stay MXFP4." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3359 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the GLM-5.2 MXFP4 MI355X SGLang AgentX image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260920, whose SGLang commit c610c40399 contains sgl-project/sglang#37152: the ROCm HiCache JIT gains copy rounds narrower than 128 B, which is what lets GLM's 576 B MLA fp8 row reach the transfer kernels instead of falling back." + - "Remove the explicit direct/page_first_direct HiCache overrides so the TP4/EP4 HiCache arm runs SGLang's default kernel/page_first path; the dormant DP-attention and Mooncake branches drop the same two overrides. sgl-project/sglang#35233 is what makes that path correct on ROCm, by handing the transfer kernels the device-accessible alias of the registered host pool; upstream measured kernel/page_first at 338 tok/s against 291 tok/s for direct/page_first_direct at concurrency 14 on this same TP4/EP4 MXFP4 MI355X arm." + - "Raise dram-utilization from 0.80 to 0.85 and replace the TP4/EP4 arm's ratio-1.0 host KV pool (~115.19 GB/rank measured) with a fixed 180 GB/rank pool; the DSA indexer scales to ~41.25 GB/rank, leaving ~125 GB of startup headroom under the 1,274 GB SA budget. Set that arm's sweep to concurrency 4, 8, 10, 12, 14, and 16, and limit the TP8/EP1 GPU-resident arm to 1, 2, and 4." + - "将 GLM-5.2 MXFP4 MI355X SGLang AgentX 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260917 更新到 lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260920;该镜像的 SGLang commit c610c40399 已包含 sgl-project/sglang#37152:ROCm HiCache JIT 新增了窄于 128 B 的 copy round,GLM 的 576 B MLA fp8 行因此能够走 transfer kernel 而不再回退。" + - "删除显式的 direct/page_first_direct HiCache 覆盖,使 TP4/EP4 HiCache arm 运行 SGLang 默认的 kernel/page_first 路径;处于休眠状态的 DP-attention 与 Mooncake 分支同样删除这两个覆盖。该路径在 ROCm 上的正确性来自 sgl-project/sglang#35233——它把 registered host pool 的 device-accessible alias 交给 transfer kernel;上游在同一个 TP4/EP4 MXFP4 MI355X arm 上实测并发 14 时 kernel/page_first 为 338 tok/s,direct/page_first_direct 为 291 tok/s。" + - "将 dram-utilization 从 0.80 提升到 0.85,并把 TP4/EP4 arm 的 ratio-1.0 host KV pool(实测约 115.19 GB/rank)改为固定 180 GB/rank;DSA indexer 同比扩展到约 41.25 GB/rank,在 SA 的 1,274 GB 预算内保留约 125 GB 启动余量。该 arm 的并发 sweep 为 4、8、10、12、14、16,TP8/EP1 GPU-resident arm 仅保留 1、2、4。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3329 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Add GLM-5.3 FP8 MI355X AgentX (agentic-coding) via TileRT prefill/decode disaggregation with MTP, 1 prefill node (TP8) + 1 decode node (TP8), concurrency [1], mirroring the B200 glm5.1-fp8-b200-tilert-agentic entry on the amd_utils orchestration." + - "Two images per job, both first-party and neither carrying the TileRT wheel: the decode rank runs ghcr.io/tile-ai/tilert-rocm-decode:0.1.6 (ROCm PyTorch runtime plus the mooncake ROCm transfer engine built from source) and the prefill rank runs ghcr.io/tile-ai/tilert-rocm-prefill:0.1.6 (the upstream vllm/vllm-openai-rocm release image plus that same transfer engine, with vLLM itself unpatched and no file inside it modified), selected per rank via PREFILL_IMAGE in prefill.additional-settings. setup_deps.sh pip-installs the official tilert==0.1.6 wheel into both ranks at container start (full install plus the fastapi/uvicorn/httpx router deps on decode, --no-deps connector plugin on prefill, whose vLLM image already ships those), so neither image is pinned to a wheel version; this mirrors the B200 glm5.1-fp8-b200-tilert pattern of a first-party TileRT decode image. The transfer engine is built from source because the published mooncake ROCm wheels are compiled without ENABLE_MULTI_PROTOCOL, which leaves the cross-host locality gate (kvcache-ai/Mooncake#2753) out of the binary so every cross-node KV transfer is treated as node-local and fails in hipIpcOpenMemHandle." + - "KV moves P->D over mooncake (kv-p2p-transfer: mooncake); both roles use a bf16 MLA KV cache (prefill --kv-cache-dtype bfloat16, decode bf16). This is the only layout TileRT 0.1.6 can consume from vLLM on ROCm: TileRT infers the layout from the cache tensor stride and accepts 1152 B/token (bf16) or 656 B/token (fp8_ds_mla); vLLM's ROCM_AITER_MLA_SPARSE backend has no fp8_ds_mla and its plain fp8 writes a flat 576 B/token row that the connector rejects at register_kv_caches. max-model-len 1048576 on both roles (GLM-5.3's full context window); MTP wired via spec-decoding=mtp: TileRT decode drafts at depth 3 (DECODE_MTP_SIZE, the only depth the ROCm GLM profile builds, and the key the golden acceptance curve is read at), while the vLLM prefill rank only materialises the MTP layer's KV and runs speculative-config mtp with 1 draft token. TileRT decode is bs=1 only, so conc-list is [1]; topology 1 prefill node (TP8) + 1 decode node (TP8)." + - "Orchestration: runners/launch_mi355x-amds.sh dispatches framework=tilert through the existing amd_utils chain (submit.sh -> job.slurm -> server.sh -> new server_tilert.sh); job.slurm selects the image per rank and a tilert env block, setup_deps.sh gains a per-role tilert install branch, models_tilert.yaml holds the GLM-5.3 prefill environment (the VLLM_ROCM_USE_AITER* block) and extra-flag hooks. TileRT decode weights are converted once from the HF checkpoint under a flock and cached next to it on the shared model volume. Cross-node teardown uses the amd_utils router-port-close barrier." + - "The pd_router runs with --parser none and a 1800-second queue timeout so requests wait on the bs=1 decode engine instead of failing fast; AgentX traces are filtered at the 1048576-token full GLM context both roles are launched with; AIPerf scrapes only the vLLM prefill Prometheus endpoint (TileRT decode exposes no /metrics route)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3330 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Set RDMAV_FORK_SAFE=1 on both TileRT ranks for AgentX, preventing silent RDMA-to-TCP fallback that degrades KV-transfer latency and TTFT; TPOT is unaffected." + - "为 AgentX 的 TileRT 两侧设置 RDMAV_FORK_SAFE=1,避免静默从 RDMA 回退到 TCP 导致 KV 传输延迟和 TTFT 恶化;TPOT 不受影响。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3330 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Serve GLM-5.3's full 1048576-token context on both TileRT ranks with the bf16 MLA wire layout, gpu-memory-utilization 0.85 on the vLLM prefill, and the container-creation barrier timeout taken from CONTAINER_BARRIER_TIMEOUT (5400s) instead of a hardcoded 1800s. bf16 is the only layout TileRT 0.1.6 can consume from vLLM on ROCm: TileRT infers the layout from the cache tensor stride and accepts 1152 B/token (bf16) or 656 B/token (fp8_ds_mla), while ROCM_AITER_MLA_SPARSE has no fp8_ds_mla and its plain fp8 writes a flat 576 B/token row that the connector rejects at register_kv_caches. Per rank at this context (287.98 GiB MI355X): decode holds 90.72 GiB of weights, a 93.25 GiB engine cache window and a 99.06 GiB PD receive buffer; prefill holds 90.45 GiB of weights, 40.3 GiB of profiling/non-torch memory, 91.71 GiB of vLLM KV and a 99.06 GiB PD staging buffer on TP rank 0 outside vLLM's budget. Both PD buffers are dense in max_seq_len and must live in pinned host memory for this context to start: on the GPU the decode side is node-marginal (torch.OutOfMemoryError on one node, 99.06 GiB requested with 95.94 GiB free; served on another) and the prefill side cannot fit at any utilization. setup_deps.sh therefore applies patches/tilert-0.1.6-pd-buffers-in-dram.patch to the pip-installed tilert 0.1.6 at container start on both ranks (engine-patch waiver docs/waiver/3330.md; vLLM stays unpatched), placing both PD buffers in 2 MiB-backed pinned host memory behind TILERT_PD_BUFFER_DEVICE=cpu. The backing is verified before registration because the ionic RDMA VFs cap 4 KiB-page registrations at ~3.9 GiB per HCA (a ~2^20 page-table-entry budget), while 2 MiB-backed regions of 100 GiB register and take cross-node mooncake writes at ~21 GiB/s. The patch is removed when a TileRT release carries DRAM PD buffers. The barrier change covers cold image pulls: each rank pulls its own ~32 GB image, and the decode image alone took about 25 minutes of the old 30-minute budget on an uncached node. The launcher also passes sync.py --grace 60: the barrier's post-pass port-hold defaults to max(60, timeout/2) seconds in the foreground, which at 5400s was a 45-minute idle sleep on every rank (jobs 45373/45374)." + - "以 bf16 MLA 线上布局在两侧 TileRT rank 提供 GLM-5.3 完整的 1048576 上下文,vLLM prefill 的 gpu-memory-utilization 设为 0.85,容器创建屏障超时改为由 CONTAINER_BARRIER_TIMEOUT(5400 秒)提供而非硬编码 1800 秒。bf16 是 TileRT 0.1.6 在 ROCm 上唯一能从 vLLM 读取的布局:TileRT 依据缓存张量步长推断布局,只接受每 token 1152 字节(bf16)或 656 字节(fp8_ds_mla),而 ROCM_AITER_MLA_SPARSE 没有 fp8_ds_mla,其普通 fp8 写出每 token 576 字节的扁平行,会在 register_kv_caches 被 connector 拒绝。该上下文下每 rank(287.98 GiB 的 MI355X):decode 侧含 90.72 GiB 权重、93.25 GiB 引擎缓存窗口和 99.06 GiB 的 PD 接收缓冲;prefill 侧含 90.45 GiB 权重、40.3 GiB 分析/非 torch 内存、91.71 GiB vLLM KV,以及 TP rank 0 上位于 vLLM 预算之外的 99.06 GiB PD 暂存缓冲。两个 PD 缓冲均按 max_seq_len 密集分配,必须放在锁页主机内存中该上下文才能启动:放在 GPU 上时 decode 侧处于节点边缘(一节点 OOM,请求 99.06 GiB 而仅剩 95.94 GiB;另一节点可服务),prefill 侧在任何利用率下都放不下。因此 setup_deps.sh 在两侧容器启动时对 pip 安装的 tilert 0.1.6 应用 patches/tilert-0.1.6-pd-buffers-in-dram.patch(引擎补丁豁免 docs/waiver/3330.md;vLLM 未改动),在 TILERT_PD_BUFFER_DEVICE=cpu 下将两个 PD 缓冲放入 2 MiB 大页支撑的锁页主机内存。注册前会校验大页覆盖:ionic RDMA VF 对 4 KiB 页的注册上限约为每 HCA 3.9 GiB(约 2^20 条页表项),而 2 MiB 大页区域可注册 100 GiB 并以约 21 GiB/s 接收跨节点 mooncake 写入。TileRT 发布自带 DRAM PD 缓冲的版本后移除该补丁。屏障改动用于覆盖镜像冷拉取:每个 rank 各拉取约 32 GB 的镜像,仅 decode 镜像在未缓存节点上就实测约占旧 30 分钟预算的 25 分钟。启动脚本同时向 sync.py 传入 --grace 60:屏障通过后默认在前台保持端口 max(60, timeout/2) 秒,在 5400 秒下即每个 rank 空转 45 分钟(作业 45373/45374)。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3330 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Add glm5.3* to resolve_trace_source's 1M-context model pattern so GLM-5.3 AgentX uses the unfiltered semianalysis_cc_traces_weka_062126 corpus instead of the 256k-filtered variant. Explicit WEKA_LOADER_OVERRIDE behavior is unchanged." + - "将 glm5.3* 加入 resolve_trace_source 的 1M 上下文模型匹配分支,使 GLM-5.3 AgentX 默认使用未截断的 semianalysis_cc_traces_weka_062126 语料,而非 256k 过滤版本。显式 WEKA_LOADER_OVERRIDE 的行为保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3330 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Replace the PD-buffers-in-DRAM route with tilert==0.1.6.post1 on both TileRT ranks (PyPI, 2026-09-22): the shipped wheel shards the mooncake receive buffer (decode, receive_server.py) and the connector staging buffer (prefill TP rank 0, prefill_connector.py) by layer across the eight devices (TILERT_PD_SHARDS, default 8, capped at device_count; layer lid on device lid % 8; the glm5_2 profile has 79 layers, so the heaviest shard holds 10 of them: 99.06 GiB x 10/79 = 12.54 GiB per card) instead of one dense 99.06 GiB buffer per role, and both fit on the GPU at the full 1048576-token context without host memory. Each shard is registered with mooncake on its own device, hello_layout carries per-shard bases and rdma_plan writes layer by layer into the matching remote shard, so the KV path is device-to-device again (81 GB in 751 ms = 108 GB/s on 2x8 MI350X per CrimsonDump/InferenceX@56a1e5a, against the 20.9 GiB/s measured for the host-to-host hop) and mla_nsa.convert() dequantises each layer on the shard that received it rather than on cuda:0. patches/tilert-0.1.6-pd-buffers-in-dram.patch, docs/waiver/3330.md and TILERT_PD_BUFFER_DEVICE (the AgentX recipe, server_tilert.sh, job.slurm, setup_deps.sh, including the 0.1.6 pin guard and the patch apt package) are removed; neither image is modified. Router metadata 0.1.6 -> 0.1.6.post1: pd_router.py is one of the six files post1 changes, its fixed 600 s request timeouts to vLLM and decode become TILERT_PD_HTTP_TIMEOUT_S (default 3600). GPU_MEM_UTIL stays 0.85; the 12.54 GiB staging shard sits outside vLLM's budget next to the ~6.3 GiB non-torch baseline. server_tilert.sh now honours EVAL_ONLY/RUN_EVAL on the AgentX path (GSM8K on the same router instead of an unconditional trace replay) and exports PORT for benchmark_lib's run_lm_eval; evals run with real MTP verification since simulated acceptance is already disabled under those flags. Measured by CrimsonDump on 2x8 MI350X at this context with bf16 KV, 3600 s AgentX at concurrency 1: submission_valid true, 239 successful requests, 0 errors, TTFT p50 5711.8 ms, ITL p50 2.18 ms, peak device memory 202.1 GiB per decode card and 269.2 GiB per prefill card of 287.69 GiB." + - "在两侧 TileRT rank 上以 tilert==0.1.6.post1(PyPI,2026-09-22)取代 PD 缓冲进 DRAM 的方案:正式 wheel 把 mooncake 接收缓冲(decode,receive_server.py)与 connector 暂存缓冲(prefill TP rank 0,prefill_connector.py)按层分片到 8 张卡(TILERT_PD_SHARDS,默认 8,以 device_count 为上限;第 lid 层落在第 lid % 8 张卡;glm5_2 profile 共 79 层,最重分片含 10 层:99.06 GiB x 10/79 = 每卡 12.54 GiB),不再是每个角色一块致密的 99.06 GiB 缓冲,于是在完整 1048576 上下文下两者都能放进 GPU,无需主机内存。各分片在所在卡上向 mooncake 注册,hello_layout 携带各分片基址,rdma_plan 逐层写入对应的远端分片,KV 通路重回设备到设备(2x8 MI350X 上 81 GB / 751 ms = 108 GB/s,见 CrimsonDump/InferenceX@56a1e5a;对比主机到主机实测 20.9 GiB/s),mla_nsa.convert() 在收到该层的分片上反量化而非集中于 cuda:0。移除 patches/tilert-0.1.6-pd-buffers-in-dram.patch、docs/waiver/3330.md 与 TILERT_PD_BUFFER_DEVICE(AgentX 配方、server_tilert.sh、job.slurm、setup_deps.sh,含 0.1.6 版本锁定检查与 patch apt 包);两侧镜像均无改动。router 元数据 0.1.6 -> 0.1.6.post1:pd_router.py 是 post1 改动的六个文件之一,其对 vLLM 与 decode 固定的 600 秒请求超时改为 TILERT_PD_HTTP_TIMEOUT_S(默认 3600)。GPU_MEM_UTIL 保持 0.85;12.54 GiB 暂存分片与约 6.3 GiB 非 torch 基线同在 vLLM 预算之外。server_tilert.sh 现在在 AgentX 路径也遵循 EVAL_ONLY/RUN_EVAL(在同一 router 上跑 GSM8K,而非无条件回放轨迹),并为 benchmark_lib 的 run_lm_eval 导出 PORT;这些标志下模拟接受已关闭,评测使用真实 MTP 验证。CrimsonDump 在 2x8 MI350X、该上下文、bf16 KV 下实测 3600 秒 AgentX 并发 1:submission_valid true、239 条成功、0 错误、TTFT p50 5711.8 ms、ITL p50 2.18 ms,decode 每卡峰值 202.1 GiB、prefill 每卡 269.2 GiB(卡容量 287.69 GiB)。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3366 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add GB300 SGLang DeepSeek-V4.1-Flash AgentX TP2/EP2 and TP4/EP4 candidates on official nightly-dev-cu13-20260922-582389ce, digest sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40." + - "The DSpark arm keeps canonical thinking-on five-token AL 3.51 and real eval verification, using the official nightly shipped draft precision and KV layout without custom precision patches; full-model evaluation uses real verification." + - "Keep original Engram tables in GPU memory on TP4 to avoid measured host huge-page fragmentation; TP2 uses per-rank anonymous host tables and local checkpoint paths for file-scoped cache advice without host sysctl changes. Keep GPU KV cache, 0.80 static memory fraction, request/graph cap 64 and automatic backends. Select 8192-token prefill chunks only at TP2 C64/C128; retain 4096 at C32 and all other points." + - "Prioritize DSpark across the full AgentX concurrency curve and test 16 decode steps between prefill chunks with unchanged shipped precision and canonical acceptance; remove the STP sweep arm because measured DSpark points dominate its completed diagnostics." + - "Reserve 64 SWA prefix tails per concurrency at C2 and above, capped at 4096, within the existing static memory budget. The 1024-tail TP4 C16 test improved throughput 4.32x over the matched default-tail baseline while preserving checkpoint math and precision." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3344 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Bump tilert 0.1.6.post1 -> 0.1.6.post2 (PyPI, 2026-09-23) on both TileRT ranks; router metadata follows. post2 changes six pd_vllm files: the ROCm decode inject (glm5_rocm_engine.py) now broadcasts each layer's KV to all eight rank caches with grouped RCCL ncclBroadcast (TILERT_INJECT_MODE, default rccl, falling back to per-source-device rotating streams, then the old serial copy) instead of serial default-stream copy_ calls, and the prefill extract (mla_nsa.py) gathers each layer with one index_select per device on side streams (TILERT_EXTRACT_MODE, default fast). The ROCm fp8 KV layout, multi-sender staging (TILERT_PD_SENDERS) and host-resident PD buffers (TILERT_PD_BUFFER_DEVICE) are new but opt-in and left off, so the 1M-context bf16 KV, layer-sharded on-GPU buffer configuration is otherwise unchanged. TileRT reports AgentX TTFT p50 5.7 s -> 2.2 s at concurrency 1. No patches, images unchanged." + - "两侧 TileRT rank 上 tilert 由 0.1.6.post1 升级到 0.1.6.post2(PyPI,2026-09-23),router 元数据随之更新。post2 修改了 pd_vllm 的六个文件:ROCm decode 注入(glm5_rocm_engine.py)改为用分组 RCCL ncclBroadcast 把每层 KV 广播到全部 8 个 rank 的缓存(TILERT_INJECT_MODE,默认 rccl,失败时回退到按源设备轮转的多流拷贝,再回退到旧的串行拷贝),不再在默认流上串行 copy_;prefill 提取(mla_nsa.py)在各设备的旁路流上每层一次 index_select 完成收集(TILERT_EXTRACT_MODE,默认 fast)。ROCm fp8 KV 布局、多发送端暂存(TILERT_PD_SENDERS)与主机内存 PD 缓冲(TILERT_PD_BUFFER_DEVICE)为新增的可选项,默认关闭,因此 1M 上下文、bf16 KV、按层分片的 GPU 缓冲配置其余不变。TileRT 报告并发 1 下 AgentX TTFT p50 由 5.7 秒降至 2.2 秒。无补丁,镜像不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3389 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Qualify DeepSeek-V4.1-Flash H200 SGLang on the latest official September 22 CUDA 13 nightly, matching the published vLLM starting topologies TP4/EP1 and TP8/EP1 at concurrency 1, 2, 4, 8, 16, 32, 64 and 128." + - "Use native Marlin MXFP4 MoE for both EP1 topologies: tensor-sharded expert widths 576 and 288 require supported padding, while the shipped SM90 CUTLASS method rejects these dimensions. Preserve shipped-default target and DSpark precision." + - "Retain per-rank anonymous Engram host tables, GPU-resident KV, memory fraction 0.70, prefill chunk 4096, prefill/decode interval 16 and 1024 SWA prefix tails for the exact-topology baseline." + - "Retain validated H200 dense-GEMM launch configurations without changing kernel code or precision; keep five-token DSpark, golden acceptance for performance and real verification for full GSM8K." + - "Run a new canonical full sweep with 16 performance and 16 full-evaluation cells. Earlier EP4/EP8 and DP8 measurements remain historical evidence and do not qualify the EP1 grid." + - "Extend only H200 SGLang AgentX performance cells at C64/C128 to 24-hour Slurm allocations and 24.5-hour GitHub jobs after the original eight-hour allocation expired during progressing warmup. Preserve normal warmup, the full 3600-second profile, image, model, precision and topology." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3341 + +- config-keys: + - dsv41flash-fp4-mi355x-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Use the user-selected official lmsysorg/sglang:dev-dsv41-mi35x model preview, pinned to sha256:c609f74f01b206af9664fb66da8ebef47642a7328a2478eabeebb352e014661a, with the native MI350X TP4/EP4 DSpark5 recipe, GPU Engram and native decode graphs." + - "Keep radix caching enabled for AgentX and bound prefill chunks to the 4096-token bound after native FP4 prefill scratch exhausted HBM with the 16384-token default in five canonical cells. Reserve additional native scratch headroom with static memory fraction 0.70 after C32 still exhausted HBM at 0.80 with 4096-token chunks. Enable native allocator garbage collection at threshold 0.8 with process fraction 0.99 after physical HBM still filled during long prefill at static fraction 0.70. Cap server admission at 32 after native C32 warmup reached 63 running requests and failed with HIP illegal access; preserve all client fanout as queued work. Disable native prefill graphs after the admission-capped C32 test still failed with HIP illegal access during breakable prefill; keep decode graphs and treat this as a bounded runtime correction pending qualification. Cap the full KV pool at 3145728 tokens after the pinned native compressed-cache store and its compiled gfx950 code showed signed int32 byte offsets that overflow for large full-token locations. This supported capacity limit keeps ratio-1 storage below 2 GiB without patching kernels or reducing the model context limit; validate canonical C32 and full GSM before selection. Scheduler cadence remains at the official default. Preserve full 1M context, shipped-default DSpark precision, canonical thinking-on AL3.51 throughput and real full1319 GSM verification." + - "Use image-native attention, dense FP8 and MoE kernels, allocator defaults and EP4 tuning data; remove the downstream nightly compatibility installer, source payload, tuning copy, hash checks and obsolete waiver documentation. The model-preview image explicitly supersedes the earlier latest-nightly selection for this lane." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3343 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260923-06008c17, pinned at multi-architecture manifest sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 (ARM64 sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3); match TP2/EP1 and TP4/EP1 with native DSpark precision and golden acceptance policy + - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' + - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." + - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. + - Retain min(64*concurrency, 1024) SWA prefix tails within the measured GB200 KV budget; select prefill/decode interval 16 only at TP4 C16 after canonical performance and full GSM8K qualification. + - "Match the published vLLM grid exactly: TP2/EP1 and TP4/EP1, each C1/2/4/8/16/32/64/128. Use stock DSpark precision; TP2 uses static memory 0.92, chunk 2048, graph/running cap 16, interval 16, expandable allocator and min(128*CONC,1024) SWA tails. Requalify all 16 performance and full GSM8K points. Avoid deadline-only failures with the supported 12-hour partition maximum for C64/C128 performance while preserving complete warmup and 3600-second scoring." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 + +- config-keys: + - dsv41flash-fp4-h200-sglang-agentic-dspark + description: + - "Enable official decoder SWA bounded replay for H200 DeepSeek V4.1 Flash native DSpark" + - "Pin SGLang nightly-dev-cu13-20260923-06008c17; qualify TP4/EP1 and TP8/EP1 C1-128 with unchanged shipped draft precision" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3392 + +- config-keys: + - dsv41flash-fp4-h100-sglang-agentic-dspark + - dsv41flash-fp4-h100-sglang-agentic-dspark-dpa + scenario-type: + - agentic-coding + description: + - "Move H100 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to upstream SGLang nightly-dev-cu13-20260922-582389ce and retain automatic attention selection from the current cookbook." + - "Use stock nightly DSpark precision and kernels, with row-sharded anonymous host Engram tables and a resolved local snapshot for upstream file-scoped page-cache eviction and huge-page backing." + - "Retain TP8/EP8, GPU-resident KV, 4096-token prefill chunks, static memory fraction 0.70, the shipped draft precision defaults, and the existing AgentX acceptance and real-verification eval behavior." + - "Use DSpark across the full AgentX concurrency curve with 16 decode steps between prefill chunks, unchanged shipped precision and canonical acceptance; completed STP diagnostics did not contribute a measured frontier point." + - "Install H100-specific block32 FP8 GEMM tiling configurations for six shapes, adding independently validated draft N512/K5120 and N1280/K5120 choices while retaining stock settings above batch320 and batch190 respectively, including supported SPLIT_K/SWAP_AB small-batch choices for N1792/K5120 and N576/K5120; retain previous large-batch settings and default tiling for the regressing N5120/K288 batch-64 case; preserve stock weights, scales and output precision and record the resolved config hashes." + - "Use expandable CUDA allocator segments to reduce fragmentation from long-context indexer masks without changing checkpoint precision or the static KV budget." + - "Select the supported native MXFP4 Marlin MoE backend on Hopper, retaining BF16 activations and the nightly's shipped draft precision." + - "Retain 8 SWA prefix tails per concurrency at C1/C2 after rejecting the 128-tail floor in matched canonical testing; use 32 per concurrency at C4 and above, reaching 640 at C20 within the static cache budget." + - "Qualify supported DP8 attention and DP LM-head at C4/C8/C16/C20 with consistent-hash session routing and 64 SWA tails per rank; retain the complete plain TP8 curve and unchanged shipped precision, static budget and native context." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3345 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Re-sweep the MI355X DeepSeek-V4.1-Flash vLLM AgentX arm on the ROCm 10.0 nightly nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423 (published 2026-09-24T05:45:09Z, digest sha256:5df261803a5b4e8c4d6bbe58a32c9cd597140b6fdfa7358c0ef0d0211c33ab32), replacing nightly-rocm100-3df4ae15. Only the image changes: the TP=4 and TP=2 rows keep concurrency 1, 2, 4, 8, 16, 32, 64 and 128, and every recipe setting is unchanged, including Engram placement per TP and concurrency, the prefill chunk and sequence cap ladder, graph capture, and five-token DSpark with golden acceptance 3.51 for throughput and real block verification for evals." + - "The pin is mainly for vllm-project/vllm#57435, which fuses the inverse RoPE into the sparse decode reduce on the DSv4.1 ROCm path. It merged on 2026-09-22, after nightly-rocm100-3df4ae15 was cut on 2026-09-21, so no point on the current curve carries it. The new pin is 176 commits ahead of the old one and carries the change." + - "在 ROCm 10.0 nightly nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423(发布于 2026-09-24T05:45:09Z,摘要 sha256:5df261803a5b4e8c4d6bbe58a32c9cd597140b6fdfa7358c0ef0d0211c33ab32)上重新扫描 MI355X DeepSeek-V4.1-Flash vLLM AgentX 臂,替换 nightly-rocm100-3df4ae15。本次仅更换镜像:TP=4 与 TP=2 两行保持并发 1、2、4、8、16、32、64 与 128,其余 recipe 设置全部不变,包括按 TP 与并发决定的 Engram 放置、prefill 分块与序列上限阶梯、图捕获,以及吞吐用黄金接受长度 3.51、评测用真实分块验证的五 token DSpark。" + - "此次固定主要是为了 vllm-project/vllm#57435:它在 DSv4.1 的 ROCm 路径上将 inverse RoPE 融合进 sparse decode 的 reduce。该 PR 于 2026-09-22 合并,晚于 2026-09-21 切出的 nightly-rocm100-3df4ae15,因此当前曲线上的任何数据点都不包含该改动。新镜像比旧镜像领先 176 个提交,已包含该改动。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3404 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Switch the MI355X vLLM DeepSeek-V4-Pro AgentX sweep from MTP K3 to DSpark K6, using DeepSeek-V4-Pro-0813 revision 72e1d3230f6c080a530b0a1d46f8eb4602340597, probabilistic drafting, thinking-on golden acceptance length 3.77 for throughput, and real target verification for eval." + - "Update the image to the latest published and locally tested vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19 while preserving the existing TP8/EP1, DP-attention, router, KV-offload, concurrency, and duration settings." + - "Local 8x MI355X DPA-TP8/EP1 concurrency-4 validation improved P90 interactivity from 59.72 to 97.36 tok/s/user and total throughput per GPU from 2,261.58 to 2,742.19 tok/s versus MTP K3, with zero request errors." + - "This PR does not change the data types of the DeepSeek-V4-Pro-0813 draft layers; it only touches the benchmark launcher and config to switch the speculative-decoding method from MTP K3 to DSpark K6, keeping the checkpoint's shipped FP4 draft precision and the vLLM image dtype defaults unchanged." + - "将 MI355X vLLM DeepSeek-V4-Pro AgentX 扫描从 MTP K3 切换为 DSpark K6,使用 DeepSeek-V4-Pro-0813 revision 72e1d3230f6c080a530b0a1d46f8eb4602340597、概率式 drafting、吞吐测试 thinking-on golden acceptance length 3.77,以及评估时的真实 target verification。" + - "将镜像更新至当前最新且已完成本机实测的 vllm/vllm-openai-rocm:nightly-e9757321527ca1ecd514c07c1418dd2c53da3d19,同时保持现有 TP8/EP1、DP attention、router、KV offload、并发列表和运行时长不变。" + - "本地 8x MI355X DPA-TP8/EP1 并发 4 验证中,相比 MTP K3,P90 交互速度从 59.72 提升至 97.36 tok/s/user,单卡总吞吐量从 2,261.58 提升至 2,742.19 tok/s,且无请求错误。" + - "本 PR 不改变 DeepSeek-V4-Pro-0813 draft 层的数据类型;仅修改基准测试启动脚本与配置,将投机解码方式从 MTP K3 切换为 DSpark K6,保持 checkpoint 自带的 FP4 draft 精度与 vLLM 镜像的 dtype 默认设置不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3406 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the GLM-5.2 MXFP4 MI355X SGLang AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260920 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 (digest sha256:59eeb83aea8fb5dff4a0c8964da2adf690af9494b2b3271ea13a54018adbaa39); keep all serving flags, HiCache settings, and sweep points unchanged." + - "将 GLM-5.2 MXFP4 MI355X SGLang AgentX 镜像从 lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260920 更新到 lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923(digest sha256:59eeb83aea8fb5dff4a0c8964da2adf690af9494b2b3271ea13a54018adbaa39);其余服务参数、HiCache 设置与 sweep 点保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3399 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark + scenario-type: + - agentic-coding + description: + - "Move all six points (c4, c16, c32, c48, c192, c256) onto the UMBP linker; c128 dropped. c4/c16 were no-offload and c32/48 were HiCache; c16 and c32/48 prefill go TP8 -> TP4. DSpark gamma 6 at c4/c16, 3 above." + - "Bump to image lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923." + - "UMBP tier on 2 MiB hugepages by default: job.slurm reserves them on prefill nodes and releases them on every exit path, hugepage_sanity.sh fails a short grow, and server_sglang.sh refuses a run that fell back to 4 KiB pages." + - "Allow pure-TP prefill on the linker, add --optimistic-prefill-attempts 2 on umbp-linker prefill, raise the UMBP client ready probe to 120 s, derive CONTAINER_BARRIER_TIMEOUT from UMBP_SA_WAIT_SECONDS + 600, and wire UMBP_DRAM_CEILING_GB through." + - "Add DeepSeek-V4-Pro-0813 golden AL for draft lengths 4, 5, 7 and 8 (3.36 / 3.61 / 3.73 / 3.47)." + - "Agentic PD router: pin --decode-policy round_robin so decode no longer inherits the prefill --policy (consistent_hashing)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3256 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 MXFP4 ATOM PD-disaggregated AgentX on MI355X (cluster:mi355x-amds, image rocm/atom-dev:nightly_202609221542), the ATOM counterpart to dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp; 1P1D TP8, mooncake RDMA KV transfer, atomesh PD router." + - "Search space: conc 1 and 16 on plain TP8 and conc 64 and 128 with DP attention, all four on MTP with GPU-resident KV at dram-utilization 0.80; plus conc 256 with DP attention, the DSpark draft model and the lmcache host KV tier (multi connector, max_pending_saves 8, slot_sidecar_staging_slots 4, LRU, 256-token chunks) at dram-utilization 0.50 with dp-sticky idx2idx routing. conc 1/16/64 keep the shape run 35643358891 used and change only the image." + - "conc-128 on this image (run 35810485934, 3627 s, 13917 of 15339 requests profiled, 0 errors): 464105 total tok/s (29007 tok/s/GPU over 16 GPUs), 3934 output tok/s (246 tok/s/GPU), TTFT mean 9.75 s / p50 8.70 / p90 15.89, TPOT mean 12.98 ms / p90 14.43, interactivity mean 77.0 / p90 69.3, e2e mean 23.01 s, 3.84 QPS, 96.6% theoretical prefix-cache hit rate." + - "conc-256 on this image (run 35825955863, 3630 s, 21317 of 24162 requests profiled, 0 errors): 776348 total tok/s (48522 tok/s/GPU), 5794 output tok/s (362 tok/s/GPU), TTFT mean 18.10 s / p50 12.62 / p90 32.78, TPOT mean 15.95 ms / p90 17.57, interactivity mean 62.7 / p90 56.9, e2e mean 33.89 s, 5.88 QPS, 96.8% theoretical prefix-cache hit rate. Not a clean scaling comparison against conc-128: this tier also swaps MTP for the DSpark draft model and adds the lmcache host KV tier." + - "Accuracy on the conc-128 DP-attention cell (run 35851118405, lm-eval gsm8k 5-shot, eval concurrency 128, n=1319): em_strict 0.9719 +/- 0.0045, em_flexible 0.9712 +/- 0.0046. conc 1, 16 and 64 are unmeasured on this image." + - "online_quant is not used on this arm: server_atom.sh only emits --online_quant_config when models_atom.yaml sets online_quant_config/online_quant_dpa_config (read at server_atom.sh:65-66, assembled at :156-172, injected into the prefill/decode server commands at :317/561/652), and the DeepSeek-V4-Pro-AgentX entry and its -0813-AgentX alias set neither, so ONLINE_QUANT_ARG stays empty and the flag is never passed. The served DeepSeek-V4-Pro-0813 checkpoint is already offline MXFP4, so runtime quantization is unnecessary (online_quant targets BF16/unquantized checkpoints, e.g. the ptpc_fp8 configs in deprecated/models_atom.yaml). DSpark accuracy is therefore not touched by online_quant: no target layer is re-quantized at runtime, so acceptance length and the gsm8k numbers above reflect the offline MXFP4 target plus the DSpark draft head only." + - "server_atom.sh exports PORT before sourcing benchmark_lib.sh on the eval path. run_lm_eval's check_env_vars guard runs before it parses --port and job.slurm's docker -e allowlist forwards ROUTER_PORT but never PORT, so every multi-node lm-eval cell aborted 8 s in; because check_env_vars exits rather than returning, node 0 also skipped its router teardown and the decode node hung in 'Waiting until router closes...' until the job was cancelled (run 35643358891)." + - "Per ATOM DeepSeek-V4-Agentic-PD-Max recipe: prefix caching on, FP8 KV and index cache, block-size 256, TBO on prefill only, max-num-seqs = 2x concurrency." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3158 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + description: + - "Update SGLang ROCm image from v0.5.18-rocm720-mi35x-20260828 to v0.5.20-rocm720-mi35x-20260924 (latest nightly)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3423 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Re-sweep the MI355X DeepSeek-V4.1-Flash vLLM AgentX arm on the next ROCm 10.0 nightly, replacing nightly-rocm100-7f1a5398. Only the image changes: the TP=4 and TP=2 rows keep concurrency 1, 2, 4, 8, 16, 32, 64 and 128, and every recipe setting is unchanged, including Engram placement per TP and concurrency, the prefill chunk and sequence cap ladder, graph capture, and five-token DSpark with golden acceptance 3.51 for throughput and real block verification for evals." + - "The pin is mainly for vllm-project/vllm#58510, which computes the MXFP8 GEMM on native 32x32 block scales for gfx950. This arm serves a checkpoint whose non-routed weights are MXFP8 on exactly that architecture, so the target image is the first nightly whose commit contains that merge rather than simply the next one published. The exact tag, its publication time and digest are filled in once that build exists." + - "在下一个 ROCm 10.0 nightly 上重新扫描 MI355X DeepSeek-V4.1-Flash vLLM AgentX 臂,替换 nightly-rocm100-7f1a5398。本次仅更换镜像:TP=4 与 TP=2 两行保持并发 1、2、4、8、16、32、64 与 128,其余 recipe 设置全部不变,包括按 TP 与并发决定的 Engram 放置、prefill 分块与序列上限阶梯、图捕获,以及吞吐用黄金接受长度 3.51、评测用真实分块验证的五 token DSpark。" + - "此次固定主要是为了 vllm-project/vllm#58510:它在 gfx950 上使用原生 32x32 block scale 计算 MXFP8 GEMM。本臂所服务的 checkpoint 其非 routed 权重正是该架构上的 MXFP8,因此目标镜像是第一个其提交包含该合并的 nightly,而非简单地取下一个发布的 nightly。具体标签、发布时间与摘要将在该构建出现后补充。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3420 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4.1-Flash FP4 ATOM AgentX on MI355X following the upstream TP2 / TP4 recipe: TP2 concurrency [1, 2, 8, 16, 32, 64], TP4 concurrency [2, 8, 16, 32, 64], no expert parallelism or KV offload." + - "Use rocm/atom-dev:nightly_202609250902 (digest sha256:9aaeed59d78decfc51eaabda072ba095c1d64d6cfbdb64ecb28ea74cf3deb18a), BF16 KV, FP8 index cache, max-num-seqs 128, 16K batching/prefill chunks, prefix caching with block size 16, 8K state checkpoints, level 3 and FULL graphs with dense capture sizes through 32 at concurrency 32." + - "Use five-token DSpark with thinking_on golden AL 3.51 for throughput and real acceptance for eval, the shipped draft, and dsml_v41 tool parsing. Replay semianalysis_cc_traces_weka_062126 for 3600 seconds per point with five warmup requests per lane." + - "DSpark draft layers 37-39 keep their checkpoint precision: the recipe passes no --online_quant_config, so online_quant_config resolves to None and make_v4_quant_config builds the same per-layer quant spec for the drafter as for the target. The MTP weights (mtp.0/1/2) load inline from the same checkpoint with spec_decode=True, and config.get_layer_quant_config has no mtp.* special case, so layers 37-39 are neither re-quantized nor up/down-cast by ATOM and retain their on-disk dtype. --kv_cache_dtype bf16 and --index-cache-dtype fp8 touch cache storage only, not draft/MTP weights." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3387 + +- config-keys: + - glm5.2-fp8-mi325x-sglang-agentic-mtp + description: + - "Update SGLang ROCm image to v0.5.20-rocm720-mi30x; replace retired --cuda-graph-max-bs with --cuda-graph-max-bs-decode." + - "将 SGLang ROCm 镜像更新至 v0.5.20-rocm720-mi30x,并以 --cuda-graph-max-bs-decode 替换已移除的 --cuda-graph-max-bs。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3362 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + description: + - "Update the vLLM ROCm image from v0.29.0 to v0.30.0; benchmark configuration unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3361 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + description: + - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the Kimi-K3 MI355X vLLM AgentX image from nightly-rocm100-af1c0149 to nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423 (published 2026-09-24T05:45:09Z, digest sha256:5df261803a5b4e8c4d6bbe58a32c9cd597140b6fdfa7358c0ef0d0211c33ab32) and re-measure the arm. Only the image changes: the TP=8 rows keep concurrency 1 and 4 GPU-resident, 8, 10, 12 and 14 on vllm-simple DRAM offload, and 44, 48 and 70 on TP8/DCP8 with vllm-simple, and benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh is unchanged." + - "The old pin dates from 2026-09-16 and the new one is 441 commits ahead. It carries three Kimi-K3 correctness fixes that the current curve predates: vllm-project/vllm#51483 stops a stateless first chunk being classified as a decode, vllm-project/vllm#57098 fixes the Kimi-K3 reasoning parser, and vllm-project/vllm#57430 adds routed-expert quantization support. It also carries ROCm CPU KV-offload work that this arm exercises at every concurrency above 4, including vllm-project/vllm#57160 private pinned tensors for ROCm CPU KV offload and vllm-project/vllm#50045 offload back-pressure detection." + - "将 Kimi-K3 MI355X vLLM AgentX 镜像从 nightly-rocm100-af1c0149 刷新至 nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423(发布于 2026-09-24T05:45:09Z,摘要 sha256:5df261803a5b4e8c4d6bbe58a32c9cd597140b6fdfa7358c0ef0d0211c33ab32)并重新测量该臂。本次仅更换镜像:TP=8 各行保持并发 1 与 4 的 GPU 常驻 KV,8、10、12、14 的 vllm-simple DRAM 卸载,以及 44、48、70 的 TP8/DCP8 加 vllm-simple;benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh 未作改动。" + - "旧镜像切出于 2026-09-16,新镜像领先 441 个提交,包含当前曲线所不含的三项 Kimi-K3 正确性修复:vllm-project/vllm#51483 不再将无状态的首个 chunk 当作 decode,vllm-project/vllm#57098 修复 Kimi-K3 reasoning parser,vllm-project/vllm#57430 支持 routed expert 量化。新镜像还包含本臂在并发大于 4 时均会使用的 ROCm CPU KV 卸载改动,包括 vllm-project/vllm#57160(ROCm CPU KV 卸载改用私有 pinned 张量)与 vllm-project/vllm#50045(卸载背压检测)。" + - "This image bump does not change the DSpark draft model data type. Only the image: line changes and kimik3_fp4_mi355x_mtp.sh is unchanged; the draft loads unmodified from the published Inferact/Kimi-K3-DSpark checkpoint via --speculative-config (model=Inferact/Kimi-K3-DSpark, method=dspark). The only dtype in that speculative-config is kv_cache_dtype=fp8, which sets the draft KV-cache storage precision, not the draft weights. No flag overrides or re-quantizes the draft-model weights, so the draft dtype is preserved from its checkpoint across this re-sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3419 + +- config-keys: + - minimaxm3-fp8-h200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the MiniMax-M3 H200 vLLM AgentX image from nightly-d9105ea8 (2026-09-07) to nightly-ddd6fbca148a867aad1fcab7ec72f582b9977db4 (vllm-project/vllm@ddd6fbca, 2026-09-26) and re-measure the arm." + - "Nightlies from 2026-09-15 on lazy-mmap large checkpoints on h200-dgxc's VIRTIOFS HF cache ~14x slower and miss the 3600 s readiness window (seen on #3134). Like the DeepSeek-V4.1-Flash H200 recipe, the server now uses --safetensors-load-strategy prefetch with VLLM_ENGINE_READY_TIMEOUT_S=7200 and a 7200 s health-check window. Hopper runs FA3, so the Blackwell FA4 fp8-descale issue does not apply and the EAGLE3 draft keeps attention_backend FLASH_ATTN." + - "将 MiniMax-M3 H200 vLLM AgentX 镜像从 nightly-d9105ea8(2026-09-07)刷新至 nightly-ddd6fbca148a867aad1fcab7ec72f582b9977db4(vllm-project/vllm@ddd6fbca,2026-09-26)并重新测量该臂。" + - "自 2026-09-15 起的 nightly 在 h200-dgxc 的 VIRTIOFS HF 缓存上以 lazy mmap 加载大 checkpoint 慢约 14 倍,会超出 3600 秒就绪窗口(见 #3134)。与 DeepSeek-V4.1-Flash H200 配方一致,服务端改用 --safetensors-load-strategy prefetch,并设置 VLLM_ENGINE_READY_TIMEOUT_S=7200 与 7200 秒健康检查窗口。Hopper 使用 FA3,Blackwell FA4 fp8 descale 问题不适用,EAGLE3 draft 保持 attention_backend FLASH_ATTN。" + - "This change does not alter the EAGLE3 draft model data type. The draft loads unmodified from the published Inferact/MiniMax-M3-EAGLE3-GQA checkpoint via --speculative-config (method=eagle3). kv-cache-dtype fp8 sets KV-cache storage precision, not the draft weights, and no flag overrides or re-quantizes the draft weights." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3493 + +- config-keys: + - minimaxm3-fp8-h100-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Move the MiniMax-M3 H100 vLLM AgentX image from the v0.27.1 release to nightly-ddd6fbca148a867aad1fcab7ec72f582b9977db4 (vllm-project/vllm@ddd6fbca, 2026-09-26, the newest nightly) and re-measure the arm. Only the image changes; the recipe already carries VLLM_ENGINE_READY_TIMEOUT_S=7200 and a 7200 s health-check window, and keeps lazy safetensors loading with 26 GiB of CPU weight offload per GPU. The TP8 rows keep concurrency 1-5 GPU-resident and 6, 8 on Mooncake DRAM offload." + - "Hopper runs FA3, so the Blackwell FA4 fp8-descale issue does not apply and the EAGLE3 draft keeps attention_backend FLASH_ATTN." + - "将 MiniMax-M3 H100 vLLM AgentX 镜像从 v0.27.1 正式版切换至最新 nightly-ddd6fbca148a867aad1fcab7ec72f582b9977db4(vllm-project/vllm@ddd6fbca,2026-09-26)并重新测量该臂。仅更换镜像;配方已设置 VLLM_ENGINE_READY_TIMEOUT_S=7200 与 7200 秒健康检查窗口,并保持 lazy safetensors 加载与每 GPU 26 GiB CPU 权重卸载。TP8 各行保持并发 1-5 GPU 常驻,6、8 使用 Mooncake DRAM 卸载。" + - "Hopper 使用 FA3,Blackwell FA4 fp8 descale 问题不适用,EAGLE3 draft 保持 attention_backend FLASH_ATTN。" + - "This change does not alter the EAGLE3 draft model data type. The draft loads unmodified from the published Inferact/MiniMax-M3-EAGLE3-GQA checkpoint via --speculative-config (method=eagle3). kv-cache-dtype fp8 sets KV-cache storage precision, not the draft weights, and no flag overrides or re-quantizes the draft weights." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3503 + +- config-keys: + - minimaxm3-fp8-h200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Validate MiniMax-M3 AgentX on H200 with vLLM FP8 after moving the end-to-end project into inferencex-e2e/. Preserve the existing image, recipe, and benchmark settings." + - "将端到端项目迁入 inferencex-e2e/ 后,验证 H200 上的 vLLM FP8 MiniMax-M3 AgentX 运行路径,沿用现有镜像、配方和基准测试设置。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3525 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update B300 vLLM AgentX to DSpark6 on a new image with a sampled concurrency grid and per-mode --kv-cache-memory-bytes pins." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3477 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update B200 AgentX to vLLM nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b and enable local argmax reduction for the FlashInfer EAGLE3 draft." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the GLM-5.2 MXFP4 MI355X SGLang AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 (digest sha256:baadaba198e23c46c1dd651b5edefae8000bd5430943bd10b246b563905b3ecf); keep all serving flags, HiCache settings, and sweep points unchanged." + - "将 GLM-5.2 MXFP4 MI355X SGLang AgentX 镜像从 lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 更新到 lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924(digest sha256:baadaba198e23c46c1dd651b5edefae8000bd5430943bd10b246b563905b3ecf);其余服务参数、HiCache 设置与 sweep 点保持不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3446 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + description: + - "Pin B300 DeepSeek-V4.1-Flash vLLM to nightly ddd6fbca with FlashInfer sparse attention, MXFP4 indexer KV, sparse logits, fp8 KV cache, and a 7200 s readiness timeout." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3458 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + description: + - "Pin B200 DeepSeek-V4.1-Flash vLLM to nightly ddd6fbca with FlashInfer sparse attention, MXFP4 indexer KV, sparse logits, fp8 KV cache, and a 7200 s readiness timeout." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3459 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926." + - "Switch attention to fp8 unified KV; enable grouped decode on TP8-only arms at conc >= 16." + - "Enable breakable CUDA-graph prefill (BCG) at conc <= 4." + - "Enable EP8 + MegaMoE for DP-attention arms at conc >= 128 (including conc 384)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3430 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-eagle + scenario-type: + - agentic-coding + description: + - "Add B300 DeepSeek-V4-Pro SGLang AgentX with EAGLE (3 steps, 4 draft tokens, golden AL 2.49) on lmsysorg/sglang:dev-cu13-nightly-09242: TP4 and TP8 at c1-32 with GPU KV, and DEP8 with a HiCache DRAM tier (ratio 2) at c64-512." + - "DEP8 uses the dsv4 attention backend, mem-fraction-static 0.84, prefill-decode-interval 32, mixed chunking, shortest-prefill-first scheduling, DP speculative prefill coordination and a router with the circuit breaker disabled, without the DP LM head." + - "Runs on the native srt-slurm recipe benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3426 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the GB300 DeepSeek-V4.1-Flash SGLang AgentX curve to lmsysorg/sglang:dev-cu13-nightly-0924 (digest pinned): pure TP4 (EP1) at C1/C2 with Engram in HBM, and TP4/EP4 at C4+ with per-rank host Engram, 16K prefill chunks, no prefill-decode interval, 4096 SWA prefix tails and a 128 decode graph batch; all TP4 points use static ragged verify; TP2 is unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3421 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update B300 AgentX to vLLM nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b with local argmax reduction." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3547 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "All points now run vllm/vllm-openai-rocm:v0.30.0 (previously v0.29.0)." + - "LMCache DRAM point (TP8, concurrency 16): one LMCache server per node with 1296 GB L1 shared by all 8 TP ranks, replacing 8 per-rank servers of 162 GB L1 each." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3545 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 (Docker Hub digest sha256:a2b277dff0be18113d4eb013b607aa281b6020c3fe3e240770eebe9935b2015f) in configs/amd-master.yaml and the srt-slurm AgentX recipe." + - "Move TP2 concurrency 16 from GPU-resident KV to HiCache and remove the TP2 GPU-resident concurrency-20 point. Keep TP4 GPU-resident [1, 4, 8, 12, 16]. The resulting TP2 grids are resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]." + - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio) on every TP2 HiCache point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3558 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 (Docker Hub digest sha256:a2b277dff0be18113d4eb013b607aa281b6020c3fe3e240770eebe9935b2015f) for the fixed 8k1k arms. Serving flags and sweep points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3559 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp + scenario-type: + - agentic-coding + description: + - "Enable the MiniMax-M3 reasoning parser on the Dynamo frontend so reasoning blocks are classified consistently instead of being returned as ordinary content." + - "Set reasoning_parser: minimax_m3 in each TensorRT-LLM engine configuration to keep the frontend and engine parser settings aligned." + - "在 Dynamo 前端启用 MiniMax-M3 推理解析器,使推理内容按一致方式分类,而不会作为普通内容返回。" + - "在每个 TensorRT-LLM 引擎配置中设置 reasoning_parser: minimax_m3,使前端与引擎的解析器设置保持一致。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3195 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:nightly_202609251613, tracking recipes/Agentic-Kimi-K3.md as retuned in ROCm/ATOM#2382." + - "Enable FlyDSL FP8 prefill attention (ATOM_USE_FLYDSL_FP8_PREFILL_ATTN=1) on every concurrency; ATOM defaults it off." + - "Hold a ready prefill for four decode passes from concurrency 16 up (ATOM_PREFILL_DECODE_INTERVAL=4, ATOM_PREFILL_DELAYER_MAX_QUEUE_MS=5000). Concurrency 1, 4 and 14 are unchanged." + - "Restore the DCP8 LMCache bands on the native srt-slurm recipe: concurrency 14 and 16 (DSpark 3, ReplaySSM) and 48, 56 and 72 (no draft) run ATOM's in-process lmcache_offload connector through roles.agg.args.extra-kv-connectors (srt-slurm patch 507), with 128 GB/rank up to 48 and 192 GB/rank at 56 and 72. Concurrency 1 and 4 stay GPU-resident." + - "No change to the Inferact/Kimi-K3-DSpark draft's precision: online_quant_config still excludes every draft linear (layers.*, context_proj), so its weights and activations stay BF16, and it keeps the target's FP8 KV cache (kv_cache_dtype fp8). FlyDSL FP8 prefill attention applies only to the target, since the draft runs its block pass as decode attention." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3407 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + description: + - "Add Kimi-K3 MXFP4 vLLM agentic-coding on MI355X (TP8, DSpark): dcp1 c1/c4 GPU-resident and c8-c14 with SimpleCPUOffload DRAM offload, plus a new dcp8 c44/c48/c70 throughput band (mtp synthetic acceptance, no draft); vLLM ROCm image vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89" + - "The Inferact/Kimi-K3-DSpark draft (K3DSparkModel, model_type k3_dspark, 5 layers, hidden 7168, torch_dtype bfloat16, no quantization_config) keeps every layer at its pristine dtype. vLLM builds all its modules with quant_config from get_draft_quant_config() (vllm/models/kimi_k3/nvidia/dspark_mla.py), which returns None because K3DSparkModel is explicitly excluded from the DeepSeek-V4 branch that would set draft.quantization = target.quantization (vllm/config/speculative.py:1431); so context_proj (ReplicatedLinear), context_kv_proj (MergedColumnParallelLinear) and each decoder layer's MLA q/kv projections and dense KimiMLP (gate/up/down) load unquantized in BF16 -- no ptpc_fp8, no mxfp4, no INT4 weights. The draft has no FusedMoE/block_sparse_moe, so VLLM_ROCM_USE_AITER_MOE_SITUV2 (A8W4) never touches it, and its KV cache stays fp8 (kv_cache_dtype). This PR only changes draft depth (num_speculative_tokens 4->7) and enables INT4 custom quick all-reduce (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, vllm/distributed/device_communicators/quick_all_reduce.py); because the draft shares the target's TP8 group (vllm/v1/spec_decode/draft_model.py), that INT4 applies to its tensor-parallel reductions too -- a collective-reduction transport precision, not any draft weight, activation, or KV dtype." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3561 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202609171455 to nightly_202609281543 and enable FlyDSL paged decode (ATOM_PA_FLYDSL=1, ATOM_PA_FLYDSL_PLAN=1)." + - "Restore the LMCache DRAM-offload points: TP2 concurrency 20, 25 and 30, and TP4 concurrency 40 and 48 (in-process lmcache_offload, 257 GB CPU tier per rank)." + - "Set ATOM_MONO_ENABLE=0 on TP4 concurrency 15, 20, 24 and 28; the new image enables mono decode by default, which hangs these indexer-CP points." + - "Drop TP4 concurrency 32." + - "No data-type or precision change to the EAGLE3 draft model (Inferact/MiniMax-M3-EAGLE3-GQA): online_quant_config (ptpc_fp8, which ATOM also applies to the draft), kv_cache_dtype fp8 and index-cache-dtype fp8 are unchanged; the FlyDSL, mono-decode and lmcache_offload toggles affect the decode path and KV offload, not draft precision." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3388 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B-FP8 AgentX on MI355X with SGLang image lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 and a new srt-slurm recipe. TP4 has five GPU-resident points (c1, c4, c8, c12, c14) and nine HiCache DRAM points (c16, c18, c20, c22, c24, c28, c32, c36, c40)." + - "Use AITER attention and allreduce fusion, FP8 KV (fp8_e4m3), and EAGLE MTP (3 steps, topk 1, 4 draft tokens); do not enable ROCm INT4 quick all-reduce. HiCache uses ratio 1.5, write_through_selective, kernel I/O and page_first layout. The matching cookbook recipe is sgl-project/sglang#41849." + - "The embedded MTP head runs at its stored precision: block-FP8 expert weights and checkpoint dtype for mtp.fc and gates. No separate draft, draft dtype override or submission-side quantization is used." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3602 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark + scenario-type: + - agentic-coding + description: + - "Port the DeepSeek-V4-Pro-0813 MI355X ATOM 1P1D AgentX config from the legacy amd_utils path to a native srt-slurm recipe (benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml), one override variant per point. Server flags, env, AToMesh routing policies and decode CUDA-graph capture sizes match the legacy server_atom.sh for every tier: TP8 at concurrency 1 and 16, DP attention with prefill TBO at 64 and 128, and DP attention with ATOM's in-process LMCache CPU offload (lmcache_offload, 187 GB per prefill rank) at 256. Golden acceptance 3.01 for DSpark with three draft tokens is now injected by the srt-slurm path, the same value the legacy models_atom.yaml hardcoded." + - "The LMCache tier uses srt-slurm's extra-kv-connectors (NVIDIA/srt-slurm#507, in v2.36.0) to add lmcache_offload next to the generated Mooncake connector." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3543 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the ATOM image from rocm/atom-dev:nightly_202609161445 (ATOM 0.1.6rc1.dev477+g709c25be0) to rocm/atom-dev:nightly_202609291501 (digest sha256:e0d7134efff68c251acdc2612a9b6c238bc4da5d613832b2bd1251f34bd93355; ATOM 0.1.7.dev46+g74fd942b0, ROCm 7.2.4). Image-only change on the existing native srt-slurm recipe: all ten points (TP8/EP1 at concurrency 1, 2, 4, 8, 16; TP8/DPA8/EP8 native RCCL at 48, 64, 96, 128, 256), DSpark K6 with the thinking_on golden AL 3.77 for throughput and real acceptance for eval, KV and index-cache dtypes, graph capture and all environment settings are unchanged." + - "Relevant ATOM changes in the range: V4 decode reuses the sparse prefill ASM (ROCm/ATOM#2271), greedy sampler picks via aiter.topk_select (ROCm/ATOM#2244), and FP8 block scales declared scale_fmt ue8m0 are now stored as E8M0 on gfx950 by default (ROCm/ATOM#2419; previously FP32 unless ATOM_FP8_BLOCKSCALE_USE_E8M0_SCALE=1). The upstream DeepSeek-V4 recipes are unchanged across the range, and every recipe flag and choice (all2all-backend rccl, dp-load-balance least_tokens, moe-backend standard) remains valid." + - "No data-type or precision change to the DeepSeek-V4-Pro-0813 DSpark draft: no online quantization is configured, so it keeps its checkpoint precision; the E8M0 scale storage represents the checkpoint's power-of-two block scales exactly. kv-cache-dtype and index-cache-dtype touch cache storage only." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3605 + +- config-keys: + - glm5.3-fp8-mi355x-tilert-agentic + scenario-type: + - agentic-coding + description: + - "Bump tilert 0.1.6.post2 -> 0.1.6.post3 (PyPI, 2026-09-28) on both TileRT ranks; router metadata follows. post3 turns on three prefill-to-decode latency features by default: multi-sender staging (TILERT_PD_SENDERS=8, each TP rank extracts and sends its own layers), per-layer pipelined KV send (TILERT_PD_PIPELINE=1, a layer is extracted and RDMA-written as soon as it is computed), and router-side incremental chat tokenization (TILERT_ROUTER_TOKENIZE=1, the router tokenizes only the new turn through a per-segment cache and hands vLLM token ids; a start-up self-check against vLLM /tokenize keeps it off on any mismatch). Move the prefill rank to a vLLM 0.28 + ATOM image (ghcr.io/tile-ai/tilert-rocm-prefill:0.1.6.post1, built on rocm/atom-dev:vllm-v0.28.0-nightly_20260923) with CUDA graphs, async scheduling, prefix caching and the GLM-5.2 ATOM MI355X agentic recipe's AITER settings. Decode image, 1M context, bf16 KV and layer-sharded PD buffers unchanged. TileRT reports AgentX TTFT p90 4.35 s -> 1.60 s at concurrency 1 (local 2x8 MI350X against the post2 MI355X run)." + - "Set MOONCAKE_DISABLE_HIP_DMABUF=0 on the prefill rank: the new prefill image bakes in =1, which forces Mooncake onto plain ibv_reg_mr for the VRAM staging buffer and fails registration with EINVAL (-202) on every rank." + - "Pin the prefill KV cache to 100 GiB (kv-cache-memory, ~1.14M tokens): with gpu-memory-utilization alone vLLM fills the budget before TileRT allocates its 12.54 GiB staging buffer and ATOM leaves 5.3 GiB of CUDA graphs unprofiled, so the sampler warmup ran out of memory." + - "两侧 TileRT rank 上 tilert 由 0.1.6.post2 升级到 0.1.6.post3(PyPI,2026-09-28),router 元数据随之更新。post3 默认打开三项 prefill 到 decode 的时延优化:多发送端暂存(TILERT_PD_SENDERS=8,每个 TP rank 抽取并发送自己负责的层)、逐层流水线发送 KV(TILERT_PD_PIPELINE=1,每层算完即抽取并 RDMA 写出)、router 侧增量对话分词(TILERT_ROUTER_TOKENIZE=1,router 经按段缓存只对新一轮分词,把 token id 交给 vLLM;启动自检与 vLLM /tokenize 不一致即保持关闭)。prefill rank 改用 vLLM 0.28 + ATOM 镜像(ghcr.io/tile-ai/tilert-rocm-prefill:0.1.6.post1,基于 rocm/atom-dev:vllm-v0.28.0-nightly_20260923),开启 CUDA graph、异步调度、prefix caching,并沿用 GLM-5.2 ATOM MI355X agentic 配方的 AITER 设置。decode 镜像、1M 上下文、bf16 KV 与按层分片的 PD 缓冲不变。TileRT 报告并发 1 下 AgentX TTFT p90 由 4.35 秒降至 1.60 秒(本地 2x8 MI350X 对比 post2 的 MI355X 官方轮次)。" + - "prefill rank 设置 MOONCAKE_DISABLE_HIP_DMABUF=0:新 prefill 镜像内置该变量为 1,迫使 Mooncake 对显存暂存缓冲改走普通 ibv_reg_mr,所有 rank 注册均以 EINVAL(-202)失败。" + - "prefill KV cache 固定为 100 GiB(kv-cache-memory,约 114 万 token):仅用 gpu-memory-utilization 时,vLLM 在 TileRT 分配 12.54 GiB 暂存缓冲之前就占满预算,且 ATOM 未计入 5.3 GiB 的 CUDA graph 显存,导致采样器预热时显存不足。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3623 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the TP4 HiCache band from concurrency 40 up to 80, adding concurrency 48, 56, 64, 72 and 80. The arm goes from 14 to 19 points; the GPU-resident band [1, 4, 8, 12, 14] and the existing HiCache points [16, 18, 20, 22, 24, 28, 32, 36, 40] are unchanged. The original grid was copied from the TP4 band of qwen3.5-fp8-b200-sglang-agentic-mtp, which stops at 32 because a B200 node has far less HBM per GPU; MI355X at 288 GB per GPU has the device KV headroom to keep feeding the host-DRAM tier past that, and these five points sample the throughput end the tier makes available." + - "Size the HiCache host pool in tiers by concurrency instead of by ratio alone. Concurrency 16 through 28 keep ratio-only sizing and are unchanged from what merged in #3602. Concurrency 32 through 56 pin hicache-size 253, the value already used by the MI355X MXFP4 AgentX arm. Concurrency 64 through 80 pin hicache-size 400, since the largest points need the deepest host tier to hold prefix across the 256k-capped corpus. hicache-size overrides hicache-ratio where it is set. Of the points that merged in #3602 only concurrency 32, 36 and 40 change behaviour; their numbers are expected to move." + - "The five new points reuse the existing HiCache override shape: enable-hierarchical-cache, hicache-ratio 1.5, hicache-size 400, hicache-write-policy write_through_selective, hicache-io-backend kernel and hicache-mem-layout page_first. Admission continues to follow 2x CONC, and the decode graph batch follows it capped at 128, so concurrency 72 and 80 both clamp cuda-graph-max-bs-decode to 128 while max-running-requests reaches 144 and 160." + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260927 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 on this arm only. The other MI355X arms are left on their current tags." + - "No serving flag outside the HiCache block changes, and no other config key is touched. Each of the 19 matrix points resolves to exactly one recipe override, with no override left unused." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3611 + +- config-keys: + - dsr1-fp8-mi325x-sglang + - dsr1-fp8-mi325x-sglang-mtp + - qwen3.5-fp8-mi325x-sglang + - qwen3.5-fp8-mi325x-sglang-mtp + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + - glm5.2-fp8-mi325x-sglang-agentic-mtp + - minimaxm3-fp8-mi325x-vllm-agentic-mtp + - dsv41flash-fp4-mi325x-vllm-agentic-dspark + description: + - "Retire the mi325x-amds runner pool and route MI325X workloads to the Barite mi325x-amd pool in MI325X-UBUNTU. Keep model recipes unchanged; generated host-DRAM budgets now use the replacement cluster's declared memory capacity." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3644 + +- config-keys: + - "*" + scenario-type: + - agentic-coding + description: + - "Bump AgentX Harness from agentx-v1.0.5 (754356e9a39acc6cc6afb242d123bb57c3fb6f75) to agentx-v1.0.6 (89b21867872a5bbc4b0676bf5005c404da5e9f94) and use --scenario agentx for shared replay and runtime defaults. Preserve recipe duration, warmup, failure threshold, trace cap, routing and corpus selection as explicit inputs." + - "The preset supplies a 900000 ms TCP user timeout when unset; explicit AIPERF environment settings take precedence over preset defaults." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3646 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Repin to vllm/vllm-openai-rocm:nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516, which includes vllm-project/vllm#58671, vllm-project/vllm#58655 and vllm-project/vllm#53492." + - "Set VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA=True to enable the vllm-project/vllm#53492 gfx950 Gluon sparse-MLA kernel." + - "Add --attention-config '{\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}' and --block-size 128 to enable the vllm-project/vllm#58671 ROCm paged MXFP4 sparse-logits indexer, replacing the dense fp8 indexer path. A live A/B test (TP2 c16, matched 900s window, vllm-project/vllm#58208 reverted via vllm-project/vllm#59125 so the dense fallback doesn't crash) measured +14.7/+14.9% p50/p90 interactivity and -9.5/-10.8% p50/p90 e2e latency over the dense path, with throughput/GPU unchanged." + - "Drop c128 from both TP2 and TP4. Neither c128 point was on the Pareto frontier in #3555's run 36528242520: TP2 c64 dominated both (P90 E2EL 58 s against 185 s and 85 s, at 111k against 85k and 79k total tok/s/GPU)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3571 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update Kimi-K3 B200 AgentX to vllm/vllm-openai:nightly-ac9126e58aa7bbab1856ba6593ba4d5003fea516, set bfloat16 KDA state, and add concurrency 32/56/72/80 with TP8/PP2/DCP8 and DSpark K4." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3087 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + description: + - "Pin GB300 DeepSeek-V4.1-Flash vLLM to nightly ac68c308 with FlashInfer sparse attention, MXFP4 indexer KV, sparse logits, fp8 KV cache, and B300's batching and CUDA graph tiers; use FlashInfer CUTLASS for MXFP8 GEMMs to avoid the CuTe-DSL split-K startup failure." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3396 + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang-agentic-agg + - dsv4-fp4-gb200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add DSV4-Pro GB200 Dynamo+SGLang AgentX: TP8 aggregate C1/C4, 1P1D DEP8/DEP16 C64/C128, 1P1D DEP16/DEP32 C256, and 2P1D DEP16/DEP32 C768/C1024/C1280." + - "Use SGLang nightly-dev-20260916-c9a8fba9, DSpark block size 6, and HiCache; omit enable-w4a4-mxfp4-megamoe while retaining the MegaMoE all-to-all backend and FP4 indexer." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3629 + +- config-keys: + - glm5.2-fp4-b200-dynamo-sglang-agentic-agg + - glm5.2-fp4-b200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Disable FP8 conversion of the GLM-5.2 NextN/MTP draft MoE on B200 AgentX so the draft runs at its shipped precision; images, topology, workload and acceptance are unchanged." + - "关闭 B200 GLM-5.2 AgentX 对 NextN/MTP draft MoE 的 FP8 转换,使 draft 保持原始发布精度;镜像、拓扑、工作负载和验收不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3400 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Disable FP8 conversion of the GLM-5.2 NextN/MTP draft MoE on GB300 AgentX so the draft runs at its shipped precision; images, topology, workload and acceptance are unchanged." + - "Disaggregated prefill and decode use the 1800 s scheduler watchdog of the GLM-5.2 B200/GB200 recipes instead of the 300 s default." + - "关闭 GB300 GLM-5.2 AgentX 对 NextN/MTP draft MoE 的 FP8 转换,使 draft 保持原始发布精度;镜像、拓扑、工作负载和验收不变。" + - "分离式 prefill 和 decode 采用 GLM-5.2 B200/GB200 配方的 1800 秒调度 watchdog,替代默认 300 秒。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3402 + +- config-keys: + - dsv41flash-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the B300 DeepSeek-V4.1-Flash vLLM AgentX image from nightly ddd6fbca to nightly-dev-x86_64-cu130-ac9126e58aa7 and enable FlashInfer autotuning." + - "Run TP4 at concurrency 1-16 and replace TP2 with DEP2 (TP1 x DP2 + EP2, DeepGEMM MegaMoE) at concurrency 8-192 behind a consistent-hash vLLM Router." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3653 + +- config-keys: + - dsv41flash-fp4-gb300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the GB300 DeepSeek-V4.1-Flash vLLM AgentX image from nightly ac68c308 to nightly-dev-arm64-cu130-ac9126e58aa7 and enable FlashInfer autotuning." + - "Run TP4 at concurrency 1-16 and replace TP2 with DEP2 (TP1 x DP2 + EP2, DeepGEMM MegaMoE) at concurrency 8-192 behind a consistent-hash vLLM Router." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3652 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the GLM-5.2 MXFP4 MI355X SGLang AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260930 (digest sha256:ac30a0fe98fb3f6e258d302280110f1be44c83edc54b914037db57c82cc20828). The image is built from SGLang commit d26ec4690d10d4c69890280b313d0409c8265feb, which contains the merge commit for sgl-project/sglang#38583." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3616 + +- config-keys: + - dsv41flash-fp4-b200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the B200 DeepSeek-V4.1-Flash vLLM AgentX image from nightly ddd6fbca to nightly-dev-x86_64-cu130-ac9126e58aa7 and enable FlashInfer autotuning." + - "Run TP4 at concurrency 1-128 and replace TP2 with DEP2 (TP1 x DP2 + EP2, DeepGEMM MegaMoE) at concurrency 8-32 and DEP4 (TP1 x DP4 + EP4, MegaMoE) at concurrency 64-128, both behind a consistent-hash vLLM Router. All points set gpu-memory-utilization 0.97." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3686 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark + scenario-type: + - agentic-coding + description: + - "Add c384 and c512 points to recipes/dsv4/sglang/mi355x-fp4/agentx/disagg-umbp-dspark.yaml and move the existing c192/c256 points onto the same shape, so the DP-attention arm is uniform from c192 to c512. Prefill runs Aiter MegaMoEv2 (sgl-project/sglang#35619) at DEP8 with --moe-a2a-backend megamoe, --moe-dense-tp-size 1, --ep-size 8 and SGLANG_AMD_USE_FLYDSL_MEGA_MOE=1; decode stays on the EP1 baseline MoE path, so the roles differ only in the MoE backend and the memory the mori heap takes. amd-master.yaml tracks prefill ep 1 -> 8 on c192/c256." + - "Set SGLANG_DSV4_UNIFIED_KV_FP8=1 on both roles of c192, c256, c384 and c512, splitting the single bf16 DeepSeekV4UnifiedKVPool into parallel nope-fp8 and rope-bf16 pools on the unified_kv_triton kernels the recipe already selects. Both PD peers carry the same value because the handshake rejects disagreeing KV layouts. The DSpark draft worker stays bf16." + - "Override MORI_SHMEM_MODE=STATIC_HEAP with an 8G MORI_SHMEM_HEAP_SIZE on the MegaMoE prefill role: the recipe base uses ISOLATION, which builds no symmetric heap, and MegaMoEV2 addresses its dispatch and combine buffers through one. The heap is hipMalloc'd in full outside mem-fraction-static, which is 0.92 on these points, so it is sized to the ~4.2 GiB MegaMoEV2 occupies at MTPR 8192 rather than to the larger value the single-node mi355x-fp4-mtp recipe can afford at 0.85." + - "Raise c32 from DSpark gamma 3 to gamma 6, matching c4 and c16, and follow the recipe's 64 x (draft + 1) rule for MORI_MAX_DISPATCH_TOKENS_DECODE, 256 -> 448." + - "Set SGLANG_MLA_HCA_KV_SPLITS=8 on c4, overriding the image default of 4 for the HCA compress-ratio-128 decode stream in the unified_kv kernels." + - "Restore --decode-policy round_robin on the sglang-router frontend. The pre-#3428 launcher hardcoded it for the agentic PD path and the srt-slurm port dropped it, which silently applied --policy consistent_hashing to decode as well. Affects every point in this entry." + - "Stop decode inheriting prefill's UMBP linker flags through the recipe's args anchor: --enable-unified-cache-external-linker, --unified-cache-external-linker-backend, --hicache-storage-backend-extra-config, --enable-cache-report and --optimistic-prefill-attempts. server_sglang.sh appended these to PREFILL_SERVER_CONFIG only and recorded that decode does not read them; decode runs no UMBP tier, so the ported recipe had the linker enabled against an empty UMBP_STANDALONE_ADDRESS. Affects every point in this entry." + - "Add --request-timeout-secs 14400 to the sglang-router frontend of c384 and c512 only. The gateway's 1800 s default aborts every request still queued behind an overloaded prefill at once, and an abort landing mid-RDMA-write strands the mori TransferStatus and wedges that prefill DP's transfers permanently." + - "Sizing of the new points follows the existing c192/c256 rule: admission is 2x CONC on both roles and decode captures graphs up to admission divided by dp-size, giving max-running-requests 768/1024 and cuda-graph-bs-decode 1-96/1-128." + - "Bump the image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260923 to lmsysorg/sglang-rocm:v0.5.21-rocm724-mi35x-20261001, which carries the MegaMoE kernels, and update model.container to match. A master-config entry carries one image and search-space arms cannot override it, so every point in this entry moves onto the new image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3664 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto + scenario-type: + - agentic-coding + description: + - "Add GB300 Qwen3.5-397B-A17B-FP8 AgentX recipes: four disaggregated TP1xPP4 MTP prefill / DEP4 MTP decode points and five aggregate TP4/TP8 MTP points." + - "New recipes and matrix entries only; existing Qwen3.5 GB300 entries are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3231 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Enable SGLANG_OPT_USE_TOPK_V2 (false -> true) so the DSA indexer uses the top-k v2 kernel." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3724 + +- config-keys: + - "*" + description: + - "Update the shared srt-slurm pin from v2.39.1 to v2.43.4, including staged Dynamo wheel dependency resolution." + - "Migrate five GLM-5.2 H200 AgentX recipes and the DeepSeek-V4 MI355X AgentX override bundle to schema 2; rely on automatic golden acceptance injection for the GLM-5.2 recipes." + - "Use schema 2 benchmark client placement in eleven Kimi-K3 GB200 and GB300 AgentX recipes." + - "Place discovery services on the existing infra node in three H200 AgentX recipes with required telemetry." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3755 + +- config-keys: + - minimaxm3-fp8-h200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the MiniMax-M3 H200 vLLM image and enable the FP8 MSA indexer cache." + - "Use GPU-resident KV for concurrency 12 and add concurrency 14; remove the Mooncake DRAM-offload variant." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3736 + +- config-keys: + - dsv41flash-fp4-mi355x-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Move to lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261006." + - "Switch from TP4/EP4 at concurrency 1-32 to TP2/EP1 at concurrency 1, 2, 4, 8, 16, 32 and 64, with host Engram tables (per_rank layout), shared-experts fusion and AITER FP8 GEMMs. The prefill chunk is 16384 tokens through C32 and 4096 at C64; the PDI is 16 through C16 and 4 from C32." + - "Add TP2's sibling band at TP4/EP1, concurrency 1, 2, 4, 8 and 16, on the same recipe." + - "Run decoder SWA bounded replay with prefill graphs disabled on every point." + - "Throughput keeps the committed thinking-on golden AL 3.51 selected by the srt connector; evals use real verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3696 + +- config-keys: + - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg + - glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp + - glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Keep the GLM-5.2 NextN/MTP draft at shipped precision on GB200." + - "Use the digest-pinned upstream SGLang v0.5.18 CUDA 13 image with NIXL 1.4.0 for the two disaggregated recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3401 + +- config-keys: + - dsv41flash-fp4-gb200-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the GB200 DeepSeek-V4.1-Flash vLLM AgentX image from nightly cd10ed6f to nightly-dev-arm64-cu130-ac9126e58aa7 and enable FlashInfer autotuning." + - "Run TP4 at concurrency 1, 4, 8 and 32 and replace TP2 with DEP2 (TP1 x DP2 + EP2) at concurrency 8-32 and DEP4 (TP1 x DP4 + EP4) at concurrency 64-128, both with DeepGEMM MegaMoE and adaptive verification behind a consistent-hash vLLM Router. All points set gpu-memory-utilization 0.97." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3692 + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang-agentic-agg + - dsv4-fp4-gb200-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB200 DeepSeek-V4-Pro AgentX disaggregated recipes from hierarchical cache to the Mooncake unified-cache external linker with group semantics, as #3187 did for GB300. MoE settings are unchanged: the recipes keep MegaMoE without W4A4 MXFP4." + - "Decode nodes lend host DRAM to the store through srt-slurm mooncake-store services: four 180GB standalone stores per decode node, gated on their readiness ports with a 600-second timeout. Prefill keeps a 140GB segment per rank. GB200 nodes have the same host memory as GB300 nodes, so the sizes carry over." + - "The prefill role and the decode stores name their RDMA devices (MOONCAKE_DEVICE=mlx5_0..3, as the GB200 vLLM Mooncake recipes do) and set MC_TCP_BIND_ADDRESS to their node, so each store transfer engine advertises the node address instead of the first active interface." + - "The benchmark client runs on the dedicated etcd/nats node, because prefill nodes now also hold Mooncake segments, and Tachometer's per-node exporters are off for this throughput ladder." + - "The aggregate TP8 c1/c4 recipes are unchanged and stay on hierarchical cache; their config key is listed so this sweep covers the whole GB200 Pareto ladder, including the low-latency points." + - "Measured at concurrency 128 (1P DEP8 / 1D DEP16) against the current recipe with the same image, srt-slurm and golden acceptance length: 26,566 vs 23,763 total tok/s/GPU (+11.8%) at the same P90 TPOT (7.31 vs 7.22 ms), with TTFT p50 3.27 vs 4.95 s." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3701 + +- config-keys: + - "*" + description: + - "Accept native SRT power samples v3 with optional GPU temperature while preserving v1/v2 replay and existing energy validation. Temperature remains in raw telemetry for ingestion and display." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3685 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Restore DeepSeek-V4.1-Flash FP4 ATOM AgentX on MI355X (#3387) on the native srt-slurm recipe benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml; the legacy script and master-config entry were removed in #3461 / #3463. Points change from #3387: TP2 concurrency [1, 2, 4, 5, 6, 8, 16, 32, 64, 128] (adds 4, 5, 6 and 128), TP4 concurrency [1, 4, 5, 6, 8, 32] (adds 1, 4, 5 and 6; drops 2, 16 and 64); no expert parallelism or KV offload. Concurrency 1-8 covers the batch sizes that V4.1 mono decode serves (up to 8 requests a step)." + - "Update the image from rocm/atom-dev:nightly_202609250902 to nightly_202610070333 (digest sha256:2f605a3e8b459a0385f784edb0335c11892277fee650fe42044e4af39e36dd01; ATOM 0.1.7.dev60+g7fc0926c6, ROCm 7.2.4). ROCm/ATOM#2419 adds group32 FP8 weight preshuffle and BF16 RoPE tables for DeepSeek-V4.1. ROCm/ATOM#2479 adds V4.1 mono decode, which is on by default and serves decode steps of up to 8 requests for this TP2/TP4, BF16-KV, five-token DSpark deployment, and the FP4 index plane that the upstream recipes/DeepSeek-V4.1-Flash-Agentic.md now uses. The image's AITER includes ROCm/aiter#6174, which adds gfx950 tuned rows for the six DeepSeek-V4.1-Flash TP4 bf16 GEMM (N, K) pairs that previously fell back to torch, including the DSpark verify batches and the TP4 lm_head. ROCm/ATOM#2483 replaces the Triton mono step fence, which Triton 3.7 on gfx950 compiled into a loop that could spin for ever: on nightly_202610061452 and nightly_202610061600 the TP2 c1 canary hung in warmup on the DSpark draft's first mono step and never served (runs 37486079757 and 37493871230)." + - "Server settings follow the upstream recipe as before (BF16 KV, FP4 index cache (FP8 before; follows the upstream recipe at 7fc0926c), max-num-seqs 128, 16K batching/prefill chunks, prefix caching with block size 16, 8K state checkpoints, level 3, FULL graphs, dsml_v41 tool parsing), except that every point now captures the dense list [1..32, 48, 64, 128] instead of only concurrency 32. Five-token DSpark takes the thinking_on golden AL 3.51 from infx/srt_slurm/synthetic_acceptance.py for throughput and real acceptance for eval. Replay semianalysis_cc_traces_weka_062126 for 3600 seconds per point with five warmup requests per lane." + - "No data-type or precision change to the DSpark draft (layers 37-39, loaded inline from the checkpoint): no --online_quant_config is passed, so the draft keeps its checkpoint precision; ROCm/ATOM#2419 only lets its fused wkv GEMM consume the preshuffled FP8 layout and fuses inverse RoPE into the output projection. --kv_cache_dtype bf16 and --index-cache-dtype fp4 touch cache storage only." + - "TP4 concurrency 32 sets ATOM_PREFILL_DECODE_INTERVAL=16: after each prefill, ATOM runs at least 16 decode passes and batches the prefills that arrive in the meantime (ATOM's PrefillDelayer, off by default). Prefill no longer interrupts decode as often, so P90 interactivity rises while TTFT rises. Local 3600 s replay: 108.2 -> 150.9 tok/s/user, ITL p90 9.24 -> 6.63 ms, TTFT p50 0.51 -> 1.14 s, throughput unchanged." + - "TP2 concurrency 64 sets ATOM_PREFILL_DECODE_INTERVAL=4. At this load each batched prefill forward serves several requests, so throughput rises as well as interactivity. Local 3600 s replay: 30.3 -> 38.8 tok/s/user, 105.4k -> 115.2k tok/s/GPU, 6487 -> 6941 requests, TTFT p90 2.3 -> 6.4 s. All other points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3603 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Refresh GLM-5.2 NVFP4 GB300 Dynamo-SGLang AgentX with lmsysorg/sglang:nightly-dev-20260914-4358a161 (bundled Dynamo 1.5.0.dev20260913). Newer nightlies were not usable: nightly-dev-20261002-67eab570 includes sgl-project/sglang#40135, whose cgroup-bounded host-memory check rejects the 135 GB-per-rank HiCache pools on these nodes, and nightly-dev-20260918-20518d85 produced corrupted 1P4D GSM8K output (0.46) and a prefill GPU page fault at c192 with server arguments identical to the passing 4358a161 runs and one schema-2 recipe per benchmark point: aggregate TP8 at concurrency 1, 2 and 4; disaggregate 1P2D c48, 1P4D c45 and 1P6D c45 (DEP4 prefill / TP4 decode); and 2P1D c192 (DEP8 prefill / DEP16 decode)." + - "Aggregate points use EAGLE MTP 5-1-6 (5 steps, topk 1, 6 draft tokens). Disaggregate points use MTP 2-1-3 on both prefill and decode. The srt-slurm connector selects the committed GLM-5.2 MTP golden AL for each draft length; no recipe sets a synthetic acceptance length." + - "Run from the nightly's bundled Dynamo. Use FlashInfer CuteDSL for disaggregate prefill MoE/FP4 GEMM and for the DEP16 decode (DeepEP low-latency, NVFP4 dispatch). Set aggregate max-running-requests and cuda-graph-max-bs-decode to twice each point's concurrency. Use mem-fraction-static 0.75 on disaggregate prefills and a 1000000-second scheduler watchdog on all serving roles. Use the HiCache kernel I/O backend for the c192 prefill and direct I/O elsewhere." + - "Drop settings that are no-ops or SGLang defaults in this image: SGLANG_ENABLE_THINKING and SGLANG_REASONING_EFFORT (only read by the DeepSeek/Kimi custom chat encoders, not by GLM-5.2), SGLANG_ENABLE_UNIFIED_RADIX_TREE (unified radix tree is the default), SGLANG_HICACHE_DEBUG_LOG/SAMPLE_RATE (unread), SGLANG_MOE_NVFP4_DISPATCH on roles without an all-to-all backend, --kv-cache-dtype fp8_e4m3 and --dsa-prefill/decode-backend trtllm (DSA defaults on SM10x), and on TP roles --moe-runner-backend flashinfer_trtllm, --disable-shared-experts-fusion and --enable-flashinfer-allreduce-fusion (automatic). Also drop SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK, so SGLang's default startup check for unbalanced free GPU memory across a worker's ranks is active." + - "No draft precision change: the embedded GLM-5.2 EAGLE MTP head runs at the precision shipped by the checkpoint, and no draft dtype or quantization override is set." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3659 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Move the GB300 DeepSeek-V4-Pro AgentX disaggregated ladder from hierarchical cache to the Mooncake unified-cache external linker on the image's own SGLang, bump the image to lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9, and bump the Dynamo wheel to 1.5.0.dev20260914, whose SGLang worker no longer needs ServerArgs.get_model_config." + - "Decode nodes lend host DRAM to the store through srt-slurm mooncake-store services: four 180GB standalone stores per decode node, started before the workers against the managed master and its HTTP metadata server and gated on their readiness ports with a 600-second timeout, because cold container starts on decode nodes exceeded srtctl's 120-second default. Prefill keeps a 140GB segment per rank." + - "Prefill runs 16k-token chunks per DP rank (chunked-prefill-size 131072, SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK 17408) with mem-fraction-static 0.80, which the DSV4 indexer needs at that chunk size." + - "The benchmark client runs on the dedicated etcd/nats node instead of the head prefill node, where its ~300GB of host memory OOM-killed prefill_0, and Tachometer's per-node exporters are off for this throughput ladder because they slowed decode steps by a few percent." + - "The recipe names the RDMA devices (MOONCAKE_DEVICE=mlx5_0..3) on the prefill role and the decode stores, the processes that act as store clients, like the GB300 vLLM Mooncake recipes: without an explicit list the store client falls back to NVLink for same-domain peers and aborts every worker at the warmup put." + - "The ladder's widest point moves from concurrency 1920 to 2400 on the same 4P1D topology, where prefill still had headroom, and the 1P1D point also runs at concurrency 120. At equal P90 interactivity it delivers about twice the throughput per GPU of the aggregate TP4 concurrency-8 point, which is removed from dsv4-fp4-gb300-dynamo-sglang-agentic-agg." + - "The aggregate recipe sets PP_SIZE and PCP_SIZE to 1 in its benchmark environment: the AgentX power path now checks TP, PP_SIZE and PCP_SIZE before replay, and multi-node aggregate jobs do not receive them from the workflow." + - "The prefill role and the decode store services set MC_TCP_BIND_ADDRESS to their node so each store transfer engine advertises the node address instead of the first active interface; some GB300 nodes expose a BMC Redfish interface with the same 10.0.1.2 address on every node, which peers cannot reach." + - "W4A4 MXFP4 MegaMoE stays on the prefill role only. The MegaMoE MMA type is a global setting and the DSpark draft shares the MegaMoE backend under DP attention, so on decode W4A4 would run the draft MoE with FP4 activations instead of the image default (fp8xfp4). The prefill DSpark worker runs the target forward and writes the draft KV from target hidden states without running the draft MoE. The FP4 indexer stays on for both roles: the draft has no indexer, and the indexer cache layout must match between prefill and decode." + - "将 GB300 DeepSeek-V4-Pro AgentX 分离式配置从 hierarchical cache 切换到镜像自带 SGLang 的 Mooncake unified-cache external linker,将镜像升级到 lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9,并将 Dynamo wheel 升级到 1.5.0.dev20260914,其 SGLang worker 不再依赖 ServerArgs.get_model_config。" + - "decode 节点通过 srt-slurm 的 mooncake-store 服务向 store 借出主机 DRAM:每个 decode 节点 4 个 180GB 的独立 store,在 worker 之前启动,连接托管的 master 及其 HTTP metadata server,并以就绪端口作为启动检查,超时设为 600 秒,因为 decode 节点上冷启动容器曾超过 srtctl 默认的 120 秒。prefill 每个 rank 保留 140GB segment。" + - "prefill 每个 DP rank 按 16k token 切分(chunked-prefill-size 131072,SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK 17408),并将 mem-fraction-static 降至 0.80,这是该切分大小下 DSV4 indexer 所需。" + - "压测客户端改为运行在独立的 etcd/nats 节点上,而不是 head prefill 节点(其约 300GB 主机内存曾导致 prefill_0 被 OOM kill);该吞吐阶梯关闭 Tachometer 的逐节点 exporter,因为它们会使 decode 单步变慢数个百分点。" + - "配方在 prefill 角色与 decode 节点的 store 上(即作为 store 客户端的进程)显式指定 RDMA 设备(MOONCAKE_DEVICE=mlx5_0..3),与 GB300 vLLM Mooncake 配方的做法一致:未显式指定时,store 客户端会对同 NVLink 域的对端退回 NVLink,并在 warmup put 阶段使所有 worker 退出。" + - "阶梯中吞吐最高的点在相同的 4P1D 拓扑下由并发 1920 调整为 2400(该并发下 prefill 仍有余量),1P1D 点同时增加并发 120。在相同的 P90 交互性下,其单卡吞吐约为聚合 TP4 并发 8 点的两倍,因此从 dsv4-fp4-gb300-dynamo-sglang-agentic-agg 中移除该聚合点。" + - "聚合式配方在 benchmark 环境中设置 PP_SIZE 与 PCP_SIZE 为 1:AgentX 功耗路径现在会在请求回放前检查 TP、PP_SIZE 与 PCP_SIZE,而多节点聚合任务不会从工作流获得这两个变量。" + - "prefill 角色与 decode 的 store 服务将 MC_TCP_BIND_ADDRESS 设为所在节点,使每个 store 传输引擎对外公布节点自身地址,而不是第一个活动网卡;部分 GB300 节点带有 BMC Redfish 网卡,其地址在每个节点上都是 10.0.1.2,其他节点无法访问。" + - "W4A4 MXFP4 MegaMoE 仅在 prefill 角色开启。MegaMoE 的 MMA 类型是全局设置,且在 DP attention 下 DSpark 草稿模型共用 MegaMoE 后端,因此在 decode 上开启 W4A4 会使草稿 MoE 以 FP4 激活运行,低于镜像默认的 fp8xfp4。prefill 端的 DSpark worker 只运行目标模型前向,并由目标隐藏状态写入草稿 KV,不运行草稿 MoE。FP4 indexer 在两个角色上都保持开启:草稿模型没有 indexer,且 prefill 与 decode 的 indexer 缓存布局必须一致。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3187 + +- config-keys: + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4.1-Flash FP4 AgentX on GB300 with the Dynamo frontend (KV router, session affinity) and SGLang nightly-dev-20260928-81f27fb3 / ai-dynamo 1.5.0.dev20260914, DSpark block size 5." + - "Use the mounted shared Hugging Face cache for every SGLang server role." + - "Aggregated arms: the two lowest-latency points of the curve, a pure TP4 (EP1) c1 point (the standalone recipe's TP4 c1 settings, static ragged verify and Engram in HBM, behind Dynamo; 5,983 tok/s/GPU @ 376.5 tok/s/user P90, P90 TTFT 1.6 s) and a TP2/EP2 c1 point with static ragged verify and Engram in host DRAM (11,733 @ 363.8, 1.5 s)." + - "Disaggregated curve with Mooncake KV transfer and TP2/EP2 workers: 1P2D with a 1 s Dynamo session-affinity TTL at c16 (one decode worker per node; 19,305 tok/s/GPU @ 331.6 tok/s/user P90, P90 TTFT 0.7 s), c48 (59,243 @ 251.3, 1.4 s) and c64 (80,750 @ 210.4, 2.0 s); 1P1D at c96 (143,459 @ 132.7, 4.1 s) and, with the SGLang HiCache host-memory prefix-cache tier on the prefill worker, at c160 (187,463 @ 99.0, 14.0 s); 2P1D HiCache at c256 (179,107 @ 68.3, 15.2 s). Every point passed its GSM8K gate; at matched P90 interactivity the curve is 1.8-2.7x the published MI355X ATOM tokens/s/GPU." + - "新增 GB300 上 DeepSeek-V4.1-Flash FP4 AgentX 的 Dynamo 前端(KV 路由、会话亲和)+ SGLang nightly-dev-20260928-81f27fb3 / ai-dynamo 1.5.0.dev20260914 配方,DSpark block size 5。" + - "所有 SGLang server role 使用已挂载的共享 Hugging Face cache。" + - "聚合分支:曲线上延迟最低的两个点——纯 TP4(EP1)c1(沿用单机配方的 TP4 c1 设置、static ragged verify 与 HBM 内 Engram,置于 Dynamo 之后;5,983 tok/s/GPU @ 376.5 tok/s/user P90,P90 TTFT 1.6 s)与采用 static ragged verify、Engram 置于主机内存的 TP2/EP2 c1(11,733 @ 363.8,1.5 s)。" + - "分离式曲线(Mooncake KV 传输,TP2/EP2 worker):1P2D 配合 1 秒 Dynamo 会话亲和 TTL 的 c16(每节点一个 decode worker;19,305 tok/s/GPU @ 331.6 tok/s/user P90,P90 TTFT 0.7 s)、c48(59,243 @ 251.3,1.4 s)与 c64(80,750 @ 210.4,2.0 s);1P1D 的 c96(143,459 @ 132.7,4.1 s)以及在 prefill worker 上启用 SGLang HiCache 主机内存前缀缓存层后的 c160(187,463 @ 99.0,14.0 s);2P1D HiCache 的 c256(179,107 @ 68.3,15.2 s)。所有点均通过 GSM8K 验证;在相同 P90 交互性下,该曲线为已发布 MI355X ATOM tokens/s/GPU 的 1.8-2.7 倍。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3598 + +- config-keys: + - dsv41flash-fp4-b200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the DeepSeek-V4.1-Flash FP4 B200 SGLang AgentX image to nightly-dev-cu13-20261004-295a53c9, pinned by multiarch digest." + - "Refresh the aggregated curve with TP4/EP4 points at concurrency 1, 4, 8, 16, 24 and 32, plus TP4/EP4/DP4 points at concurrency 48 and 128 behind sglang-router." + - "Apply the point-specific prefill chunking, scheduling, admission, memory and SWA prefix-tail settings from the single-node recipe." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3745 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202609281543 (ATOM 0.1.7.dev43+gaa0c5c3a1) to nightly_202610070333 (digest sha256:2f605a3e8b459a0385f784edb0335c11892277fee650fe42044e4af39e36dd01; ATOM 0.1.7.dev60+g7fc0926c6, ROCm 7.2.4). ROCm/ATOM#2479 moves MiniMax-M3 mono decode onto the shared mono framework, publishes every mono mailbox flag only after the stores it announces have drained, and rewrites the sparse selection as exact top-k by counting; ROCm/ATOM#2483 rebuilds the per-step mailbox clear and cross-rank fence as one FlyDSL launch, replacing a Triton fence that Triton 3.7 on gfx950 compiled into a loop that could spin for ever; ROCm/ATOM#2422 fixes the mono long-context decision. The upstream recipes/MiniMax-M3-Agentic-InferenceX.md is unchanged." + - "Enable MiniMax-M3 mono decode at every TP4 point by removing ATOM_MONO_ENABLE=0 from the indexer-CP points C15, C20, C24 and C28. Mono serves decode steps of up to 16 tokens (four requests with EAGLE3 K3), covering all of TP4 C1, C2 and C4. ATOM builds mono for TP4 only, so TP2 runs the unfused path." + - "Add TP2 concurrency 4 (max-num-seqs 8); the GPU-resident TP2 points are now C1, C2 and C4." + - "Record LMCache 0.5.6.dev139+gf22dec28, the wheel in the new image, for both DRAM-offload arms; the offload settings are unchanged." + - "No data-type or precision change to the EAGLE3 draft model (Inferact/MiniMax-M3-EAGLE3-GQA): online_quant_config (ptpc_fp8), kv_cache_dtype fp8 and index-cache-dtype fp8 are unchanged; mono decode changes the target model's decode kernels, not draft precision." + - "更新镜像至 rocm/atom-dev:nightly_202610070333(ATOM 0.1.7.dev60+g7fc0926c6,含 ROCm/ATOM#2483 的 mono step fence 修复);去掉 TP4 C15/C20/C24/C28 的 ATOM_MONO_ENABLE=0,所有 TP4 点位启用 MiniMax-M3 mono decode(每步最多 16 个 token,即 EAGLE3 K3 下 4 个请求,覆盖 C1/C2/C4);ATOM 只为 TP4 构建 mono,TP2 走非融合路径。新增 TP2 C4,LMCache 版本记录为 0.5.6.dev139+gf22dec28。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3750 + +- config-keys: + - qwen3.8next-fp4-gb300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.8-Flash-Next NVFP4 GB300 TP1/EP1 AgentX with native NEXTN MTP and SGLang HiCache." + - "Pin the stock SGLang v0.5.21-cu130 image and checkpoint revision; enable engine metrics and cache reporting." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3816 + +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5 FP8 GB300 AgentX PP4 prefill and DEP16 decode variants with MTP6, HiCache, and decode sequence-length splits 1." + - "Select five disaggregated configurations and three existing aggregate configurations using the SGLang nightly image and pinned Dynamo revision." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3840 + +- config-keys: + - dsv41flash-fp4-b300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Repin DeepSeek-V4.1-Flash FP4 B300 SGLang AgentX from nightly-dev-cu13-20260922-582389ce to nightly-dev-cu13-20261004-295a53c9, pinned by multiarch digest, and publish nine points: TP4/EP1 at concurrency 1 and 4, TP2/EP1 at concurrency 1, 4, 16, 24, 32 and 48, and aggregated TP4/EP4/DP4 at concurrency 192." + - "For the TP arms, switch to EP1, use 16K prefill chunks, and set point-specific prefill-decode scheduling." + - "Add an aggregated DP4 C192 point with four internal TP4/EP4 data-parallel ranks behind sglang-router 0.3.2, using cache-aware routing and correlation-ID affinity." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3744 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Bump the image from lmsysorg/sglang:dev-cu13-nightly-0924 to lmsysorg/sglang:nightly-dev-cu13-20260929-79cafec0 and update model.container to match. With the recipe otherwise unchanged, local GB300 runs on the new image reproduce the PR #3421 sweep (TP4 c1 439.8 vs 436 tok/s/user median, TP4 c32 249.5 vs 250)." + - "Set SGLANG_DSPARK_FOLDED_SAMPLING=2 on every point. The AUTO default keeps DSpark draft sampling eager for DeepSeek-V4.1 TP shards; nsys at TP4 c1 shows the decode step drop from 10.98 ms to 6.79 ms and GPU idle per step from 5.98 ms to 1.96 ms when it is folded into the draft graph. Local one-hour runs (synthetic acceptance 3.51): TP4 c1 5,845 -> 6,931 tok/s/GPU at 439.8 -> 649.8 tok/s/user median (386.8 -> 539.2 P90); TP4 c32 56,719 -> 57,704 at 249.5 -> 274.2 median; TP2 c64 147,452 -> 149,666 at 96.3 -> 99.4 median." + - "TP4 c64: 8K prefill chunks with a 16-step prefill-decode interval instead of 16K chunks with none. Local runs with the fold: 82,586 -> 85,093 tok/s/GPU at 86.8 -> 136.8 tok/s/user median (46.3 -> 72.1 P90); P90 TTFT 5.7 -> 12.5 s." + - "TP2 c128: admit all 128 sessions (max-running-requests and cuda-graph-max-bs-decode 64 -> 128) with a 4-step prefill-decode interval and mem-fraction-static 0.85. The PR #3421 sweep measured 18,631 tok/s/GPU at 130.9 tok/s/user median at this point; locally the new settings gave 147,763 tok/s/GPU at 37.9 tok/s/user median with a 51 s P90 TTFT (measured before the fold was added)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3782 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use five EAGLE speculative steps and six draft tokens at concurrency 1, 4, 8, and 20 on GLM-5.2 NVFP4 B300 SGLang." + - "Keep three speculative steps and four draft tokens at the other concurrencies." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3890 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump to lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261007." + - "Run prefill context parallelism on the TP-only arms at conc 16, 32 and 48; conc 1 and 4 keep BCG." + - "Enable SGLANG_OPT_FP8_WO_A_FUSED_INVROPE." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3778 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610070333 to nightly_202610100929 (digest sha256:cc60ba06b28050a7fdec71e21f1853c782a9ec8850f031e84ebf0e49f3180974; ATOM 0.1.8.dev14+g73defd9b8, ROCm 7.2.4) and update model.container to match. The recipe, points and server settings are unchanged; upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged between 7fc0926c6 and 73defd9b8." + - "ROCm/ATOM#2507 turns decoder SWA bounded replay on by default for DeepSeek-V4.1: a prefill runs layers 21..39 on each request's last 133 rows only, the same approximation as SGLang's --enable-decoder-swa-bounded-replay. Upstream reports +10.7% / +22.7% / +38.1% P90 interactivity at TP2 c16 / c32 / c64 on this AgentX workload, with no GSM8K or long-context retrieval change beyond noise." + - "ROCm/ATOM#2512 samples the DSpark Markov block from each TP rank's vocab shard instead of all-gathering the full logits (bit-identical ids; draft head 657 -> 302 us per decode step at TP2 c32). ROCm/ATOM#2513 packs the FP4 index logits by row visibility and moves BF16 decode to the two-source paged kernel (needs ROCm/aiter#6334). ROCm/ATOM#2495 fixes the FP4 sparse indexer output contract." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3901 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202609291501 to rocm/atom-dev:nightly_202610071700 (digest sha256:bb3912c372dda0c82fc6831afdd76e366a372d5e9fe223373499db63b06cce65)." + - "Move the C48, C64, C96 and C128 DEP points from RCCL expert parallelism to ATOM's MegaMoE backend (all2all-backend high-throughput, moe-backend mega), with EPLB at every point except C64." + - "Cap max-num-seqs at 128 from C128 up (C128 was 256, C256 was 512); C48, C64 and C96 keep admission at 2x CONC." + - "Run C256 and the new C384 on the same MegaMoE DEP setup with ATOM's lmcache_mp CPU KV tier: one LMCache server per node (1000 GB L1, LMCache 0.5.6.dev139+gf22dec28 as shipped in the image), started by srtctl's lmcache-server service, within a 1019 GB budget (dram-utilization 0.34)." + - "C1 to C16 are unchanged." + - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4-Pro-0813 runs as shipped." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3819 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from nightly-rocm100-ac9126e58aa7bbab1856ba6593ba4d5003fea516 to nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe, the first ROCm 10.0 nightly with the DeepSeek-V4.1 mono decode kernels of vllm-project/vllm#60397, and update model.container to match. The image also brings vllm-project/vllm#59685, #56638, #60159 and #60389." + - "Enable the mono decode kernels (VLLM_ROCM_MONO_DECODE=1) at TP2 and TP4: a decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, runs each backbone layer as two persistent launches (30 of 40 layers whole, the other 10 the FFN launch only). Larger steps take the regular path, so C16-C64 are expected unchanged." + - "Enable VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 (vllm-project/recipes#1061) for tensor-parallel all-reduces above the quick-reduce size threshold." + - "Restore --max-num-batched-tokens 16384 at TP2 c64: vllm-project/vllm#58014 removed the profiling over-reservation the 8192 cap worked around. The prefill chunk is now 16384 at every point and the ladder stays at concurrency 1-64." + - "No data-type or precision change to the DSpark draft: vllm-project/vllm#60397 skips the draft layers (layer id >= num_hidden_layers), so the bundled DSpark draft of deepseek-ai/DeepSeek-V4.1-Flash runs on vLLM's regular path as shipped." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3690 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from vllm/vllm-openai-rocm:nightly-rocm100-36768d1bfd39094681cdbc8cb37d4b31c0729c89 to vllm/vllm-openai-rocm:nightly-rocm100-81198e97ba7eee2a22540caaa756b7fdddcb4d93." + - "Sweep dcp1 concurrency 1 and dcp8 concurrency 1 GPU-resident, and dcp8 concurrency 4, 10, 12, 14, 16, 44 and 48 with the SimpleCPUOffloadConnector DRAM tier (224.875 GB/rank). This replaces dcp1 concurrency 1/4, dcp1 DRAM-offload concurrency 8-14 and dcp8 concurrency 44/48/70. The dcp8 points use the a2a DCP backend." + - "DSpark draft length per point: 7 at concurrency 1, 5 at 4, 4 at 10, 3 at 12, and 4 at 14, 16, 44 and 48 (concurrency 14 was 3). The throughput band (44, 48) previously had no speculative-config and now runs the 4-token DSpark draft. As at every draft point, agentic runs replace block rejection with the golden acceptance length for that draft length." + - "Admission is 2x concurrency. CUDA graphs capture every even size up to 2x concurrency x (1 + draft tokens), replacing the previous every-size lists (up to 8192 in the throughput band)." + - "Every point requests cudagraph_mode FULL_AND_PIECEWISE (the mid band was FULL) with breakable CUDA graphs disabled (VLLM_USE_BREAKABLE_CUDAGRAPH=0, previously 1 at concurrency 1 and 4). The AMD Kimi-K3 model has no torch.compile submodule, so vLLM overrides the mode to FULL_DECODE_ONLY at load, as the concurrency 1 and 14 server logs show." + - "Add stream-interval 10 and VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, which routes DCP multi-token verify through the AITER ASM round-robin decode, to every point." + - "Concurrency 1 (dcp1 and dcp8) adds quantization-config linear fp8_per_channel: vLLM quantizes the target's linear layers to FP8 W8A8 at load, except kv_b_proj, in_proj_qkvgfab, f_b_proj, fused_qkv_a_proj, self_attn.g_proj and layer 0 mlp.down_proj, which stay BF16. The routed experts stay MXFP4." + - "No change to the Inferact/Kimi-K3-DSpark draft's precision: its model config takes the speculative config's own quantization (unset) and never receives quantization-config, and get_draft_quant_config() returns None for K3DSparkModel, so its weights stay BF16 and its KV cache stays fp8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3833 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Update the GLM-5.2 NVFP4 GB300 TensorRT-LLM AgentX image from 1.3.0rc26.dev202609040000 to 1.3.0rc28.dev202609210001 on all six points." + - "Switch NIXL KV transfer to the C++ transfer-agent bounce buffer (max_average_descriptor_size 4MB, min_descriptor_count 1) and lower kv_cache_bounce_size_mb from 5120 to 1024." + - "Move Dynamo on all six points to the ai-dynamo 1.6.0.dev20261010 wheel, which includes ai-dynamo/dynamo#15628 (routed node address selection for metadata and transport advertisement)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3707 + +- config-keys: + - dsv41flash-fp4-mi355x-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Bump to lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261012." + - "Trade TTFT for P90 ITL with a longer prefill-decode interval: 192 with lpm scheduling at concurrency 1-8 (TP2 and TP4) and 96 at concurrency 16." + - "Enable SGLANG_OPT_HIP_OPUS_SPARSE_PREFILL and ROCM_QUICK_REDUCE_QUANTIZATION=INT8, previously NONE." + - "Run TP2 concurrency 32 and 64 on DP2 attention with TP MoE instead of TP-only." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3926 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610100929 to nightly_202610121516 (digest sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c; ATOM 0.1.8.dev29+ga0527c9db, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match." + - "Drop TP2 concurrency 5 and 128 and TP4 concurrency 4, 5 and 6. TP2 runs concurrency 1, 2, 4, 6, 8, 16, 32 and 64 and TP4 runs 1, 8 and 32: 11 points instead of 16." + - "Server settings are unchanged. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is identical between 73defd9b8 and a0527c9db." + - "ROCm/ATOM#2545 reworks decoder SWA bounded replay to follow the V4.1 report: the forward that finishes a prompt runs the encoder (layers 0-19) and layer 20's compression on every row, layers 20-38 on the last 128 rows with their windows truncated there, and layer 39 on the last row only." + - "ROCm/ATOM#2545 also turns on encoder SWA bounded replay by default: a prefix hit rebuilds the encoder's sliding-window state by recomputing one KV page below the hit instead of reading a state checkpoint, and no state checkpoints are kept. The DeepSeek-V4.1 KV page, which is also the prefix-cache hash block, drops from 256 to 128 tokens." + - "ATOM's mono decode path (on by default) now takes any decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, not only DSpark verify steps. Steps of 12 or more rows that are a multiple of TP run its sublayers token-parallel, with reduce-scatters instead of all-reduces. The draft runner never runs token-parallel." + - "No data-type or precision change to the DSpark draft. The checkpoint's bundled mtp.0-2 stages are stored like the target's layers: FP8 E4M3 weights with 32x32 UE8M0 block scales for the attention projections, shared experts and main_proj; MXFP4 routed experts; BF16 norms, router gate, Markov head and confidence head; F32 attention sinks and hc mixers. ATOM builds the draft with the target's native_quant_config() and make_v4_quant_config(), with no online quantization in this recipe, so it runs as shipped: FP8 activations on the FP8 GEMMs and the target's BF16 KV cache. Between 73defd9b8 and a0527c9db only the rows the draft reads under bounded replay change." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3928 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from nightly-rocm100-7d0b4e57aac4c4323225b0eee9ae960ce75ffffe to nightly-rocm100-5555a2d8f7116b2b4877f66516f4128176cf722a, the 2026-10-13 ROCm 10.0 nightly, which contains vllm-project/vllm#59039 and #61082. Update model.container to match." + - "No point changes: concurrency 1, 2, 4, 8, 16, 32 and 64 on TP2 and TP4 (14 points), all re-measured on the new image." + - "Remove --no-swa-bounded-replay, which turns on decoder SWA bounded replay from vllm-project/vllm#59039: in eager prefill steps, layer 20 (the last KV source layer) writes its KV for every token, and the rest of layer 20 and layers 21-39 run on each request's last 128 tokens only. Prefix-cache SWA replay stays off, since vLLM still forces it off on ROCm (the window clamp it needs lives only in the FlashInfer and FlashMLA prefill kernels)." + - "Replace VLLM_ROCM_MONO_DECODE=1 with --monokernel. vllm-project/vllm#61082 moves the switch to KernelConfig.monokernel and removes the environment variable, so the old setting would leave the mono decode kernels off." + - "vllm-project/vllm#61082 also runs all 40 backbone layers whole as two persistent launches. Before, 30 ran whole and the other 10 (layers 0 and 1 and the compressor / indexer layers 2, 8, 14, 20, 24, 28, 32 and 36) kept vLLM's attention plus the FFN launch. The step limit is unchanged at 48 rows, i.e. concurrency 1-8 with five DSpark drafts; larger steps take the regular path." + - "No data-type or precision change to the DSpark draft. The checkpoint's bundled mtp.0-2 stages are stored like the target's layers: FP8 E4M3 weights with 32x32 UE8M0 block scales for the attention projections, shared experts and main_proj; MXFP4 routed experts; BF16 norms, router gate, Markov head and confidence head; F32 attention sinks and hc mixers. vLLM builds the draft with the checkpoint's own quantization config (get_draft_quant_config) and, with no kv_cache_dtype in speculative-config, the target's fp8_ds_mla KV cache, so it runs as shipped. The mono decode kernels skip the draft layers, and the draft load path is the same in 7d0b4e57 and 5555a2d8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3959 + +- config-keys: + - qwen3.5-fp4-gb200-dynamo-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Add a Qwen3.5 NVFP4 V2 GB200 Dynamo-SGLang disaggregated MTP configuration for the 8k/1k scenario" + - "Add nine checked-in srt-slurm recipes covering TP and data-parallel-attention topologies" + - "Use chat-formatted benchmark inputs with Mooncake KV transfer" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2594 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Bump the image from lmsysorg/sglang:nightly-dev-cu13-20260929-79cafec0 to nightly-dev-cu13-20261012-e6072935 (digest sha256:b5c488694d74c13587af6057b90220498c4869571c8e1e3c43dd6f43ae721306) and update model.container to match. The new nightly makes the Rust radix-tree core the default (sgl-project/sglang#39627), which removes a Python insert walk that grew with prompt length and depth on 200K-400K-token AgentX prompts, and supports DP attention with the MegaMoE all-to-all backend (sgl-project/sglang#43228)." + - "Trim the curve from 16 to 7 points: TP4 c1, c4 and c16; TP2 c16 and c32; DEP2 c64 and c128. TP2 c1/c2/c4/c8 and TP4 c2/c8/c32/c64/c128 are dominated on the tok/s/GPU vs P90 tok/s/user Pareto by the remaining points." + - "TP4 c1: 16K prefill chunks with no prefill-decode interval instead of 4K chunks with a 16-step interval. Long cache-miss prefills now finish in a quarter of the steps, and subagent requests no longer wait behind a long parent prefill. Local one-hour run on the new image: 7,314 tok/s/GPU at 625.7 tok/s/user P90 (670.3 median), P90 TTFT 1.38 s." + - "TP4 c4 and c16 move from EP4 to EP1: at batch 1-5, EP4 makes each decode step wait for the rank holding the most-selected experts. c16 also takes a 192-step prefill-decode interval, 4 tokenizer workers and SGLANG_NUMA_BIND_V2=0; a prefill forward costs about 200 ms of host dispatch on Grace and stalls every running decode." + - "TP2 c16/c32: 16K prefill chunks with decoder SWA bounded replay and the prefill CUDA graph disabled, a 96/48-step prefill-decode interval, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and 4 tokenizer workers; SGLANG_NUMA_BIND_V2=0 at c16 (previously 4K chunks with the default 16-step interval). Local one-hour runs on the new image: c16 55,560 tok/s/GPU at 264.6 tok/s/user P90 (324.9 median); c32 113,613 at 183.1 (242.7)." + - "c64 and c128 move from TP2 to DEP2 (TP2/DP2/EP2 with DP attention, DP LM head and MegaMoE) behind sglang-router (cache_aware, dp-aware, routing key from the AIPerf correlation id), with total_requests load balancing, shortest-prefill-first scheduling, 16K chunks, 4096 SWA prefix tails and 4 tokenizer workers. c64 uses a 24-step prefill-decode interval and admits 128; c128 a 16-step interval, mem-fraction-static 0.85, admits 256 with a 128 graph batch. Local one-hour runs: c64 200,265 tok/s/GPU at 114.3 tok/s/user P90 (132.7 median), versus 149,666 at 99.4 median for the TP2 point it replaces; c128 271,364 at 81.6 P90 (93.0 median), versus 147,763 at 37.9 median." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3859 + +- config-keys: + - dsr1-fp8-mi355x-atom + description: + - "Bump ATOM image to the v0.1.7 release build (rocm/atom-dev:nightly_202610080328, digest-pinned)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3911 + +- config-keys: + - glm5.2-fp8-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Tune GLM-5.2 FP8 B300 SGLang MTP depth, admission caps, and HiCache scheduling with one recipe per concurrency; pin the upstream image by digest." + - "Extend the concurrency grid to 20, 24 and 28 with uncapped admission (2x CONC) to cover the high-throughput end of the frontier." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3836 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + description: + - Pin minimaxm3 B300 FP4 vllm mtp AgentX to the 2026-10-13 image vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a@sha256:e5c9e6bb82f645e8fca04e4d395743ee5ed473a62b98942d65f83c35d0c69301. + - Preserve deployment topology, speculative mode, concurrency points, KV-offload settings and evaluation coverage. + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4000 + +- config-keys: + - glm5.2-fp4-b300-dynamo-sglang-agentic-agg-tp4 + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 NVFP4 B300 AgentX on Dynamo (KV router) + SGLang with one aggregated TP4 engine per point, EAGLE speculative decoding and HiCache host-DRAM offload." + - "Concurrency 1, 8, 16 and 20 run plain TP4; concurrency 24, 32 and 48 add DP4 attention and the prefill delayer." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3953 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 to lmsysorg/sglang:nightly-dev-cu13-20261011-d32d8fa0, pinned by multiarch digest (sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c)." + - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." + - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." + - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - kimik3-fp4-mi355x-vllm-disagg-agentic + scenario-type: + - agentic-coding + description: + - "Add native Kimi-K3 PD with FP8 KV and GMU 0.90 on official ROCm nightly 81198e97ba7eee2a22540caaa756b7fdddcb4d93, pinned by digest in master and recipe." + - "Use TP8 per worker: 1P1D c48/c56, 1P2D c24/c48 and 1P3D c12/c24, with automatic golden acceptance, default request-ID randomization and default evaluation selection." + - "Use the AgentX scenario's default 10% live failure threshold, retaining a stream interval of 10 on both roles and optional server-metrics validation." + - "Set the prefill sequence limit to twice the global concurrency and divide that capacity across decode workers; 1P1D c56 uses 112 sequences per role with a decode graph capture limit of 280." + - "Enable INT4 QuickReduce with a 256 MiB size limit and FlyDSL stage-2 FP8 on prefill and decode workers." + - "Configure MoRIIO READ and prefill-only SimpleCPUOffloadConnector through native role arguments. Use recipe-scoped draft checkpoint and RDMA device/provider mounts. Use the upstream discovery-template fix from srt-slurm #508 through the shared v2.51.1 dependency." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3582 + +- config-keys: + - qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto + scenario-type: + - agentic-coding + description: + - "Add a Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX configuration with aggregate MTP recipes for TP2/EP2, TP2, and TP4 worker layouts." + - "Use the pinned lmsysorg/sglang:nightly-dev-20261013-7224fce1 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." + - "Serve the NextN draft layer at its shipped precision: no recipe sets SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE." + - "Replace aliases removed in this image (cuda-graph-max-bs becomes cuda-graph-max-bs-decode; SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 becomes SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0) and drop settings that have no effect in it." + - "Run the TP2/EP2 point with the default linear-replayssm-cache-len (16) instead of 8: this image's GDN ReplaySSM spec-verify commit kernel needs a replay window of at least 16, and 8 fails to compile at the first verify." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2978 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48; the latent KV is striped across the four ranks, so these points run without HiCache, which DSA DCP rejects." + - "Keep the EAGLE draft with 5 steps, top-k 1 and 6 draft tokens, FP8 KV, the triton DSA prefill and decode backends, admission at twice the concurrency and a decode graph limit of min(2 x concurrency, 64)." + - "Validate decode-context-parallel-size against the matrix DCP size for SGLang recipes as well as vLLM and ATOM." + - "Requires an SGLang ROCm image with sgl-project/sglang#43735 and the triton sparse-MLA LSE path for DCP." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4064 From a4a0f34c7038f677b41e1e593068222f85d38b75 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang <280536569+jiejingzhangamd@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:17:45 +0000 Subject: [PATCH 4/5] fix(amd): use merged DSA DCP ROCm image for GLM-5.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GLM-5.2 SGLang AgentX 配置更新到包含上游 #44322 的 20261028 ROCm 7.2 镜像,并更新对应性能变更记录。 --- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 085a7da146..6bde71bff5 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1417,7 +1417,7 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: # c4 intentionally overlaps arms (1) and (3) to compare the topology crossover. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260930 + image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 940ec5d8a2..ec0971eae0 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9759,5 +9759,5 @@ - "Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48; the latent KV is striped across the four ranks, so these points run without HiCache, which DSA DCP rejects." - "Keep the EAGLE draft with 5 steps, top-k 1 and 6 draft tokens, FP8 KV, the triton DSA prefill and decode backends, admission at twice the concurrency and a decode graph limit of min(2 x concurrency, 64)." - "Validate decode-context-parallel-size against the matrix DCP size for SGLang recipes as well as vLLM and ATOM." - - "Requires an SGLang ROCm image with sgl-project/sglang#43735 and the triton sparse-MLA LSE path for DCP." + - "Use lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 for all three arms; the published nightly includes sgl-project/sglang#43735 and #44322, enabling Triton DSA DCP with FP8 KV and MTP on ROCm." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4064 From faabb0d1597f57e064ca1120cef11ecf913a2e8c Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 30 Oct 2026 14:00:09 -0700 Subject: [PATCH 5/5] docs(changelog): rewrite GLM-5.2 FP4 MI355X SGLang DCP4 block from the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update the image for all three arms (TP4/EP4 HiCache, TP4/EP4 DCP4, TP8/EP1) from lmsysorg/sglang:v0.5.22-rocm720-mi35x (digest-pinned release) to the lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 nightly, built from SGLang main e0df901d. - Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48. The latent KV is striped across the four ranks, so these points keep KV on the GPU without HiCache, which SGLang rejects with DCP. - DCP4 points keep the base recipe: EAGLE with 5 steps, top-k 1 and 6 draft tokens, an FP8 KV cache and the Triton DSA prefill and decode backends. Admission (max-running-requests) is 32, 48, 80 and 96, twice the concurrency, and the decode CUDA graph limit is 32, 48, 64 and 64. - The new image adds sgl-project/sglang#44322, which enables DCP on the Triton sparse-MLA backend on ROCm, including FP8 KV writes and MTP draft sharding. It also adds DSA fixes for MTP draft decode (#42565) and GLM-5.2 CUDA-graph side-stream replay (#45349). #43735 (DSA DCP with MTP) was already in v0.5.22. - Draft data types are unchanged: the MTP layer 78 ships in BF16 and is listed in the checkpoint's Quark exclude list, so SGLang loads it unquantized at both the old and new commits. Its KV cache is FP8, shared with the target. - 三个配置组(TP4/EP4 HiCache、TP4/EP4 DCP4、TP8/EP1)的镜像从 lmsysorg/sglang:v0.5.22-rocm720-mi35x(按 digest 固定的正式版)更新为 lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 nightly,基于 SGLang main e0df901d 构建。 - 新增 TP4/EP4 解码上下文并行(DCP4)测试点,并发 16、24、40、48。latent KV 分散在四个 rank 上,因此这些点无需 HiCache 即可把 KV 保留在 GPU 上(SGLang 不允许 DCP 与 HiCache 同时使用)。 - DCP4 测试点沿用基础配置:EAGLE 5 步、top-k 1、6 个草稿 token,FP8 KV cache,Triton DSA prefill/decode 后端。准入上限(max-running-requests)为 32、48、80、96,即并发的两倍;decode CUDA graph 上限为 32、48、64、64。 - 新镜像包含 sgl-project/sglang#44322,在 ROCm 的 Triton sparse-MLA 后端上启用 DCP,包括 FP8 KV 写入和 MTP 草稿分片;还包含 MTP 草稿解码的 DSA 修复(#42565)和 GLM-5.2 CUDA graph 侧流重放修复(#45349)。#43735(DSA DCP 搭配 MTP)在 v0.5.22 中已存在。 - 草稿模型数据类型不变:MTP 第 78 层以 BF16 发布,并列在 checkpoint 的 Quark exclude 列表中,因此新旧两个提交下 SGLang 都以非量化方式加载它。其 KV cache 为 FP8,与目标模型共享。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 36c6812b4b..941a4d6234 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11709,8 +11709,9 @@ scenario-type: - agentic-coding description: - - "Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48; the latent KV is striped across the four ranks, so these points run without HiCache, which DSA DCP rejects." - - "Keep the EAGLE draft with 5 steps, top-k 1 and 6 draft tokens, FP8 KV, the triton DSA prefill and decode backends, admission at twice the concurrency and a decode graph limit of min(2 x concurrency, 64)." - - "Validate decode-context-parallel-size against the matrix DCP size for SGLang recipes as well as vLLM and ATOM." - - "Use lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 for all three arms; the published nightly includes sgl-project/sglang#43735 and #44322, enabling Triton DSA DCP with FP8 KV and MTP on ROCm." + - "Update the image for all three arms (TP4/EP4 HiCache, TP4/EP4 DCP4, TP8/EP1) from lmsysorg/sglang:v0.5.22-rocm720-mi35x (digest-pinned release) to the lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 nightly, built from SGLang main e0df901d." + - "Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48. The latent KV is striped across the four ranks, so these points keep KV on the GPU without HiCache, which SGLang rejects with DCP." + - "DCP4 points keep the base recipe: EAGLE with 5 steps, top-k 1 and 6 draft tokens, an FP8 KV cache and the Triton DSA prefill and decode backends. Admission (max-running-requests) is 32, 48, 80 and 96, twice the concurrency, and the decode CUDA graph limit is 32, 48, 64 and 64." + - "The new image adds sgl-project/sglang#44322, which enables DCP on the Triton sparse-MLA backend on ROCm, including FP8 KV writes and MTP draft sharding. It also adds DSA fixes for MTP draft decode (#42565) and GLM-5.2 CUDA-graph side-stream replay (#45349). #43735 (DSA DCP with MTP) was already in v0.5.22." + - "Draft data types are unchanged: the MTP layer 78 ships in BF16 and is listed in the checkpoint's Quark exclude list, so SGLang loads it unquantized at both the old and new commits. Its KV cache is FP8, shared with the target." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4064