diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml index 5e3e101597..fb52811040 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -152,6 +152,67 @@ override_tp4_c16_hicache: CONC: '16' KV_OFFLOADING: dram +# DSA DCP rejects HiCache; DCP4 stripes the latent KV across the TP4 ranks instead. +override_tp4_dcp4_c16: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none + +override_tp4_dcp4_c24: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 48 + cuda-graph-max-bs-decode: 48 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: none + +override_tp4_dcp4_c40: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 80 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '40' + KV_OFFLOADING: none + +override_tp4_dcp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + decode-context-parallel-size: 4 + max-running-requests: 96 + cuda-graph-max-bs-decode: 64 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: none + override_tp8_c1: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 7a26f6ea4a..2a48b134b3 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1088,16 +1088,19 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark: - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Two arms: +# decoding. Three arms: # (1) TP4/EP4 with HiCache KV offloading to DRAM at concurrency [4, 8, 10, 12, 14, 16]; # serves the high-concurrency, throughput-dominant regime. -# (2) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4]; +# (2) TP4/EP4 with DCP4 at concurrency [16, 24, 40, 48]; striping the latent KV +# across the four ranks holds the working set on device without HiCache, +# which DSA DCP rejects. +# (3) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4]; # EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid # concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. -# c4 intentionally overlaps both arms to compare the topology crossover. +# c4 intentionally overlaps arms (1) and (3) to compare the topology crossover. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang:v0.5.22-rocm720-mi35x@sha256:c4b61360b251d62bb17da63f175d571ab22479eb46a144548d79f3ed3998ea11 + image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds @@ -1110,6 +1113,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.85 search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, dcp-size: 4, kv-offloading: none, conc-list: [16, 24, 40, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } # GLM-5.2 FP8 agentic-coding benchmark on MI355X via SGLang with MTP speculative diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b0cd1b0b76..2a55b3f48d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11770,6 +11770,18 @@ - "Prefill: shortest-prefill-first, chunk 8192, mem-fraction-static 0.85, HiCache 200 GB per rank write_back." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4686 +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image for all three arms (TP4/EP4 HiCache, TP4/EP4 DCP4, TP8/EP1) from lmsysorg/sglang:v0.5.22-rocm720-mi35x (digest-pinned release) to the lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261028 nightly, built from SGLang main e0df901d." + - "Add TP4/EP4 points with decode context parallelism (DCP4) at concurrency 16, 24, 40 and 48. The latent KV is striped across the four ranks, so these points keep KV on the GPU without HiCache, which SGLang rejects with DCP." + - "DCP4 points keep the base recipe: EAGLE with 5 steps, top-k 1 and 6 draft tokens, an FP8 KV cache and the Triton DSA prefill and decode backends. Admission (max-running-requests) is 32, 48, 80 and 96, twice the concurrency, and the decode CUDA graph limit is 32, 48, 64 and 64." + - "The new image adds sgl-project/sglang#44322, which enables DCP on the Triton sparse-MLA backend on ROCm, including FP8 KV writes and MTP draft sharding. It also adds DSA fixes for MTP draft decode (#42565) and GLM-5.2 CUDA-graph side-stream replay (#45349). #43735 (DSA DCP with MTP) was already in v0.5.22." + - "Draft data types are unchanged: the MTP layer 78 ships in BF16 and is listed in the checkpoint's Quark exclude list, so SGLang loads it unquantized at both the old and new commits. Its KV cache is FP8, shared with the target." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4064 + - config-keys: - minimaxm31flash-fp4-mi355x-sglang-agentic-dspark description: