From a5db3a3e7e42db8e39fa43b7997bc5a72f2c2187 Mon Sep 17 00:00:00 2001 From: zhuyuhua-v Date: Thu, 29 Oct 2026 07:47:05 -0500 Subject: [PATCH 1/2] feat(agentx): run GLM-5.2 FP4 MI355X ATOM TP8 C1/C2 on mono decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track the TP8 recipe that ROCm/ATOM PR 2736 adds to recipes/Agentic-GLM-5.2.md, on rocm/atom-dev:nightly_202610271459 (ATOM 06dbf6692), the first nightly that contains it. - TP8 C1/C2 run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens (AL 2.99), max-num-seqs equal to CONC, graphs [1] and [1,2]: the TP4 C1/C2 command with -tp 8. TP8 C4 and every TP4/DCP4 point keep their settings and rerun only for the image. - The atomesh router digest moves to the same nightly (5332ae78). 中文：按 ROCm/ATOM PR 2736 加入 recipes/Agentic-GLM-5.2.md 的 TP8 配方更新 MI355X ATOM AgentX 配置，镜像换为首个包含该 PR 的 rocm/atom-dev:nightly_202610271459（ATOM 06dbf6692）。TP8 C1/C2 使用 GLM-5.2 mono decode（ATOM_MONO_ENABLE=1），3 个 draft token（AL 2.99）， max-num-seqs 等于并发数，graph 为 [1] 和 [1,2]，即 TP4 C1/C2 命令改为 -tp 8。TP8 C4 及所有 TP4/DCP4 点位设置不变，只因换镜像而重跑。atomesh router digest 同步为同一 nightly（5332ae78）。 Signed-off-by: zhuyuhua-v --- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 24 +++++++++++-------- inferencex-e2e/configs/amd-master.yaml | 5 ++-- inferencex-e2e/perf-changelog.yaml | 11 +++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 65ed6c5782..4b1d466480 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:1bb1909ada9e3d178e315a3f89d041c05c7e90e7cc32de8d507ff5ab4cd4702c + container_image: rocm/atom-dev@sha256:5332ae78c51985d1bd8462935bdd80817f12350c39f8e8db7d93ab9037ff5f1c enable_multiple_frontends: false observability: enabled: false @@ -33,15 +33,15 @@ base: PYTHONNOUSERSITE: '1' AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: '1' - # GLM-5.2 mono decode is on by default in ATOM; only TP4 C1/C2 run it. + # GLM-5.2 mono decode is on by default in ATOM; only TP4/TP8 C1/C2 run it. ATOM_MONO_ENABLE: '0' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' -# One variant per point. Admission is 2x CONC, except TP4 C1/C2, which admit -# exactly CONC with graphs captured up to it. TP4 C1/C2 run mono decode at +# One variant per point. Admission is 2x CONC, except TP4/TP8 C1/C2, which admit +# exactly CONC with graphs captured up to it. TP4/TP8 C1/C2 run mono decode at # three draft tokens; C4/C8 draft five and C10 four on the production path. The # DCP4 arm shards the decode KV, adds the LMCache DRAM tier at 256 GiB per rank # (1024 GiB, 1099.5 GB per server, within the 1109 GB TP4 share at @@ -210,9 +210,11 @@ override_tp8_c1: agg: gpus: 8 args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' - num-speculative-tokens: 5 + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' benchmark: env: CONC: '1' @@ -222,9 +224,11 @@ override_tp8_c2: agg: gpus: 8 args: - max-num-seqs: 4 - cudagraph-capture-sizes: '[1,2,4]' - num-speculative-tokens: 5 + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' benchmark: env: CONC: '2' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ed18bec2d6..792862e6ac 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1495,9 +1495,10 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: # C1/C2 run the GLM-5.2 mono decode at K3. Large concurrency (C16-C48) adds DCP4 # to shard the decode KV and the LMCache DRAM tier. The recipe sets the draft # depth per point (K3 for C1/C2 and C48, K5 for C4/C8, K4 for C10-C40). The TP8 -# arm stays GPU-resident with no KV offload. +# arm stays GPU-resident with no KV offload; TP8 C1/C2 also run mono decode at +# K3 (ROCm/ATOM PR 2736) and C4 stays at K5. glm5.2-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202610181506 + image: rocm/atom-dev:nightly_202610271459 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 18e8e72c5c..49ad7269d4 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11556,3 +11556,14 @@ - "Disagg: 6 DEP2 prefill workers with a DEP8 decode at conc 280-360 or a DEP16 decode at conc 300; DeepGEMM MegaMoE, NIXL transfer plus a shared MooncakeStore DRAM pool (160 GB per rank), Dynamo KV router." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4468 scenario-type: [agentic-coding] + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Image rocm/atom-dev:nightly_202610181506 (ATOM 0.1.8.dev77+gdf86c4829) -> nightly_202610271459 (ATOM 0.1.9.dev59+g06dbf6692, digest sha256:5332ae78c51985d1bd8462935bdd80817f12350c39f8e8db7d93ab9037ff5f1c), the first nightly that contains ROCm/ATOM PR 2736. The atomesh router container_image digest moves from 1bb1909a to 5332ae78, the same nightly as the workers." + - "TP8 C1 and C2 follow the TP8 recipe added to recipes/Agentic-GLM-5.2.md by ROCm/ATOM PR 2736: the TP4 C1/C2 command with -tp 8. Both run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) on the external-indexer path at 3 draft tokens (golden AL 2.99), with max-num-seqs equal to the concurrency and CUDA graphs at [1] and [1,2]. C1 was 5 draft tokens, max-num-seqs 2 and graphs [1,2]; C2 was 5 draft tokens, max-num-seqs 4 and graphs [1,2,4]." + - "TP8 C4, TP4 C1-C10 and TP4+DCP4 C16-C48 keep their settings and ATOM_MONO_ENABLE values; they rerun only for the image change. LMCache stays at 0.5.6.dev139+gf22dec28, the wheel pinned in both images." + - "ATOM changes in the new image that matter here: mono decode admits TP8 and keeps the fused indexer at TP4 only (ROCm/ATOM #2736). Shared paths the TP4, DCP4 and TP8 C4 points also pick up: one graph mechanism per region, including the glm5_next indexer and kpool (#2688); MoE routing before the rows move (#2725); chunked DSA indexer decode logits (#2607); speculative output buffer reuse (#2670); MLA Triton decode on the maintained AITER API (#2669)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4608 From 1e7bb50eab0afee4f5c189fbc18b4b13ac2aab5f Mon Sep 17 00:00:00 2001 From: zhuyuhua-v Date: Wed, 4 Nov 2026 10:00:07 -0600 Subject: [PATCH 2/2] feat(agentx): serve GLM-5.2 MI355X ATOM on the Quark MXFP4 AttnFP8 checkpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch glm5.2-fp4-mi355x-atom-agentic-mtp from amd/GLM-5.2-MXFP4 to amd/GLM-5.2-Quark-MXFP4-AttnFP8 and serve it as shipped. - Drop online_quant_config. The checkpoint already stores attention and the indexer wq_b/wk in PTPC FP8 and every other linear in MXFP4, including the layer 0-2 dense MLP and the MTP layer 78; lm_head, eh_proj, the routers and indexer.weights_proj stay BF16. - Drop AITER_QUICK_REDUCE_QUANTIZATION=INT4, so all-reduce is no longer integer-quantized. 中文：glm5.2-fp4-mi355x-atom-agentic-mtp 的模型从 amd/GLM-5.2-MXFP4 换成 amd/GLM-5.2-Quark-MXFP4-AttnFP8，按 checkpoint 原样服务：去掉 online_quant_config（该 checkpoint 已把 attention 和 indexer wq_b/wk 存为 PTPC FP8，其余线性层包括 layer 0-2 dense MLP 和 MTP 第 78 层均为 MXFP4； lm_head、eh_proj、router 和 indexer.weights_proj 保持 BF16），并去掉 AITER_QUICK_REDUCE_QUANTIZATION=INT4，all-reduce 不再做整数量化。 Signed-off-by: zhuyuhua-v --- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 6 +----- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 5 ++++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 4b1d466480..82ac611d0a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -18,12 +18,9 @@ base: nodes: 1 workers: 1 args: - served-model-name: amd/GLM-5.2-MXFP4 + served-model-name: amd/GLM-5.2-Quark-MXFP4-AttnFP8 gpu-memory-utilization: 0.95 enable_prefix_caching: true - # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps - # online ptpc_fp8 quantization off the draft while experts stay MXFP4. - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' max-num-batched-tokens: 16384 kv_cache_dtype: fp8 index_cache_dtype: fp4 @@ -31,7 +28,6 @@ base: method: mtp env: PYTHONNOUSERSITE: '1' - AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: '1' # GLM-5.2 mono decode is on by default in ATOM; only TP4/TP8 C1/C2 run it. ATOM_MONO_ENABLE: '0' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 9686edf265..907ce30a1b 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -852,7 +852,7 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: # K3 (ROCm/ATOM PR 2736) and C4 stays at K5. glm5.2-fp4-mi355x-atom-agentic-mtp: image: rocm/atom-dev:nightly_202610271459 - model: amd/GLM-5.2-MXFP4 + model: amd/GLM-5.2-Quark-MXFP4-AttnFP8 model-prefix: glm5.2 runner: cluster:mi355x-amds precision: fp4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 0dc034df76..2c5978a464 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11996,7 +11996,10 @@ - agentic-coding description: - "Image rocm/atom-dev:nightly_202610181506 (ATOM 0.1.8.dev77+gdf86c4829) -> nightly_202610271459 (ATOM 0.1.9.dev59+g06dbf6692, digest sha256:5332ae78c51985d1bd8462935bdd80817f12350c39f8e8db7d93ab9037ff5f1c), the first nightly that contains ROCm/ATOM PR 2736. The atomesh router container_image digest moves from 1bb1909a to 5332ae78, the same nightly as the workers." + - "Checkpoint amd/GLM-5.2-MXFP4 -> amd/GLM-5.2-Quark-MXFP4-AttnFP8 (Hugging Face revision 985ea5ad), served as shipped with no online_quant_config. It stores attention q_a/q_b/kv_a/kv_b/o_proj and the indexer wq_b/wk in PTPC FP8 (static per-channel weights, dynamic per-token activations) and every other linear in MXFP4, including the routed and shared experts, the layer 0-2 dense MLP and the MTP layer 78; lm_head, eh_proj, the MoE routers and indexer.weights_proj stay BF16. The previous recipe quantized the BF16 attention, indexer and dense MLP of amd/GLM-5.2-MXFP4 to PTPC FP8 at load time through online_quant_config." + - "Remove AITER_QUICK_REDUCE_QUANTIZATION=INT4, so tensor-parallel all-reduce is no longer integer-quantized. In the pinned AITER, INT4 quick reduce took BF16 messages of at least 16 MiB at TP4 (about 1,366 tokens of the 6144-wide hidden state, so prefill steps only) and at least 2 GiB at TP8, which no GLM-5.2 step reaches." + - "Draft: the MTP head is the checkpoint's own layer 78 at its stored precision (MXFP4 experts and MLP, PTPC FP8 attention, BF16 eh_proj), with no draft-related quantization setting in the recipe. It was BF16 in amd/GLM-5.2-MXFP4." - "TP8 C1 and C2 follow the TP8 recipe added to recipes/Agentic-GLM-5.2.md by ROCm/ATOM PR 2736: the TP4 C1/C2 command with -tp 8. Both run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) on the external-indexer path at 3 draft tokens (golden AL 2.99), with max-num-seqs equal to the concurrency and CUDA graphs at [1] and [1,2]. C1 was 5 draft tokens, max-num-seqs 2 and graphs [1,2]; C2 was 5 draft tokens, max-num-seqs 4 and graphs [1,2,4]." - - "TP8 C4, TP4 C1-C10 and TP4+DCP4 C16-C48 keep their settings and ATOM_MONO_ENABLE values; they rerun only for the image change. LMCache stays at 0.5.6.dev139+gf22dec28, the wheel pinned in both images." + - "TP8 C4, TP4 C1-C10 and TP4+DCP4 C16-C48 keep their settings and ATOM_MONO_ENABLE values; they rerun for the image and checkpoint changes. LMCache stays at 0.5.6.dev139+gf22dec28, the wheel pinned in both images." - "ATOM changes in the new image that matter here: mono decode admits TP8 and keeps the fused indexer at TP4 only (ROCm/ATOM #2736). Shared paths the TP4, DCP4 and TP8 C4 points also pick up: one graph mechanism per region, including the glm5_next indexer and kpool (#2688); MoE routing before the rows move (#2725); chunked DSA indexer decode logits (#2607); speculative output buffer reuse (#2670); MLA Triton decode on the maintained AITER API (#2669)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4608