diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 65ed6c5782..82ac611d0a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:1bb1909ada9e3d178e315a3f89d041c05c7e90e7cc32de8d507ff5ab4cd4702c + container_image: rocm/atom-dev@sha256:5332ae78c51985d1bd8462935bdd80817f12350c39f8e8db7d93ab9037ff5f1c enable_multiple_frontends: false observability: enabled: false @@ -18,12 +18,9 @@ base: nodes: 1 workers: 1 args: - served-model-name: amd/GLM-5.2-MXFP4 + served-model-name: amd/GLM-5.2-Quark-MXFP4-AttnFP8 gpu-memory-utilization: 0.95 enable_prefix_caching: true - # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps - # online ptpc_fp8 quantization off the draft while experts stay MXFP4. - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' max-num-batched-tokens: 16384 kv_cache_dtype: fp8 index_cache_dtype: fp4 @@ -31,17 +28,16 @@ base: method: mtp env: PYTHONNOUSERSITE: '1' - AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: '1' - # GLM-5.2 mono decode is on by default in ATOM; only TP4 C1/C2 run it. + # GLM-5.2 mono decode is on by default in ATOM; only TP4/TP8 C1/C2 run it. ATOM_MONO_ENABLE: '0' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' -# One variant per point. Admission is 2x CONC, except TP4 C1/C2, which admit -# exactly CONC with graphs captured up to it. TP4 C1/C2 run mono decode at +# One variant per point. Admission is 2x CONC, except TP4/TP8 C1/C2, which admit +# exactly CONC with graphs captured up to it. TP4/TP8 C1/C2 run mono decode at # three draft tokens; C4/C8 draft five and C10 four on the production path. The # DCP4 arm shards the decode KV, adds the LMCache DRAM tier at 256 GiB per rank # (1024 GiB, 1099.5 GB per server, within the 1109 GB TP4 share at @@ -210,9 +206,11 @@ override_tp8_c1: agg: gpus: 8 args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' - num-speculative-tokens: 5 + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' benchmark: env: CONC: '1' @@ -222,9 +220,11 @@ override_tp8_c2: agg: gpus: 8 args: - max-num-seqs: 4 - cudagraph-capture-sizes: '[1,2,4]' - num-speculative-tokens: 5 + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' benchmark: env: CONC: '2' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 8df246d796..907ce30a1b 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -848,10 +848,11 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: # C1/C2 run the GLM-5.2 mono decode at K3. Large concurrency (C16-C48) adds DCP4 # to shard the decode KV and the LMCache DRAM tier. The recipe sets the draft # depth per point (K3 for C1/C2 and C48, K5 for C4/C8, K4 for C10-C40). The TP8 -# arm stays GPU-resident with no KV offload. +# arm stays GPU-resident with no KV offload; TP8 C1/C2 also run mono decode at +# K3 (ROCm/ATOM PR 2736) and C4 stays at K5. glm5.2-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202610181506 - model: amd/GLM-5.2-MXFP4 + image: rocm/atom-dev:nightly_202610271459 + model: amd/GLM-5.2-Quark-MXFP4-AttnFP8 model-prefix: glm5.2 runner: cluster:mi355x-amds precision: fp4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8768cfe03a..2c5978a464 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11989,3 +11989,17 @@ - "Update the Muse Spark Open GB300 curve to lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc and retune request admission, prefill-decode interval, scheduling and host cache per concurrency." - "Add TP4/EP4 concurrency 4 and 40; drop concurrency 5, 28 and 34." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4724 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Image rocm/atom-dev:nightly_202610181506 (ATOM 0.1.8.dev77+gdf86c4829) -> nightly_202610271459 (ATOM 0.1.9.dev59+g06dbf6692, digest sha256:5332ae78c51985d1bd8462935bdd80817f12350c39f8e8db7d93ab9037ff5f1c), the first nightly that contains ROCm/ATOM PR 2736. The atomesh router container_image digest moves from 1bb1909a to 5332ae78, the same nightly as the workers." + - "Checkpoint amd/GLM-5.2-MXFP4 -> amd/GLM-5.2-Quark-MXFP4-AttnFP8 (Hugging Face revision 985ea5ad), served as shipped with no online_quant_config. It stores attention q_a/q_b/kv_a/kv_b/o_proj and the indexer wq_b/wk in PTPC FP8 (static per-channel weights, dynamic per-token activations) and every other linear in MXFP4, including the routed and shared experts, the layer 0-2 dense MLP and the MTP layer 78; lm_head, eh_proj, the MoE routers and indexer.weights_proj stay BF16. The previous recipe quantized the BF16 attention, indexer and dense MLP of amd/GLM-5.2-MXFP4 to PTPC FP8 at load time through online_quant_config." + - "Remove AITER_QUICK_REDUCE_QUANTIZATION=INT4, so tensor-parallel all-reduce is no longer integer-quantized. In the pinned AITER, INT4 quick reduce took BF16 messages of at least 16 MiB at TP4 (about 1,366 tokens of the 6144-wide hidden state, so prefill steps only) and at least 2 GiB at TP8, which no GLM-5.2 step reaches." + - "Draft: the MTP head is the checkpoint's own layer 78 at its stored precision (MXFP4 experts and MLP, PTPC FP8 attention, BF16 eh_proj), with no draft-related quantization setting in the recipe. It was BF16 in amd/GLM-5.2-MXFP4." + - "TP8 C1 and C2 follow the TP8 recipe added to recipes/Agentic-GLM-5.2.md by ROCm/ATOM PR 2736: the TP4 C1/C2 command with -tp 8. Both run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) on the external-indexer path at 3 draft tokens (golden AL 2.99), with max-num-seqs equal to the concurrency and CUDA graphs at [1] and [1,2]. C1 was 5 draft tokens, max-num-seqs 2 and graphs [1,2]; C2 was 5 draft tokens, max-num-seqs 4 and graphs [1,2,4]." + - "TP8 C4, TP4 C1-C10 and TP4+DCP4 C16-C48 keep their settings and ATOM_MONO_ENABLE values; they rerun for the image and checkpoint changes. LMCache stays at 0.5.6.dev139+gf22dec28, the wheel pinned in both images." + - "ATOM changes in the new image that matter here: mono decode admits TP8 and keeps the fused indexer at TP4 only (ROCm/ATOM #2736). Shared paths the TP4, DCP4 and TP8 C4 points also pick up: one graph mechanism per region, including the glm5_next indexer and kpool (#2688); MoE routing before the rows move (#2725); chunked DSA indexer decode logits (#2607); speculative output buffer reuse (#2670); MLA Triton decode on the maintained AITER API (#2669)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4608