From dd65c3561bc9f5a32c979bdd96c5a264775cfdd6 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:27:57 +0000 Subject: [PATCH 1/3] [AMD] MiniMax-M3.1-Flash MI355X vLLM: latest nightly image and merged optimization flags Pin vllm/vllm-openai-rocm nightly-4fab5a07 (2026-11-06) by digest and enable the MiniMax-M3.1 ROCm optimizations merged since the day-zero image: drafter monokernel, parallel decode top-k, Rust frontend, INT4 quick all-reduce, skinny GEMM off, and breakable CUDA graphs at c16/c32. DBO is removed from c16-c64. Co-Authored-By: Claude Sonnet 5.5 --- .../vllm/mi355x-fp4/agentic.yaml | 53 +++++++++---------- inferencex-e2e/configs/amd-master.yaml | 8 +-- inferencex-e2e/perf-changelog.yaml | 11 ++++ 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml index 3d91e78cee..0466d191dd 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml @@ -1,12 +1,3 @@ -# MiniMax-M3.1-Flash AgentX on MI355X with vLLM, following the MI355X TP4 -# recipe at https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3.1-Flash and -# https://vllm.ai/blog/2026-10-23-minimax-m31-flash, with native DSpark. -# Checkpoint: AMD's Quark export amd/MiniMax-M3.1-Flash-Quark-MXFP4 (routed -# experts stored in OCP MXFP4, run as W4A4 on the AITER MoE; every other layer -# and the dspark/ draft as in MiniMaxAI/MiniMax-M3.1-Flash). Long prefill -# steps run with dual-batch overlap (DBO). Sequence-parallel prefill -# (VLLM_ROCM_MINIMAX_M3_SP_PREFILL) stays off: the image does not -# enable it on this checkpoint. base: schema: 2 name: minimaxm31flash-fp4-mi355x-vllm-agentic @@ -36,7 +27,7 @@ base: args: served-model-name: amd/MiniMax-M3.1-Flash-Quark-MXFP4 tensor-parallel-size: 4 - # Reserve headroom for DBO prefill and allocations after memory profiling. + # Reserve headroom for allocations after memory profiling. gpu-memory-utilization: 0.85 language-model-only: true trust-remote-code: true @@ -54,13 +45,7 @@ base: disable-hybrid-kv-cache-manager: true enable-prefix-caching: true max-model-len: 1048576 - # 32K-token prefill steps, so DBO (from 16384 tokens) can split them. max-num-batched-tokens: 32768 - # DBO: a prefill step of 16384+ tokens runs as two microbatches, one's - # all-reduce under the other's compute; decode steps stay whole. - enable-dbo: true - dbo-prefill-token-threshold: 16384 - dbo-decode-token-threshold: 32768 disable-uvicorn-access-log: true env: # The export's recommended MoE setup: AITER MoE, and the MXFP8 shared @@ -68,17 +53,27 @@ base: VLLM_ROCM_USE_AITER: '1' VLLM_ROCM_USE_AITER_MOE: '1' VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '0' - # DBO needs the V2 model runner (the V1 runner refuses DSpark). VLLM_USE_V2_MODEL_RUNNER: '1' - # Bind each HIP stream to one hardware queue. The image's HIP moves an - # active stream to the least loaded queue, which can put DBO's - # all-reduce stream on the compute stream's queue and serialize them. DEBUG_HIP_DYNAMIC_QUEUES: '0' PYTHONNOUSERSITE: '1' PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' # Outlast AIPerf's pooled-connection idle gaps. VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + # Rust API frontend from the image's vllm package: frees the Python + # API server from tokenization and streaming at AgentX request rates. + VLLM_USE_RUST_FRONTEND: '1' + # INT4 quick all-reduce: faster TP all-reduce on 16K+ token prefill steps. + VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: 'INT4' + # The draft LM head's skinny GEMM takes about 5x longer at exactly 5 + # requests than at 4 (0.28 vs 0.05 ms per call); the regular GEMM has no such cliff. + VLLM_ROCM_USE_SKINNY_GEMM: '0' + # Parallel top-16 block selection for the NVFP4 indexer; the FP8 indexer + # of the mono points bypasses it. + VLLM_ROCM_PARALLEL_INDEX_TOPK: '1' + # Fused FlyDSL kernels for DSpark 7 draft steps of up to 10 requests; + # the 3-token draft keeps the stock draft layers. + VLLM_ROCM_MINIMAX_M3_MONO_DRAFTER: '1' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' @@ -92,7 +87,7 @@ base: # and steps with prefill run on the normal path, eager. AgentX tool-call gaps # keep fewer requests decoding than CONC, so c4 and c8 still reach it. Mono # needs FP8 KV and indexer caches in the shuffled KV layout and no piecewise -# graphs. No DBO here: 8192-token prefill steps on the V1 runner. +# graphs. 8192-token prefill steps on the V1 runner. zip_override_tp4_dspark7_mono: roles: agg: @@ -102,9 +97,6 @@ zip_override_tp4_dspark7_mono: attention-config: '{"indexer_kv_dtype":"fp8"}' compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' max-num-batched-tokens: 8192 - enable-dbo: null - dbo-prefill-token-threshold: null - dbo-decode-token-threshold: null speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"TRITON_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' max-num-seqs: [2, 4, 8, 16] env: @@ -116,12 +108,19 @@ zip_override_tp4_dspark7_mono: env: CONC: ['1', '2', '4', '8'] -# c16-c32: DSpark 7 on the NVFP4 KV cache with DBO, at the base 0.85 memory -# budget (headroom for DBO prefill). Admission is 2x CONC. +# c16-c32: DSpark 7 on the NVFP4 KV cache. Breakable CUDA graphs capture the +# prefill and mixed steps too. The capture list (every multiple of 8 up to 256, +# then 384-8192; a coarse list makes decode about 10% slower) is a deprecated +# alias on the V2 runner that sets both the prefill token sizes and the decode +# request sizes (1-32 at c16); the new per-kind fields would change the decode +# graphs. Admission is 2x CONC. zip_override_tp4_dspark7: roles: agg: + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' args: + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,384,512,768,1024,1536,2048,3072,4096,6144,8192]}' speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"TRITON_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' max-num-seqs: [32, 64] benchmark: @@ -129,7 +128,7 @@ zip_override_tp4_dspark7: CONC: ['16', '32'] # c64: DSpark 3 (the blog's MI355X draft length from c8) on the NVFP4 KV cache -# with DBO, sequence-parallel prefill, 0.90 memory utilization, and a CPU KV tier. The c64 working set +# with sequence-parallel prefill, 0.90 memory utilization, and a CPU KV tier. The c64 working set # outgrows the 14.2M-token GPU KV cache; SimpleCPUOffloadConnector (eager) # holds about 24M tokens in 1349 GB of host memory (dram-utilization 0.9 of the # TP4 lane) and serves the misses. diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 9e9e23dc1a..bb9ae854df 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -904,14 +904,8 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark-tp8-c1: search-space: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1], srt-recipe: agentic.yaml } -# MiniMax-M3.1-Flash AgentX on MI355X: day-zero vLLM ROCm image from -# https://vllm.ai/blog/2026-10-23-minimax-m31-flash, verified 2026-10-23. -# AMD's Quark MXFP4 export (routed experts in MXFP4, W4A4 on the AITER MoE) -# at TP4 with native DSpark: 7 drafts, mono decode and an FP8 KV cache at -# c1-c8; 7 drafts, an NVFP4 KV cache and DBO at c16-c32; 3 drafts, DBO and a -# CPU KV tier at c64. Throughput runs use the minimax-m3.1-flash__dspark__native__probabilistic-block golden AL. minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:minimax-m31-flash@sha256:ae8681212aa5291d8c27f06e90016435e3fca144a68fd5c439231f6475f74569 + image: vllm/vllm-openai-rocm:nightly-4fab5a07475822c6eaa971dceb2e2220d0446808@sha256:6f126f27a3b58959da3c16076248fc99eb31e26d0a3ae9f8563bda7317d6a7e6 model: amd/MiniMax-M3.1-Flash-Quark-MXFP4 model-prefix: minimaxm31flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 42401e2a75..dcfe030ae4 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12293,3 +12293,14 @@ - "Add native FP8 GLM-5.3-Flash AgentX on MI355X with the official SGLang v0.5.22 ROCm image pinned by digest and the SGLang cookbook MI355X FP8 Low Latency recipe: TP4/EP4, AITER MoE, FP8 KV with TileLang DSA, embedded NextN MTP 5/1/6 served as shipped." - "HBM-only KV at conc 1, 2, 8, 16, 24 and 32; the cookbook HiCache L1 + L2 tier (180 GB per rank, write-through) at conc 40 and 48 recording its 749.33 GB host allocation (187.33 GB per rank) as a 751 GB CPU DRAM share. ROCM_QUICK_REDUCE_QUANTIZATION=NONE per the all-reduce policy." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4720 + +- config-keys: + - minimaxm31flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: [agentic-coding] + description: + - "Image vllm/vllm-openai-rocm:minimax-m31-flash (vLLM 0.32.0rc1.dev364+g8d0f05a0c) -> nightly-4fab5a07475822c6eaa971dceb2e2220d0446808 (vLLM main of 2026-11-05), pinned by digest. It carries the MiniMax-M3.1 ROCm work merged since: decode CP (#64448), qkv input-norm quant fusion (#63637), four-wave decode top-k (#65087), local DSpark 7 indexer scoring below 3 requests (#65432), FFN-side MXFP8 quant fusion (#65478), sparse-decode splits per XCD (#65481), MXFP8 GEMM tile retune for M=17-256 (#65530), DSpark drafter monokernel (#65386) and opt-in parallel top-k (#65267)." + - "New env at every point: VLLM_ROCM_MINIMAX_M3_MONO_DRAFTER=1 (fused FlyDSL DSpark 7 draft steps of up to 10 requests; the c64 3-token draft keeps the stock layers), VLLM_ROCM_PARALLEL_INDEX_TOPK=1 (NVFP4 indexer points; the FP8 indexer of c1-c8 bypasses it), VLLM_USE_RUST_FRONTEND=1, VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 and VLLM_ROCM_USE_SKINNY_GEMM=0 (removes the draft LM head's 5-request skinny-GEMM cliff)." + - "C16/C32: breakable CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1, FULL_AND_PIECEWISE with a multiple-of-8 capture list up to 256, then 384-8192). DBO is removed from C16/C32/C64 (C1-C8 already ran without it); the 32768-token prefill budget, max-num-seqs and memory utilization are unchanged." + - "C1-C8 (DSpark 7, mono decode, FP8 KV and indexer caches) and C64 (DSpark 3, NVFP4 KV, CPU KV offload) keep their speculative, KV and offload settings." + - "Draft data types unchanged: the dspark/ draft keeps its MXFP8 config (FP8 E4M3 projections with 1x32 UE8M0 scales; BF16 fc, Markov and confidence heads, norms) and an FP8 draft KV cache. The drafter kernels replace the draft layers' MXFP8 linear, norm and attention ops, and vLLM's tests/models/minimax_m3/test_dspark_drafter.py holds them to the stock layers' own numerical spread." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD From 6aec58ec51503978fec5b8414de3cb750a357e00 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:28:26 +0000 Subject: [PATCH 2/3] Link the performance changelog entry to this PR Co-Authored-By: Claude Sonnet 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index dcfe030ae4..2e64c4ed7c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12303,4 +12303,4 @@ - "C16/C32: breakable CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1, FULL_AND_PIECEWISE with a multiple-of-8 capture list up to 256, then 384-8192). DBO is removed from C16/C32/C64 (C1-C8 already ran without it); the 32768-token prefill budget, max-num-seqs and memory utilization are unchanged." - "C1-C8 (DSpark 7, mono decode, FP8 KV and indexer caches) and C64 (DSpark 3, NVFP4 KV, CPU KV offload) keep their speculative, KV and offload settings." - "Draft data types unchanged: the dspark/ draft keeps its MXFP8 config (FP8 E4M3 projections with 1x32 UE8M0 scales; BF16 fc, Markov and confidence heads, norms) and an FP8 draft KV cache. The drafter kernels replace the draft layers' MXFP8 linear, norm and attention ops, and vLLM's tests/models/minimax_m3/test_dspark_drafter.py holds them to the stock layers' own numerical spread." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4866 From ce4079c48b85472cc4be60f40c0fa1f7f067e8e9 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:17:47 +0000 Subject: [PATCH 3/3] Drop INT4 quick all-reduce: the nightly image's AITER lacks QuickAllReduceInt4.compile Workers failed in init_device at every point (first CI run, c1). Co-Authored-By: Claude Sonnet 5.5 --- .../minimaxm31flash/vllm/mi355x-fp4/agentic.yaml | 2 -- inferencex-e2e/perf-changelog.yaml | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml index 0466d191dd..7a368e46d5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml @@ -63,8 +63,6 @@ base: # Rust API frontend from the image's vllm package: frees the Python # API server from tokenization and streaming at AgentX request rates. VLLM_USE_RUST_FRONTEND: '1' - # INT4 quick all-reduce: faster TP all-reduce on 16K+ token prefill steps. - VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: 'INT4' # The draft LM head's skinny GEMM takes about 5x longer at exactly 5 # requests than at 4 (0.28 vs 0.05 ms per call); the regular GEMM has no such cliff. VLLM_ROCM_USE_SKINNY_GEMM: '0' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2e64c4ed7c..1da9aa21bf 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12299,8 +12299,8 @@ scenario-type: [agentic-coding] description: - "Image vllm/vllm-openai-rocm:minimax-m31-flash (vLLM 0.32.0rc1.dev364+g8d0f05a0c) -> nightly-4fab5a07475822c6eaa971dceb2e2220d0446808 (vLLM main of 2026-11-05), pinned by digest. It carries the MiniMax-M3.1 ROCm work merged since: decode CP (#64448), qkv input-norm quant fusion (#63637), four-wave decode top-k (#65087), local DSpark 7 indexer scoring below 3 requests (#65432), FFN-side MXFP8 quant fusion (#65478), sparse-decode splits per XCD (#65481), MXFP8 GEMM tile retune for M=17-256 (#65530), DSpark drafter monokernel (#65386) and opt-in parallel top-k (#65267)." - - "New env at every point: VLLM_ROCM_MINIMAX_M3_MONO_DRAFTER=1 (fused FlyDSL DSpark 7 draft steps of up to 10 requests; the c64 3-token draft keeps the stock layers), VLLM_ROCM_PARALLEL_INDEX_TOPK=1 (NVFP4 indexer points; the FP8 indexer of c1-c8 bypasses it), VLLM_USE_RUST_FRONTEND=1, VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 and VLLM_ROCM_USE_SKINNY_GEMM=0 (removes the draft LM head's 5-request skinny-GEMM cliff)." - - "C16/C32: breakable CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1, FULL_AND_PIECEWISE with a multiple-of-8 capture list up to 256, then 384-8192). DBO is removed from C16/C32/C64 (C1-C8 already ran without it); the 32768-token prefill budget, max-num-seqs and memory utilization are unchanged." + - "New env at every point: VLLM_ROCM_MINIMAX_M3_MONO_DRAFTER=1 (fused FlyDSL DSpark 7 draft steps of up to 10 requests; the c64 3-token draft keeps the stock layers), VLLM_ROCM_PARALLEL_INDEX_TOPK=1 (NVFP4 indexer points; the FP8 indexer of c1-c8 bypasses it), VLLM_USE_RUST_FRONTEND=1 and VLLM_ROCM_USE_SKINNY_GEMM=0 (removes the draft LM head's 5-request skinny-GEMM cliff)." + - "C16/C32: breakable CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1, FULL_AND_PIECEWISE with a multiple-of-8 capture list up to 256, then 384-8192). INT4 quick all-reduce stays off: the image's AITER lacks the QuickAllReduceInt4.compile that vLLM's FlyDSL INT4 path calls, so workers fail at startup. DBO is removed from C16/C32/C64 (C1-C8 already ran without it); the 32768-token prefill budget, max-num-seqs and memory utilization are unchanged." - "C1-C8 (DSpark 7, mono decode, FP8 KV and indexer caches) and C64 (DSpark 3, NVFP4 KV, CPU KV offload) keep their speculative, KV and offload settings." - "Draft data types unchanged: the dspark/ draft keeps its MXFP8 config (FP8 E4M3 projections with 1x32 UE8M0 scales; BF16 fc, Markov and confidence heads, norms) and an FP8 draft KV cache. The drafter kernels replace the draft layers' MXFP8 linear, norm and attention ops, and vLLM's tests/models/minimax_m3/test_dspark_drafter.py holds them to the stock layers' own numerical spread." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4866