diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml index 3d91e78cee..7a368e46d5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/mi355x-fp4/agentic.yaml @@ -1,12 +1,3 @@ -# MiniMax-M3.1-Flash AgentX on MI355X with vLLM, following the MI355X TP4 -# recipe at https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3.1-Flash and -# https://vllm.ai/blog/2026-10-23-minimax-m31-flash, with native DSpark. -# Checkpoint: AMD's Quark export amd/MiniMax-M3.1-Flash-Quark-MXFP4 (routed -# experts stored in OCP MXFP4, run as W4A4 on the AITER MoE; every other layer -# and the dspark/ draft as in MiniMaxAI/MiniMax-M3.1-Flash). Long prefill -# steps run with dual-batch overlap (DBO). Sequence-parallel prefill -# (VLLM_ROCM_MINIMAX_M3_SP_PREFILL) stays off: the image does not -# enable it on this checkpoint. base: schema: 2 name: minimaxm31flash-fp4-mi355x-vllm-agentic @@ -36,7 +27,7 @@ base: args: served-model-name: amd/MiniMax-M3.1-Flash-Quark-MXFP4 tensor-parallel-size: 4 - # Reserve headroom for DBO prefill and allocations after memory profiling. + # Reserve headroom for allocations after memory profiling. gpu-memory-utilization: 0.85 language-model-only: true trust-remote-code: true @@ -54,13 +45,7 @@ base: disable-hybrid-kv-cache-manager: true enable-prefix-caching: true max-model-len: 1048576 - # 32K-token prefill steps, so DBO (from 16384 tokens) can split them. max-num-batched-tokens: 32768 - # DBO: a prefill step of 16384+ tokens runs as two microbatches, one's - # all-reduce under the other's compute; decode steps stay whole. - enable-dbo: true - dbo-prefill-token-threshold: 16384 - dbo-decode-token-threshold: 32768 disable-uvicorn-access-log: true env: # The export's recommended MoE setup: AITER MoE, and the MXFP8 shared @@ -68,17 +53,25 @@ base: VLLM_ROCM_USE_AITER: '1' VLLM_ROCM_USE_AITER_MOE: '1' VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: '0' - # DBO needs the V2 model runner (the V1 runner refuses DSpark). VLLM_USE_V2_MODEL_RUNNER: '1' - # Bind each HIP stream to one hardware queue. The image's HIP moves an - # active stream to the least loaded queue, which can put DBO's - # all-reduce stream on the compute stream's queue and serialize them. DEBUG_HIP_DYNAMIC_QUEUES: '0' PYTHONNOUSERSITE: '1' PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' # Outlast AIPerf's pooled-connection idle gaps. VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + # Rust API frontend from the image's vllm package: frees the Python + # API server from tokenization and streaming at AgentX request rates. + VLLM_USE_RUST_FRONTEND: '1' + # The draft LM head's skinny GEMM takes about 5x longer at exactly 5 + # requests than at 4 (0.28 vs 0.05 ms per call); the regular GEMM has no such cliff. + VLLM_ROCM_USE_SKINNY_GEMM: '0' + # Parallel top-16 block selection for the NVFP4 indexer; the FP8 indexer + # of the mono points bypasses it. + VLLM_ROCM_PARALLEL_INDEX_TOPK: '1' + # Fused FlyDSL kernels for DSpark 7 draft steps of up to 10 requests; + # the 3-token draft keeps the stock draft layers. + VLLM_ROCM_MINIMAX_M3_MONO_DRAFTER: '1' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' @@ -92,7 +85,7 @@ base: # and steps with prefill run on the normal path, eager. AgentX tool-call gaps # keep fewer requests decoding than CONC, so c4 and c8 still reach it. Mono # needs FP8 KV and indexer caches in the shuffled KV layout and no piecewise -# graphs. No DBO here: 8192-token prefill steps on the V1 runner. +# graphs. 8192-token prefill steps on the V1 runner. zip_override_tp4_dspark7_mono: roles: agg: @@ -102,9 +95,6 @@ zip_override_tp4_dspark7_mono: attention-config: '{"indexer_kv_dtype":"fp8"}' compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' max-num-batched-tokens: 8192 - enable-dbo: null - dbo-prefill-token-threshold: null - dbo-decode-token-threshold: null speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"TRITON_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' max-num-seqs: [2, 4, 8, 16] env: @@ -116,12 +106,19 @@ zip_override_tp4_dspark7_mono: env: CONC: ['1', '2', '4', '8'] -# c16-c32: DSpark 7 on the NVFP4 KV cache with DBO, at the base 0.85 memory -# budget (headroom for DBO prefill). Admission is 2x CONC. +# c16-c32: DSpark 7 on the NVFP4 KV cache. Breakable CUDA graphs capture the +# prefill and mixed steps too. The capture list (every multiple of 8 up to 256, +# then 384-8192; a coarse list makes decode about 10% slower) is a deprecated +# alias on the V2 runner that sets both the prefill token sizes and the decode +# request sizes (1-32 at c16); the new per-kind fields would change the decode +# graphs. Admission is 2x CONC. zip_override_tp4_dspark7: roles: agg: + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: '1' args: + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,384,512,768,1024,1536,2048,3072,4096,6144,8192]}' speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"TRITON_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' max-num-seqs: [32, 64] benchmark: @@ -129,7 +126,7 @@ zip_override_tp4_dspark7: CONC: ['16', '32'] # c64: DSpark 3 (the blog's MI355X draft length from c8) on the NVFP4 KV cache -# with DBO, sequence-parallel prefill, 0.90 memory utilization, and a CPU KV tier. The c64 working set +# with sequence-parallel prefill, 0.90 memory utilization, and a CPU KV tier. The c64 working set # outgrows the 14.2M-token GPU KV cache; SimpleCPUOffloadConnector (eager) # holds about 24M tokens in 1349 GB of host memory (dram-utilization 0.9 of the # TP4 lane) and serves the misses. diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 22a53fd3ff..d907e46ec8 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -904,14 +904,8 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark-tp8-c1: search-space: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1], srt-recipe: agentic.yaml } -# MiniMax-M3.1-Flash AgentX on MI355X: day-zero vLLM ROCm image from -# https://vllm.ai/blog/2026-10-23-minimax-m31-flash, verified 2026-10-23. -# AMD's Quark MXFP4 export (routed experts in MXFP4, W4A4 on the AITER MoE) -# at TP4 with native DSpark: 7 drafts, mono decode and an FP8 KV cache at -# c1-c8; 7 drafts, an NVFP4 KV cache and DBO at c16-c32; 3 drafts, DBO and a -# CPU KV tier at c64. Throughput runs use the minimax-m3.1-flash__dspark__native__probabilistic-block golden AL. minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:minimax-m31-flash@sha256:ae8681212aa5291d8c27f06e90016435e3fca144a68fd5c439231f6475f74569 + image: vllm/vllm-openai-rocm:nightly-4fab5a07475822c6eaa971dceb2e2220d0446808@sha256:6f126f27a3b58959da3c16076248fc99eb31e26d0a3ae9f8563bda7317d6a7e6 model: amd/MiniMax-M3.1-Flash-Quark-MXFP4 model-prefix: minimaxm31flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e719a43f93..e0ac9190ac 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12669,3 +12669,14 @@ - "No point changes: concurrency 1, 2, 4, 8, 16 and 32 on TP8, TP4 and TP2 (18 points), all re-measured on the new image." - "DSpark draft: vLLM builds it with the checkpoint's own quantization config and the target's fp8_ds_mla KV cache, and its weights are used as shipped (FP8 E4M3 with UE8M0 block scales, MXFP4 routed experts). On the TP4 points --monokernel also runs the routed-expert FFN of the three draft layers as one launch. That launch quantizes the FFN input to MXFP8 against the same MXFP4 expert weights, as it does for the target layers, where vLLM's gfx942 MoE multiplies bf16 activations with them. The draft weights, its KV cache and its attention are unchanged, and the target model's output does not depend on this launch, only how many drafts it accepts." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4617 + +- config-keys: + - minimaxm31flash-fp4-mi355x-vllm-agentic-dspark + scenario-type: [agentic-coding] + description: + - "Image vllm/vllm-openai-rocm:minimax-m31-flash (vLLM 0.32.0rc1.dev364+g8d0f05a0c) -> nightly-4fab5a07475822c6eaa971dceb2e2220d0446808 (vLLM main of 2026-11-05), pinned by digest. It carries the MiniMax-M3.1 ROCm work merged since: decode CP (#64448), qkv input-norm quant fusion (#63637), four-wave decode top-k (#65087), local DSpark 7 indexer scoring below 3 requests (#65432), FFN-side MXFP8 quant fusion (#65478), sparse-decode splits per XCD (#65481), MXFP8 GEMM tile retune for M=17-256 (#65530), DSpark drafter monokernel (#65386) and opt-in parallel top-k (#65267)." + - "New env at every point: VLLM_ROCM_MINIMAX_M3_MONO_DRAFTER=1 (fused FlyDSL DSpark 7 draft steps of up to 10 requests; the c64 3-token draft keeps the stock layers), VLLM_ROCM_PARALLEL_INDEX_TOPK=1 (NVFP4 indexer points; the FP8 indexer of c1-c8 bypasses it), VLLM_USE_RUST_FRONTEND=1 and VLLM_ROCM_USE_SKINNY_GEMM=0 (removes the draft LM head's 5-request skinny-GEMM cliff)." + - "C16/C32: breakable CUDA graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1, FULL_AND_PIECEWISE with a multiple-of-8 capture list up to 256, then 384-8192). INT4 quick all-reduce stays off: the image's AITER lacks the QuickAllReduceInt4.compile that vLLM's FlyDSL INT4 path calls, so workers fail at startup. DBO is removed from C16/C32/C64 (C1-C8 already ran without it); the 32768-token prefill budget, max-num-seqs and memory utilization are unchanged." + - "C1-C8 (DSpark 7, mono decode, FP8 KV and indexer caches) and C64 (DSpark 3, NVFP4 KV, CPU KV offload) keep their speculative, KV and offload settings." + - "Draft data types unchanged: the dspark/ draft keeps its MXFP8 config (FP8 E4M3 projections with 1x32 UE8M0 scales; BF16 fc, Markov and confidence heads, norms) and an FP8 draft KV cache. The drafter kernels replace the draft layers' MXFP8 linear, norm and attention ops, and vLLM's tests/models/minimax_m3/test_dspark_drafter.py holds them to the stock layers' own numerical spread." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4866