Skip to content

Commit 7e76207

Browse files
ispobockispobock
authored
[AgentX] MiniMax-M3.1-Flash B300 SGLang: add TP2 and TP4 points / [AgentX] MiniMax-M3.1-Flash B300 SGLang：新增 TP2 和 TP4 点 (#4726)
1 parent 98ed2c8 commit 7e76207

4 files changed

Lines changed: 230 additions & 0 deletions

File tree

‎inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
base:
2+
schema: 2
3+
name: minimaxm31flash-fp4-b300-sglang-agentic-tp2-spf-extra
4+
resources:
5+
gpu_type: b300
6+
gpus_per_node: 8
7+
frontend:
8+
type: sglang
9+
enable_multiple_frontends: false
10+
observability:
11+
enabled: false
12+
tachometer:
13+
enabled: false
14+
engine: sglang
15+
# Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default.
16+
health_check:
17+
max_attempts: 1440
18+
interval_seconds: 5
19+
roles:
20+
agg:
21+
nodes: 1
22+
workers: 1
23+
gpus: 2
24+
# Serve flags follow the cookbook's B300 cells at
25+
# https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1.
26+
args:
27+
served-model-name: MiniMaxAI/MiniMax-M3.1-Flash
28+
trust-remote-code: true
29+
reasoning-parser: minimax-m3
30+
tool-call-parser: minimax-m3
31+
# Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs
32+
# switch to synthetic acceptance at the golden acceptance length.
33+
speculative-algorithm: DSPARK
34+
speculative-dspark-block-size: 7
35+
speculative-use-block-verification: true
36+
# FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank.
37+
speculative-draft-kv-cache-dtype: fp8_e4m3
38+
cuda-graph-backend-prefill: breakable
39+
tokenizer-worker-num: 8
40+
watchdog-timeout: 3600
41+
enable-metrics: true
42+
env:
43+
PYTHONNOUSERSITE: '1'
44+
PYTHONUNBUFFERED: '1'
45+
SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
46+
FMHA_SM100_ALLOW_JIT: '1'
47+
benchmark:
48+
env:
49+
# Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
50+
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
51+
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
52+
53+
# 2x CONC (AgentX subagent fan-out); the decode graph batch matches it.
54+
override_tp2_c44_spf_interval32:
55+
roles:
56+
agg:
57+
args:
58+
tensor-parallel-size: 2
59+
moe-runner-backend: flashinfer_trtllm_routed
60+
chunked-prefill-size: 8192
61+
max-running-requests: 88
62+
cuda-graph-max-bs-decode: 88
63+
mem-fraction-static: 0.92
64+
enable-hierarchical-cache: true
65+
hicache-size: 370
66+
hicache-write-policy: write_back
67+
schedule-policy: shortest-prefill-first
68+
prefill-decode-interval: 32
69+
stream-interval: 20
70+
benchmark:
71+
env:
72+
CONC: '44'
73+
KV_OFFLOADING: dram
74+
75+
# A larger HBM KV pool keeps more of the reused prefixes on device, so fewer are
76+
# reloaded from host DRAM.
77+
override_tp2_c50_spf_interval20_mem093:
78+
roles:
79+
agg:
80+
args:
81+
tensor-parallel-size: 2
82+
moe-runner-backend: flashinfer_trtllm_routed
83+
chunked-prefill-size: 8192
84+
max-running-requests: 100
85+
cuda-graph-max-bs-decode: 100
86+
mem-fraction-static: 0.93
87+
enable-hierarchical-cache: true
88+
hicache-size: 370
89+
hicache-write-policy: write_back
90+
schedule-policy: shortest-prefill-first
91+
prefill-decode-interval: 20
92+
stream-interval: 20
93+
benchmark:
94+
env:
95+
CONC: '50'
96+
KV_OFFLOADING: dram
97+
98+
override_tp2_c52_spf_interval20_mem093:
99+
roles:
100+
agg:
101+
args:
102+
tensor-parallel-size: 2
103+
moe-runner-backend: flashinfer_trtllm_routed
104+
chunked-prefill-size: 8192
105+
max-running-requests: 104
106+
cuda-graph-max-bs-decode: 104
107+
mem-fraction-static: 0.93
108+
enable-hierarchical-cache: true
109+
hicache-size: 370
110+
hicache-write-policy: write_back
111+
schedule-policy: shortest-prefill-first
112+
prefill-decode-interval: 20
113+
stream-interval: 20
114+
benchmark:
115+
env:
116+
CONC: '52'
117+
KV_OFFLOADING: dram

‎inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
base:
2+
schema: 2
3+
name: minimaxm31flash-fp4-b300-sglang-agentic-tp4-spf-extra
4+
resources:
5+
gpu_type: b300
6+
gpus_per_node: 8
7+
frontend:
8+
type: sglang
9+
enable_multiple_frontends: false
10+
observability:
11+
enabled: false
12+
tachometer:
13+
enabled: false
14+
engine: sglang
15+
# Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default.
16+
health_check:
17+
max_attempts: 1440
18+
interval_seconds: 5
19+
roles:
20+
agg:
21+
nodes: 1
22+
workers: 1
23+
gpus: 4
24+
# Serve flags follow the cookbook's B300 cells at
25+
# https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1.
26+
args:
27+
served-model-name: MiniMaxAI/MiniMax-M3.1-Flash
28+
trust-remote-code: true
29+
reasoning-parser: minimax-m3
30+
tool-call-parser: minimax-m3
31+
# Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs
32+
# switch to synthetic acceptance at the golden acceptance length.
33+
speculative-algorithm: DSPARK
34+
speculative-dspark-block-size: 7
35+
speculative-use-block-verification: true
36+
# FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank.
37+
speculative-draft-kv-cache-dtype: fp8_e4m3
38+
cuda-graph-backend-prefill: breakable
39+
tokenizer-worker-num: 8
40+
watchdog-timeout: 3600
41+
enable-metrics: true
42+
env:
43+
PYTHONNOUSERSITE: '1'
44+
PYTHONUNBUFFERED: '1'
45+
SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
46+
FMHA_SM100_ALLOW_JIT: '1'
47+
benchmark:
48+
env:
49+
# Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
50+
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
51+
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
52+
53+
# Shortest-prefill-first with a decode interval of 32 on TP4 with DRAM KV offload,
54+
# filling the curve between the TP2 and TP4 points.
55+
# 2x CONC (AgentX subagent fan-out); the decode graph batch matches it.
56+
zip_override_tp4_dram_spf_interval32:
57+
roles:
58+
agg:
59+
args:
60+
tensor-parallel-size: 4
61+
moe-runner-backend: flashinfer_trtllm_routed
62+
chunked-prefill-size: 8192
63+
max-running-requests: [40, 88, 104, 120]
64+
cuda-graph-max-bs-decode: [40, 88, 104, 120]
65+
mem-fraction-static: 0.90
66+
enable-hierarchical-cache: true
67+
hicache-size: 290
68+
hicache-write-policy: write_back
69+
schedule-policy: shortest-prefill-first
70+
prefill-decode-interval: 32
71+
stream-interval: 20
72+
benchmark:
73+
env:
74+
CONC: ['20', '44', '52', '60']
75+
KV_OFFLOADING: dram
76+
TOTAL_CPU_DRAM_GB: '1199'
77+
78+
# At conc 16 the working set fits in HBM; host offload only adds overhead.
79+
override_tp4_c16_spf_interval32:
80+
roles:
81+
agg:
82+
args:
83+
tensor-parallel-size: 4
84+
moe-runner-backend: flashinfer_trtllm_routed
85+
chunked-prefill-size: 8192
86+
max-running-requests: 32
87+
cuda-graph-max-bs-decode: 32
88+
mem-fraction-static: 0.90
89+
schedule-policy: shortest-prefill-first
90+
prefill-decode-interval: 32
91+
stream-interval: 20
92+
benchmark:
93+
env:
94+
CONC: '16'
95+
KV_OFFLOADING: none

‎inferencex-e2e/configs/nvidia-master.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6827,6 +6827,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark:
68276827
- search-space:
68286828
# Balanced: TP4.
68296829
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml }
6830+
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [16], srt-recipe: 'agentic-tp4-spf-extra.yaml:override_tp4_c16_spf_interval32' }
68306831
# Low-Latency: MSA-V1 MonoKernel at batch size 1.
68316832
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic-low-latency.yaml }
68326833
- dram-utilization: 0.80
@@ -6837,6 +6838,10 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark:
68376838
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40], srt-recipe: 'agentic-tp4-spf.yaml:zip_override_tp4_dram_spf_interval32[2]' }
68386839
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8], srt-recipe: 'agentic-tp4-spf.yaml:zip_override_tp4_dram_spf_interval32[3]' }
68396840
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16], srt-recipe: 'agentic-tp4-spf.yaml:zip_override_tp4_dram_spf_interval32[4]' }
6841+
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[0]' }
6842+
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[1]' }
6843+
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[2]' }
6844+
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [60], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[3]' }
68406845
- dram-utilization: 1.0
68416846
search-space:
68426847
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml }
@@ -6854,6 +6859,9 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark:
68546859
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40], srt-recipe: 'agentic-tp2-spf-interval.yaml:zip_override_tp2_spf_interval32[5]' }
68556860
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c48_spf_interval24' }
68566861
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' }
6862+
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c44_spf_interval32' }
6863+
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c50_spf_interval20_mem093' }
6864+
- { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c52_spf_interval20_mem093' }
68576865

68586866
# Official SGLang nightly; every point uses the host-DRAM Engram table.
68596867
dsv41flash-fp4-gb300-sglang-agentic-dspark:

‎inferencex-e2e/perf-changelog.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11944,3 +11944,13 @@
1194411944
- "TP4 c1/c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged."
1194511945
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603
1194611946
scenario-type: [agentic-coding]
11947+
11948+
- config-keys:
11949+
- minimaxm31flash-fp4-b300-sglang-agentic-dspark
11950+
description:
11951+
- "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32."
11952+
- "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32."
11953+
- "Add TP2 points with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 50 and 52 using shortest-prefill-first with a prefill-decode interval of 20."
11954+
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726
11955+
scenario-type: [agentic-coding]
11956+
append-only: true

0 commit comments

Comments
 (0)