From cc407c36707210c4781c9e824391982dc18eb857 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 2 Nov 2026 00:59:12 +0800 Subject: [PATCH 1/8] add b300 sglang points --- .../b300-fp4-mtp/agentic-tp2-spf-extra.yaml | 73 ++++++++++++++ .../b300-fp4-mtp/agentic-tp4-spf-extra.yaml | 95 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 6 ++ inferencex-e2e/perf-changelog.yaml | 9 ++ 4 files changed, 183 insertions(+) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml new file mode 100644 index 0000000000..2bc3e7cf0e --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml @@ -0,0 +1,73 @@ +base: + schema: 2 + name: minimaxm31flash-fp4-b300-sglang-agentic-tp2-spf-extra + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + # Serve flags follow the cookbook's B300 cells at + # https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1. + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + # Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs + # switch to synthetic acceptance at the golden acceptance length. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 + cuda-graph-backend-prefill: breakable + tokenizer-worker-num: 8 + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + FMHA_SM100_ALLOW_JIT: '1' + benchmark: + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# 2x CONC (AgentX subagent fan-out); the decode graph batch matches it. +override_tp2_c44_spf_interval32: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: 88 + cuda-graph-max-bs-decode: 88 + mem-fraction-static: 0.92 + enable-hierarchical-cache: true + hicache-size: 370 + hicache-write-policy: write_back + schedule-policy: shortest-prefill-first + prefill-decode-interval: 32 + stream-interval: 20 + benchmark: + env: + CONC: '44' + KV_OFFLOADING: dram diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml new file mode 100644 index 0000000000..9ce5df5a62 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml @@ -0,0 +1,95 @@ +base: + schema: 2 + name: minimaxm31flash-fp4-b300-sglang-agentic-tp4-spf-extra + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + # Serve flags follow the cookbook's B300 cells at + # https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1. + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + # Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs + # switch to synthetic acceptance at the golden acceptance length. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 + cuda-graph-backend-prefill: breakable + tokenizer-worker-num: 8 + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + FMHA_SM100_ALLOW_JIT: '1' + benchmark: + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# Shortest-prefill-first with a decode interval of 32 on TP4 with DRAM KV offload, +# filling the curve between the TP2 and TP4 points. +# 2x CONC (AgentX subagent fan-out); the decode graph batch matches it. +zip_override_tp4_dram_spf_interval32: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: [40, 88, 104, 120] + cuda-graph-max-bs-decode: [40, 88, 104, 120] + mem-fraction-static: 0.90 + enable-hierarchical-cache: true + hicache-size: 290 + hicache-write-policy: write_back + schedule-policy: shortest-prefill-first + prefill-decode-interval: 32 + stream-interval: 20 + benchmark: + env: + CONC: ['20', '44', '52', '60'] + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1199' + +# At conc 16 the working set fits in HBM; host offload only adds overhead. +override_tp4_c16_spf_interval32: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + mem-fraction-static: 0.90 + schedule-policy: shortest-prefill-first + prefill-decode-interval: 32 + stream-interval: 20 + benchmark: + env: + CONC: '16' + KV_OFFLOADING: none diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ca18213d0c..d4acf986fb 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8027,6 +8027,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - search-space: # Balanced: TP4. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [16], srt-recipe: 'agentic-tp4-spf-extra.yaml:override_tp4_c16_spf_interval32' } # Low-Latency: MSA-V1 MonoKernel at batch size 1. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic-low-latency.yaml } - dram-utilization: 0.80 @@ -8037,6 +8038,10 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40], srt-recipe: 'agentic-tp4-spf.yaml:zip_override_tp4_dram_spf_interval32[2]' } - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8], srt-recipe: 'agentic-tp4-spf.yaml:zip_override_tp4_dram_spf_interval32[3]' } - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16], srt-recipe: 'agentic-tp4-spf.yaml:zip_override_tp4_dram_spf_interval32[4]' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[0]' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[1]' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[2]' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [60], srt-recipe: 'agentic-tp4-spf-extra.yaml:zip_override_tp4_dram_spf_interval32[3]' } - dram-utilization: 1.0 search-space: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml } @@ -8054,6 +8059,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40], srt-recipe: 'agentic-tp2-spf-interval.yaml:zip_override_tp2_spf_interval32[5]' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c48_spf_interval24' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c44_spf_interval32' } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 88ea87392c..b25fa0bd4b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11769,3 +11769,12 @@ - "6 DEP2 prefill workers with a DEP8 decode at conc 280-400 or a DEP16 decode at conc 300; MegaMoE, DSpark block 7 with fp8 draft KV, Dynamo KV router with fastokens for prefill and least-loaded routing across decode DP ranks." - "Prefill: shortest-prefill-first, chunk 8192, mem-fraction-static 0.85, HiCache 200 GB per rank write_back." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4686 + +- config-keys: + - minimaxm31flash-fp4-b300-sglang-agentic-dspark + description: + - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." + - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + scenario-type: [agentic-coding] + append-only: true From 0cffd4c4ee437b67e61a50e873e76e7ad2912a46 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 2 Nov 2026 01:39:43 +0800 Subject: [PATCH 2/8] fill pr link --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b25fa0bd4b..cfbd3d575d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11775,6 +11775,6 @@ description: - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726 scenario-type: [agentic-coding] append-only: true From f69cefc532498f6aebe9964eb5f665cc49457090 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 2 Nov 2026 02:15:37 +0800 Subject: [PATCH 3/8] add tp2 larger kv pool points --- .../b300-fp4-mtp/agentic-tp2-spf-extra.yaml | 46 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 2 + inferencex-e2e/perf-changelog.yaml | 1 + 3 files changed, 49 insertions(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml index 2bc3e7cf0e..6466894fcb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml @@ -71,3 +71,49 @@ override_tp2_c44_spf_interval32: env: CONC: '44' KV_OFFLOADING: dram + +# A larger HBM KV pool keeps more of the reused prefixes on device, so fewer are +# reloaded from host DRAM. +override_tp2_c52_spf_interval20_kv5850k: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: 104 + cuda-graph-max-bs-decode: 104 + mem-fraction-static: 0.94 + max-total-tokens: 5850000 + enable-hierarchical-cache: true + hicache-size: 370 + hicache-write-policy: write_back + schedule-policy: shortest-prefill-first + prefill-decode-interval: 20 + stream-interval: 20 + benchmark: + env: + CONC: '52' + KV_OFFLOADING: dram + +override_tp2_c48_spf_interval24_kv5850k: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: 96 + cuda-graph-max-bs-decode: 96 + mem-fraction-static: 0.94 + max-total-tokens: 5850000 + enable-hierarchical-cache: true + hicache-size: 370 + hicache-write-policy: write_back + schedule-policy: shortest-prefill-first + prefill-decode-interval: 24 + stream-interval: 20 + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d4acf986fb..1893385fd8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8060,6 +8060,8 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c48_spf_interval24' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c44_spf_interval32' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c52_spf_interval20_kv5850k' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c48_spf_interval24_kv5850k' } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index cfbd3d575d..24d2176b44 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11775,6 +11775,7 @@ description: - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." + - "Add TP2 points with DRAM KV offload and a 5.85M-token HBM KV pool (mem-fraction-static 0.94) at conc 52 with a prefill-decode interval of 20 and conc 48 with 24, using shortest-prefill-first." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726 scenario-type: [agentic-coding] append-only: true From e90debd5ee2998692e27d8995ca3d0a743a8a7cf Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 2 Nov 2026 15:08:21 +0800 Subject: [PATCH 4/8] lower mem fraction for larger kv pool points --- .../sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml | 10 ++++------ inferencex-e2e/configs/nvidia-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml index 6466894fcb..c1bad1a47a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml @@ -74,7 +74,7 @@ override_tp2_c44_spf_interval32: # A larger HBM KV pool keeps more of the reused prefixes on device, so fewer are # reloaded from host DRAM. -override_tp2_c52_spf_interval20_kv5850k: +override_tp2_c52_spf_interval20_mem093: roles: agg: args: @@ -83,8 +83,7 @@ override_tp2_c52_spf_interval20_kv5850k: chunked-prefill-size: 8192 max-running-requests: 104 cuda-graph-max-bs-decode: 104 - mem-fraction-static: 0.94 - max-total-tokens: 5850000 + mem-fraction-static: 0.93 enable-hierarchical-cache: true hicache-size: 370 hicache-write-policy: write_back @@ -96,7 +95,7 @@ override_tp2_c52_spf_interval20_kv5850k: CONC: '52' KV_OFFLOADING: dram -override_tp2_c48_spf_interval24_kv5850k: +override_tp2_c48_spf_interval24_mem093: roles: agg: args: @@ -105,8 +104,7 @@ override_tp2_c48_spf_interval24_kv5850k: chunked-prefill-size: 8192 max-running-requests: 96 cuda-graph-max-bs-decode: 96 - mem-fraction-static: 0.94 - max-total-tokens: 5850000 + mem-fraction-static: 0.93 enable-hierarchical-cache: true hicache-size: 370 hicache-write-policy: write_back diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1893385fd8..e02a6220a5 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8060,8 +8060,8 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c48_spf_interval24' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c44_spf_interval32' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c52_spf_interval20_kv5850k' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c48_spf_interval24_kv5850k' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c52_spf_interval20_mem093' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c48_spf_interval24_mem093' } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 24d2176b44..2caf27b240 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11775,7 +11775,7 @@ description: - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." - - "Add TP2 points with DRAM KV offload and a 5.85M-token HBM KV pool (mem-fraction-static 0.94) at conc 52 with a prefill-decode interval of 20 and conc 48 with 24, using shortest-prefill-first." + - "Add TP2 points with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 52 with a prefill-decode interval of 20 and conc 48 with 24, using shortest-prefill-first." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726 scenario-type: [agentic-coding] append-only: true From 6bd10ab091569b8fcc693c3bb4fbe14953e99ae0 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 2 Nov 2026 22:49:40 +0800 Subject: [PATCH 5/8] keep tp2 c50 larger kv pool point --- .../b300-fp4-mtp/agentic-tp2-spf-extra.yaml | 29 +++---------------- inferencex-e2e/configs/nvidia-master.yaml | 3 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml index c1bad1a47a..d51e90bd14 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml @@ -74,15 +74,15 @@ override_tp2_c44_spf_interval32: # A larger HBM KV pool keeps more of the reused prefixes on device, so fewer are # reloaded from host DRAM. -override_tp2_c52_spf_interval20_mem093: +override_tp2_c50_spf_interval20_mem093: roles: agg: args: tensor-parallel-size: 2 moe-runner-backend: flashinfer_trtllm_routed chunked-prefill-size: 8192 - max-running-requests: 104 - cuda-graph-max-bs-decode: 104 + max-running-requests: 100 + cuda-graph-max-bs-decode: 100 mem-fraction-static: 0.93 enable-hierarchical-cache: true hicache-size: 370 @@ -92,26 +92,5 @@ override_tp2_c52_spf_interval20_mem093: stream-interval: 20 benchmark: env: - CONC: '52' - KV_OFFLOADING: dram - -override_tp2_c48_spf_interval24_mem093: - roles: - agg: - args: - tensor-parallel-size: 2 - moe-runner-backend: flashinfer_trtllm_routed - chunked-prefill-size: 8192 - max-running-requests: 96 - cuda-graph-max-bs-decode: 96 - mem-fraction-static: 0.93 - enable-hierarchical-cache: true - hicache-size: 370 - hicache-write-policy: write_back - schedule-policy: shortest-prefill-first - prefill-decode-interval: 24 - stream-interval: 20 - benchmark: - env: - CONC: '48' + CONC: '50' KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index e02a6220a5..8d164d0faf 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8060,8 +8060,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c48_spf_interval24' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c44_spf_interval32' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c52_spf_interval20_mem093' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c48_spf_interval24_mem093' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c50_spf_interval20_mem093' } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e4fddc3f47..96b65bc33b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11797,7 +11797,7 @@ description: - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." - - "Add TP2 points with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 52 with a prefill-decode interval of 20 and conc 48 with 24, using shortest-prefill-first." + - "Add a TP2 point with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 50 using shortest-prefill-first with a prefill-decode interval of 20." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726 scenario-type: [agentic-coding] append-only: true From ab42462db15e7937ba868d11c520c123479703a5 Mon Sep 17 00:00:00 2001 From: ispobock Date: Tue, 3 Nov 2026 01:06:00 +0800 Subject: [PATCH 6/8] add back tp2 c52 larger kv pool point --- .../b300-fp4-mtp/agentic-tp2-spf-extra.yaml | 21 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 1 + inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml index d51e90bd14..9169dfb96b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2-spf-extra.yaml @@ -94,3 +94,24 @@ override_tp2_c50_spf_interval20_mem093: env: CONC: '50' KV_OFFLOADING: dram + +override_tp2_c52_spf_interval20_mem093: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: 104 + cuda-graph-max-bs-decode: 104 + mem-fraction-static: 0.93 + enable-hierarchical-cache: true + hicache-size: 370 + hicache-write-policy: write_back + schedule-policy: shortest-prefill-first + prefill-decode-interval: 20 + stream-interval: 20 + benchmark: + env: + CONC: '52' + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index db6a464cdc..470a654984 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8144,6 +8144,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [44], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c44_spf_interval32' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c50_spf_interval20_mem093' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [52], srt-recipe: 'agentic-tp2-spf-extra.yaml:override_tp2_c52_spf_interval20_mem093' } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 819fc6fad9..3c6a2490e4 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11818,7 +11818,7 @@ description: - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." - - "Add a TP2 point with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 50 using shortest-prefill-first with a prefill-decode interval of 20." + - "Add TP2 points with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 50 and 52 using shortest-prefill-first with a prefill-decode interval of 20." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726 scenario-type: [agentic-coding] append-only: true From 1a4fd4dfd3bf72983d8fc8469ca541ab4fcde71c Mon Sep 17 00:00:00 2001 From: ispobock Date: Tue, 3 Nov 2026 13:36:22 +0800 Subject: [PATCH 7/8] add tp4 c17 point --- .../b300-fp4-mtp/agentic-tp4-spf-extra.yaml | 18 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 1 + inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml index 9ce5df5a62..e1576862e4 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml @@ -93,3 +93,21 @@ override_tp4_c16_spf_interval32: env: CONC: '16' KV_OFFLOADING: none + +override_tp4_c17_spf_interval32: + roles: + agg: + args: + tensor-parallel-size: 4 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 8192 + max-running-requests: 34 + cuda-graph-max-bs-decode: 34 + mem-fraction-static: 0.90 + schedule-policy: shortest-prefill-first + prefill-decode-interval: 32 + stream-interval: 20 + benchmark: + env: + CONC: '17' + KV_OFFLOADING: none diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 470a654984..647f867cb8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8111,6 +8111,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: # Balanced: TP4. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [16], srt-recipe: 'agentic-tp4-spf-extra.yaml:override_tp4_c16_spf_interval32' } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [17], srt-recipe: 'agentic-tp4-spf-extra.yaml:override_tp4_c17_spf_interval32' } # Low-Latency: MSA-V1 MonoKernel at batch size 1. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic-low-latency.yaml } - dram-utilization: 0.80 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3c6a2490e4..3e1dc09cd1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11816,7 +11816,7 @@ - config-keys: - minimaxm31flash-fp4-b300-sglang-agentic-dspark description: - - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." + - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and TP4 points without offload at conc 16 and 17, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." - "Add TP2 points with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 50 and 52 using shortest-prefill-first with a prefill-decode interval of 20." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726 From 04a5e36723063317059c09c9a744fb7ec54cc8cf Mon Sep 17 00:00:00 2001 From: ispobock Date: Tue, 3 Nov 2026 18:09:53 +0800 Subject: [PATCH 8/8] drop tp4 c17 point --- .../b300-fp4-mtp/agentic-tp4-spf-extra.yaml | 18 ------------------ inferencex-e2e/configs/nvidia-master.yaml | 1 - inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml index e1576862e4..9ce5df5a62 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp4-spf-extra.yaml @@ -93,21 +93,3 @@ override_tp4_c16_spf_interval32: env: CONC: '16' KV_OFFLOADING: none - -override_tp4_c17_spf_interval32: - roles: - agg: - args: - tensor-parallel-size: 4 - moe-runner-backend: flashinfer_trtllm_routed - chunked-prefill-size: 8192 - max-running-requests: 34 - cuda-graph-max-bs-decode: 34 - mem-fraction-static: 0.90 - schedule-policy: shortest-prefill-first - prefill-decode-interval: 32 - stream-interval: 20 - benchmark: - env: - CONC: '17' - KV_OFFLOADING: none diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 0f363b0584..6d3baa233f 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6828,7 +6828,6 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: # Balanced: TP4. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [16], srt-recipe: 'agentic-tp4-spf-extra.yaml:override_tp4_c16_spf_interval32' } - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [17], srt-recipe: 'agentic-tp4-spf-extra.yaml:override_tp4_c17_spf_interval32' } # Low-Latency: MSA-V1 MonoKernel at batch size 1. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic-low-latency.yaml } - dram-utilization: 0.80 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d4737f517a..26fe1d45d1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11948,7 +11948,7 @@ - config-keys: - minimaxm31flash-fp4-b300-sglang-agentic-dspark description: - - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and TP4 points without offload at conc 16 and 17, using shortest-prefill-first with a prefill-decode interval of 32." + - "Add TP4 points with DRAM KV offload at conc 20, 44, 52 and 60, and a TP4 point without offload at conc 16, using shortest-prefill-first with a prefill-decode interval of 32." - "Add a TP2 point with DRAM KV offload at conc 44 using shortest-prefill-first with a prefill-decode interval of 32." - "Add TP2 points with DRAM KV offload and a larger HBM KV pool (mem-fraction-static 0.93) at conc 50 and 52 using shortest-prefill-first with a prefill-decode interval of 20." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4726