From 67af7c3dadee03b28a489a459d69bf84e85ad405 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:48:34 +0800 Subject: [PATCH 1/7] Update GB300 DSV4.1-Flash SGLang AgentX image to nightly-dev-cu13-20261025-65633129 Move dsv41flash-fp4-gb300-sglang-agentic-dspark from nightly-dev-cu13-20261021-c1c287a4 to the digest-pinned nightly-dev-cu13-20261025-65633129, the image the Dynamo + SGLang agg key already uses. Recipes and search space are unchanged. --- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index dcc8d15576..d7f3bd3034 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9687,7 +9687,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb + image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b4cbda3afc..a9debb56b2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11480,3 +11480,10 @@ - "Update the Muse Spark Open B300 curve to lmsysorg/sglang:nightly-dev-20261021-c1c287a4 and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 at every concurrency; run TP8/EP8 at concurrency 1, 4, 8, 10, 14, 16, 20, 24 and TP4/EP4 HiCache at concurrency 16, 28, 32, 40, 44, 48." - "Use shortest-prefill-first at TP8 concurrency 10-24, stream interval 20 at TP8 concurrency 4, 20, 24 and TP4 concurrency 28-48, SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS=4 at TP4 concurrency 40-48, and at concurrency 44 and 48 request admission 24 and 26 with FCFS, no prefill-decode interval, HiCache write_back and SWA/full ratio 0.1." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4417 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + description: + - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261025-65633129, pinned by digest." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + scenario-type: [agentic-coding] From 197048dcbf935de41a80b81e1ded4564b128f737 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:49:25 +0800 Subject: [PATCH 2/7] Set perf-changelog pr-link to #4603 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a9debb56b2..9d4e507a6e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11485,5 +11485,5 @@ - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261025-65633129, pinned by digest." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603 scenario-type: [agentic-coding] From 517d5af6899fdf6b598f3e4e21755cde8c91747c Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:56:36 +0800 Subject: [PATCH 3/7] Move GB300 DSV4.1-Flash SGLang AgentX to 20261028 nightly with the tuned low-latency settings Use nightly-dev-cu13-20261028-586355a8, which includes sgl-project/sglang#46404 and #46891. TP4 c1 runs as the batch-1 NVIDIA megakernel point of the Dynamo + SGLang aggregated row, without SIM_DRAW_AHEAD so the simulated step does the same work as a real one. TP4 c4/c16 and TP2/EP2 c16/c32 pin the flashmla attention backend instead of auto (MegaAttention pads per-rank heads to 64). --- .../sglang/gb300-fp4-mtp/agentic.yaml | 48 ++++++++++++++++--- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 +- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 1d56ebd861..40c3e5e42b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -64,6 +64,8 @@ override_tp2_c16: args: tensor-parallel-size: 2 expert-parallel-size: 2 + # auto resolves to MegaAttention, which zero-pads per-rank heads to 64 (2x at TP2, 4x at TP4). + dsv4-attn-backend: flashmla <<: &tp2_bounded_replay chunked-prefill-size: 16384 enable-decoder-swa-bounded-replay: true @@ -90,6 +92,7 @@ override_tp2_c32: args: tensor-parallel-size: 2 expert-parallel-size: 2 + dsv4-attn-backend: flashmla <<: *tp2_bounded_replay prefill-decode-interval: 48 tokenizer-worker-num: 4 @@ -187,14 +190,45 @@ override_tp4_c1: args: tensor-parallel-size: 4 expert-parallel-size: 1 - chunked-prefill-size: 16384 - prefill-decode-interval: 0 - max-running-requests: 1 + mem-fraction-static: 0.91 + kv-cache-dtype: fp8_e4m3 + dsv4-attn-backend: flashmla + moe-a2a-backend: none + enable-nvidia-megakernel: true + startup-weight-load-mode: serial enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled + context-length: 1048576 + max-running-requests: 1 + cuda-graph-max-bs-decode: 1 + prefill-decode-interval: 16 + schedule-policy: lpm + swa-prefix-tails: 128 + chunked-prefill-size: 16384 + enable-metrics-for-all-schedulers: true env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' - SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_SPEC_TP_SYNC: rng,-dspark-accept-sample + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_NVIDIA_MEGAKERNEL_SLOTS: hop_a,moe,gate,attn,embed,two_shot,markov,draft_lm_head,aux_mean_stats,gate_wide,shared_ll,hop_a_batch2,engram,attn_qkv_a,attn_qkv_a_bf16,index_q_mxfp8_in,attn_indexer,hc_prenorm,hop_b,hop_b_wide,early_trigger,hop_b_start_trigger,draft,draft_moe,draft_gate,draft_tail,draft_post,draft_seam,draft_attn,draft_qfront,draft_wo_a,draft_swapab,draft_qkv_a,draft_prenorm,draft_qkv_a_ll,draft_wo_b_ll,sched_ahead,req_pos_once,kv_before_wait,target_lm_head,hop_b_quad_trigger,lm_head_pinned_tactic + SGLANG_DSPARK_FUSED_ACCEPT_TAIL: '1' + SGLANG_DSPARK_DEFER_CONTEXT_KV: '1' + SGLANG_OPT_DSV41_DSPARK_DRAFT_PREP_FORK: '1' + SGLANG_OPT_DSV41_DSPARK_POST_FOLD: '1' + SGLANG_OPT_DSV41_DSPARK_SKIP_PREFILLED_INPUTS: '1' + SGLANG_OPT_DSV41_DSPARK_ONE_LAUNCH_COMMIT_STAGE: '1' + SGLANG_OPT_DSV41_DSPARK_PREPARE_INPUTS_ONCE: '1' + SGLANG_OPT_DSV41_DSPARK_MID_FOLD: '1' + SGLANG_OPT_DSV41_TARGET_VERIFY_METADATA_GLUE: '1' + SGLANG_OPT_DSV41_TARGET_METADATA_EARLY: '1' + SGLANG_OPT_SKIP_GLUED_REPLAY_VIEW: '1' + SGLANG_OPT_DSV41_POST_FORWARD_CUDA_GRAPH: '1' + SGLANG_OPT_DSV41_BLOCKWISE_REJECTION: '1' + SGLANG_OPT_DSV41_SAMPLING_ACCEPT_GRAPH: '1' + SGLANG_SIMULATE_ACC_GREEDY: '0' + SGLANG_OPT_MXFP4_DEFERRED_FINALIZED_SHAPE: '1' + SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: '1' + NCCL_CUMEM_ENABLE: '0' + NCCL_MNNVL_ENABLE: '0' benchmark: env: CONC: '1' @@ -207,6 +241,7 @@ override_tp4_c4: args: tensor-parallel-size: 4 expert-parallel-size: 1 + dsv4-attn-backend: flashmla chunked-prefill-size: 16384 prefill-decode-interval: 0 swa-prefix-tails: 4096 @@ -228,6 +263,7 @@ override_tp4_c16: args: tensor-parallel-size: 4 expert-parallel-size: 1 + dsv4-attn-backend: flashmla chunked-prefill-size: 16384 prefill-decode-interval: 192 tokenizer-worker-num: 4 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d7f3bd3034..f69a127faa 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9687,7 +9687,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727 + image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index bcbc480cac..f0dd900ee0 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11493,6 +11493,8 @@ - config-keys: - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261025-65633129, pinned by digest." + - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261028-586355a8, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row) and #46891 (no host sync in DSpark draft-block metadata)." + - "TP4 c1 becomes a batch-1 NVIDIA megakernel point with the settings of the Dynamo + SGLang aggregated TP4 c1 row, except SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD, which draws the simulated accept length before the step and is left off so the simulated step does the same work as a real one." + - "TP4 c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603 scenario-type: [agentic-coding] From 9ebd5df65ad86952effb4413961125d962f8379e Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:59:00 +0800 Subject: [PATCH 4/7] Keep SIM_DRAW_AHEAD on the TP4 c1 megakernel point The gate only moves the simulated-acceptance draw, a host-side sample plus a fill that real acceptance never runs, ahead of the draft launch. It does not skip work that a real-acceptance step performs. --- .../dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml | 1 + inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 40c3e5e42b..292ce27a83 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -225,6 +225,7 @@ override_tp4_c1: SGLANG_OPT_DSV41_BLOCKWISE_REJECTION: '1' SGLANG_OPT_DSV41_SAMPLING_ACCEPT_GRAPH: '1' SGLANG_SIMULATE_ACC_GREEDY: '0' + SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD: '1' SGLANG_OPT_MXFP4_DEFERRED_FINALIZED_SHAPE: '1' SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: '1' NCCL_CUMEM_ENABLE: '0' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f0dd900ee0..9bac240427 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11494,7 +11494,7 @@ - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261028-586355a8, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row) and #46891 (no host sync in DSpark draft-block metadata)." - - "TP4 c1 becomes a batch-1 NVIDIA megakernel point with the settings of the Dynamo + SGLang aggregated TP4 c1 row, except SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD, which draws the simulated accept length before the step and is left off so the simulated step does the same work as a real one." + - "TP4 c1 becomes a batch-1 NVIDIA megakernel point with the settings of the Dynamo + SGLang aggregated TP4 c1 row." - "TP4 c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603 scenario-type: [agentic-coding] From a815fbbee78e09fd2c4edbeccb8a0487734d2386 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:57:30 +0800 Subject: [PATCH 5/7] Keep TP4 c1 off the megakernel until the image ships a megakernel API 2 plugin nightly-dev-cu13-20261028-586355a8 bundles sgl-nvidia-mega 6fd7ac30 (megakernel API 1), while its SGLang requires API 2 since sgl-project/sglang#45266, so --enable-nvidia-megakernel fails at startup. TP4 c1 keeps its previous engine settings and pins flashmla like the other TP-sharded points. --- .../sglang/gb300-fp4-mtp/agentic.yaml | 43 +++---------------- inferencex-e2e/perf-changelog.yaml | 3 +- 2 files changed, 7 insertions(+), 39 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 292ce27a83..073680f300 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -190,46 +190,15 @@ override_tp4_c1: args: tensor-parallel-size: 4 expert-parallel-size: 1 - mem-fraction-static: 0.91 - kv-cache-dtype: fp8_e4m3 dsv4-attn-backend: flashmla - moe-a2a-backend: none - enable-nvidia-megakernel: true - startup-weight-load-mode: serial - enable-decoder-swa-bounded-replay: true - context-length: 1048576 - max-running-requests: 1 - cuda-graph-max-bs-decode: 1 - prefill-decode-interval: 16 - schedule-policy: lpm - swa-prefix-tails: 128 chunked-prefill-size: 16384 - enable-metrics-for-all-schedulers: true + prefill-decode-interval: 0 + max-running-requests: 1 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled env: - SGLANG_SPEC_TP_SYNC: rng,-dspark-accept-sample - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_NVIDIA_MEGAKERNEL_SLOTS: hop_a,moe,gate,attn,embed,two_shot,markov,draft_lm_head,aux_mean_stats,gate_wide,shared_ll,hop_a_batch2,engram,attn_qkv_a,attn_qkv_a_bf16,index_q_mxfp8_in,attn_indexer,hc_prenorm,hop_b,hop_b_wide,early_trigger,hop_b_start_trigger,draft,draft_moe,draft_gate,draft_tail,draft_post,draft_seam,draft_attn,draft_qfront,draft_wo_a,draft_swapab,draft_qkv_a,draft_prenorm,draft_qkv_a_ll,draft_wo_b_ll,sched_ahead,req_pos_once,kv_before_wait,target_lm_head,hop_b_quad_trigger,lm_head_pinned_tactic - SGLANG_DSPARK_FUSED_ACCEPT_TAIL: '1' - SGLANG_DSPARK_DEFER_CONTEXT_KV: '1' - SGLANG_OPT_DSV41_DSPARK_DRAFT_PREP_FORK: '1' - SGLANG_OPT_DSV41_DSPARK_POST_FOLD: '1' - SGLANG_OPT_DSV41_DSPARK_SKIP_PREFILLED_INPUTS: '1' - SGLANG_OPT_DSV41_DSPARK_ONE_LAUNCH_COMMIT_STAGE: '1' - SGLANG_OPT_DSV41_DSPARK_PREPARE_INPUTS_ONCE: '1' - SGLANG_OPT_DSV41_DSPARK_MID_FOLD: '1' - SGLANG_OPT_DSV41_TARGET_VERIFY_METADATA_GLUE: '1' - SGLANG_OPT_DSV41_TARGET_METADATA_EARLY: '1' - SGLANG_OPT_SKIP_GLUED_REPLAY_VIEW: '1' - SGLANG_OPT_DSV41_POST_FORWARD_CUDA_GRAPH: '1' - SGLANG_OPT_DSV41_BLOCKWISE_REJECTION: '1' - SGLANG_OPT_DSV41_SAMPLING_ACCEPT_GRAPH: '1' - SGLANG_SIMULATE_ACC_GREEDY: '0' - SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD: '1' - SGLANG_OPT_MXFP4_DEFERRED_FINALIZED_SHAPE: '1' - SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: '1' - NCCL_CUMEM_ENABLE: '0' - NCCL_MNNVL_ENABLE: '0' + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: CONC: '1' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9bac240427..ca456f9663 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11494,7 +11494,6 @@ - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261028-586355a8, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row) and #46891 (no host sync in DSpark draft-block metadata)." - - "TP4 c1 becomes a batch-1 NVIDIA megakernel point with the settings of the Dynamo + SGLang aggregated TP4 c1 row." - - "TP4 c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." + - "TP4 c1/c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603 scenario-type: [agentic-coding] From 9bd047ac75b8275723fd864987639e29d03efc91 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:53:38 +0800 Subject: [PATCH 6/7] Move the GB300 DSV4.1-Flash SGLang curve to the 20261029 nightly and run TP4 c1 on the megakernel nightly-dev-cu13-20261029-c10e0e17 bundles sgl-nvidia-mega 54b428921749 (megakernel API 2), so TP4 c1 returns to its megakernel config. The 16 runtime-optimization variables it set one by one are replaced by SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION (sgl-project/sglang#47105). --- .../sglang/gb300-fp4-mtp/agentic.yaml | 28 +++++++++++++++---- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 3 +- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 073680f300..aba988f354 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -190,15 +190,31 @@ override_tp4_c1: args: tensor-parallel-size: 4 expert-parallel-size: 1 + mem-fraction-static: 0.91 + kv-cache-dtype: fp8_e4m3 dsv4-attn-backend: flashmla - chunked-prefill-size: 16384 - prefill-decode-interval: 0 - max-running-requests: 1 + moe-a2a-backend: none + enable-nvidia-megakernel: true + startup-weight-load-mode: serial enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled + context-length: 1048576 + max-running-requests: 1 + cuda-graph-max-bs-decode: 1 + prefill-decode-interval: 16 + schedule-policy: lpm + swa-prefix-tails: 128 + chunked-prefill-size: 16384 + enable-metrics-for-all-schedulers: true env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' - SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_SPEC_TP_SYNC: rng,-dspark-accept-sample + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_NVIDIA_MEGAKERNEL_SLOTS: hop_a,moe,gate,attn,embed,two_shot,markov,draft_lm_head,aux_mean_stats,gate_wide,shared_ll,hop_a_batch2,engram,attn_qkv_a,attn_qkv_a_bf16,index_q_mxfp8_in,attn_indexer,hc_prenorm,hop_b,hop_b_wide,early_trigger,hop_b_start_trigger,draft,draft_moe,draft_gate,draft_tail,draft_post,draft_seam,draft_attn,draft_qfront,draft_wo_a,draft_swapab,draft_qkv_a,draft_prenorm,draft_qkv_a_ll,draft_wo_b_ll,sched_ahead,req_pos_once,kv_before_wait,target_lm_head,hop_b_quad_trigger,lm_head_pinned_tactic + SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION: '1' + SGLANG_SIMULATE_ACC_GREEDY: '0' + SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD: '1' + NCCL_CUMEM_ENABLE: '0' + NCCL_MNNVL_ENABLE: '0' benchmark: env: CONC: '1' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 5a15f7e1f8..f87cc64608 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9716,7 +9716,7 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90 + image: lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 288a5681a2..9fce1062c4 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11560,7 +11560,8 @@ - config-keys: - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261028-586355a8, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row) and #46891 (no host sync in DSpark draft-block metadata)." + - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261029-c10e0e17, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row), #46891 (no host sync in DSpark draft-block metadata) and sgl-nvidia-mega 54b428921749 (megakernel API 2, sgl-project/sglang#47145)." + - "TP4 c1 runs the NVIDIA megakernel (fp8_e4m3 KV cache, mem-fraction-static 0.91, decode graph batch 1, Engram host table) with SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION=1 (sgl-project/sglang#47105), which enables the DSpark and Qwen3.5/EAGLE megakernel runtime optimizations; DSpark simulated draws run ahead of the draft launch (SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD)." - "TP4 c1/c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603 scenario-type: [agentic-coding] From 6bb311663e4acc0dca7c2bb274627a7f4879422f Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:23:46 +0800 Subject: [PATCH 7/7] Move the GB300 DSV4.1-Flash SGLang key to the 20261030 nightly for the simulated verify accept cost fix Root cause: nightly-dev-cu13-20261029-c10e0e17 predates sgl-project/sglang#46840, so simulated acceptance skipped the real verify accept cost and the TP4 c1 point was not measured on the methodology-conformant path. Fix: pin lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc by digest. It is 63 commits past the #46840 merge and bundles the same sgl-nvidia-mega 54b428921749 (megakernel API 2). Update this PR's changelog entry to name the image and #46840. Validation: Dynamo + SGLang TP4 c1 megakernel, c4 and c16 started and completed on 302b78cc with target and draft megakernels active on all four ranks. --- inferencex-e2e/configs/nvidia-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 6dc8b865a2..0aef3f248f 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8002,9 +8002,9 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [56], srt-recipe: 'agentic-tp2-spf.yaml:zip_override_tp2_hicache_spf[4]' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf.yaml:override_tp2_c48_spf_interval32' } -# Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. +# Official SGLang nightly; every point uses the host-DRAM Engram table. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47 + image: lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 32cd1d8129..d7401bed83 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11674,7 +11674,7 @@ - config-keys: - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261029-c10e0e17, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row), #46891 (no host sync in DSpark draft-block metadata) and sgl-nvidia-mega 54b428921749 (megakernel API 2, sgl-project/sglang#47145)." + - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261030-302b78cc, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row), #46840 (simulated acceptance keeps the real verify accept cost), #46891 (no host sync in DSpark draft-block metadata) and sgl-nvidia-mega 54b428921749 (megakernel API 2, sgl-project/sglang#47145)." - "TP4 c1 runs the NVIDIA megakernel (fp8_e4m3 KV cache, mem-fraction-static 0.91, decode graph batch 1, Engram host table) with SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION=1 (sgl-project/sglang#47105), which enables the DSpark and Qwen3.5/EAGLE megakernel runtime optimizations; DSpark simulated draws run ahead of the draft launch (SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD)." - "TP4 c1/c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603