diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 1d56ebd861..aba988f354 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -64,6 +64,8 @@ override_tp2_c16: args: tensor-parallel-size: 2 expert-parallel-size: 2 + # auto resolves to MegaAttention, which zero-pads per-rank heads to 64 (2x at TP2, 4x at TP4). + dsv4-attn-backend: flashmla <<: &tp2_bounded_replay chunked-prefill-size: 16384 enable-decoder-swa-bounded-replay: true @@ -90,6 +92,7 @@ override_tp2_c32: args: tensor-parallel-size: 2 expert-parallel-size: 2 + dsv4-attn-backend: flashmla <<: *tp2_bounded_replay prefill-decode-interval: 48 tokenizer-worker-num: 4 @@ -187,14 +190,31 @@ override_tp4_c1: args: tensor-parallel-size: 4 expert-parallel-size: 1 - chunked-prefill-size: 16384 - prefill-decode-interval: 0 - max-running-requests: 1 + mem-fraction-static: 0.91 + kv-cache-dtype: fp8_e4m3 + dsv4-attn-backend: flashmla + moe-a2a-backend: none + enable-nvidia-megakernel: true + startup-weight-load-mode: serial enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled + context-length: 1048576 + max-running-requests: 1 + cuda-graph-max-bs-decode: 1 + prefill-decode-interval: 16 + schedule-policy: lpm + swa-prefix-tails: 128 + chunked-prefill-size: 16384 + enable-metrics-for-all-schedulers: true env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' - SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_SPEC_TP_SYNC: rng,-dspark-accept-sample + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_NVIDIA_MEGAKERNEL_SLOTS: hop_a,moe,gate,attn,embed,two_shot,markov,draft_lm_head,aux_mean_stats,gate_wide,shared_ll,hop_a_batch2,engram,attn_qkv_a,attn_qkv_a_bf16,index_q_mxfp8_in,attn_indexer,hc_prenorm,hop_b,hop_b_wide,early_trigger,hop_b_start_trigger,draft,draft_moe,draft_gate,draft_tail,draft_post,draft_seam,draft_attn,draft_qfront,draft_wo_a,draft_swapab,draft_qkv_a,draft_prenorm,draft_qkv_a_ll,draft_wo_b_ll,sched_ahead,req_pos_once,kv_before_wait,target_lm_head,hop_b_quad_trigger,lm_head_pinned_tactic + SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION: '1' + SGLANG_SIMULATE_ACC_GREEDY: '0' + SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD: '1' + NCCL_CUMEM_ENABLE: '0' + NCCL_MNNVL_ENABLE: '0' benchmark: env: CONC: '1' @@ -207,6 +227,7 @@ override_tp4_c4: args: tensor-parallel-size: 4 expert-parallel-size: 1 + dsv4-attn-backend: flashmla chunked-prefill-size: 16384 prefill-decode-interval: 0 swa-prefix-tails: 4096 @@ -228,6 +249,7 @@ override_tp4_c16: args: tensor-parallel-size: 4 expert-parallel-size: 1 + dsv4-attn-backend: flashmla chunked-prefill-size: 16384 prefill-decode-interval: 192 tokenizer-worker-num: 4 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index f939eeeb71..02a3bf1143 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6855,9 +6855,9 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c48_spf_interval24' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [50], srt-recipe: 'agentic-tp2-spf-interval.yaml:override_tp2_c50_spf_interval20' } -# Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. +# Official SGLang nightly; every point uses the host-DRAM Engram table. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb + image: lmsysorg/sglang:nightly-dev-cu13-20261030-302b78cc@sha256:d4b7701dc94d287e4fc135dfb4c1a88df0778a936255021aeb26eb54d396e579 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 29bea27fba..c862bec17f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11935,3 +11935,12 @@ - "Update the search space to TP4/EP4 at concurrency 1 and 2, TP1/EP1 at concurrency 1, 2, 12 and 24, and TP1/EP1 with HiCache at concurrency 48 and 80." - "Switch the target and draft KV caches from BF16 to FP8 E4M3 at concurrency 12, 24, 48 and 80." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4683 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + description: + - "Update the SGLang image from nightly-dev-cu13-20261021-c1c287a4 to nightly-dev-cu13-20261030-302b78cc, pinned by digest. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row), #46840 (simulated acceptance keeps the real verify accept cost), #46891 (no host sync in DSpark draft-block metadata) and sgl-nvidia-mega 54b428921749 (megakernel API 2, sgl-project/sglang#47145)." + - "TP4 c1 runs the NVIDIA megakernel (fp8_e4m3 KV cache, mem-fraction-static 0.91, decode graph batch 1, Engram host table) with SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION=1 (sgl-project/sglang#47105), which enables the DSpark and Qwen3.5/EAGLE megakernel runtime optimizations; DSpark simulated draws run ahead of the draft launch (SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD)." + - "TP4 c1/c4/c16 and TP2/EP2 c16/c32 pin dsv4-attn-backend flashmla; auto resolves to MegaAttention, which zero-pads per-rank attention heads to 64. DP-attention points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4603 + scenario-type: [agentic-coding]