From 01c0414cc30dc851691347dd1f9c2b7280fadc7b Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Fri, 6 Nov 2026 09:11:25 -0800 Subject: [PATCH 01/12] Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs --- .../vllm/b300-fp4-mtp/agentic-ep.yaml | 121 ++++++++++++++ .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 155 +++++------------- inferencex-e2e/configs/nvidia-master.yaml | 12 +- inferencex-e2e/perf-changelog.yaml | 8 + 4 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml new file mode 100644 index 0000000000..284d54bfc3 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -0,0 +1,121 @@ +base: + schema: 2 + name: kimik3.1-fp4-b300-vllm-agentic-ep + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode JIT can take + # most of VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + tensor-parallel-size: 8 + # Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8. + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + cp-kv-cache-interleave-size: 1 + trust-remote-code: true + language-model-only: true + load-format: fastsafetensors + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + kv-cache-dtype: fp8 + # The DSpark draft layers run TOKENSPEED_MLA (speculative-config); min_split_kv bounds its split-KV workspace. + attention-config: '{"indexer_kv_dtype":"mxfp4","tokenspeed_mla_min_split_kv":16}' + # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. + mamba-ssm-cache-dtype: float32 + # KDA verify replays the accepted prefix instead of keeping one state per draft token. + use-replayssm: true + no-enable-flashinfer-autotune: true + enable-prefix-caching: true + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.95 + stream-interval: 10 + disable-uvicorn-access-log: true + # TOTAL_CPU_DRAM_GB (dram-utilization 1.0) split across the 8 TP ranks. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":374875000000,"lazy_offload":false}}' + # Hybrid decode: TP8 x DCP8 attention, routed experts EP8 on MegaMoE. + # Routed experts EP8 on MegaMoE (see the note below the base). + enable-expert-parallel: true + moe-backend: deep_gemm_mega_moe + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + # Outlast AIPerf's pooled-connection idle gaps. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + # The specialized decode runs on the V2 model runner. + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ENABLE_GEMM_RS: '0' + TOKENSPEED_MLA_FP16_PARTIALS: '0' + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2999' + +# Hybrid decode: TP8 x DCP8 attention, routed experts EP8 on MegaMoE. This path keeps the +# fused Kimi-K3 decode at every batch size (no 152-CTA wide-step MoE), at the cost of about +# 20 GiB per GPU for the MegaMoE buffers. Admission is 2x CONC; FULL_DECODE_ONLY graphs +# cover every decode batch; throughput runs replace block rejection with the golden acceptance length. + +override_c32: + roles: + agg: + args: + max-num-seqs: 64 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '32' + +override_c48: + roles: + agg: + args: + max-num-seqs: 96 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350,360,370,380,390,400,410,420,430,440,450,460,470,480]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '48' + +override_c56: + roles: + agg: + args: + max-num-seqs: 112 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350,360,370,380,390,400,410,420,430,440,450,460,470,480,490,500,510,520,530,540,550,560]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '56' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 21413562a5..456fdb0006 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -14,7 +14,7 @@ base: engine: type: vllm connector: null - # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode JIT can take # most of VLLM_ENGINE_READY_TIMEOUT_S. health_check: interval_seconds: 10 @@ -27,6 +27,10 @@ base: args: served-model-name: moonshotai/Kimi-K3.1-Pro tensor-parallel-size: 8 + # Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8. + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + cp-kv-cache-interleave-size: 1 trust-remote-code: true language-model-only: true load-format: fastsafetensors @@ -34,9 +38,13 @@ base: tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 kv-cache-dtype: fp8 - attention-config: '{"indexer_kv_dtype":"mxfp4"}' + # The DSpark draft layers run TOKENSPEED_MLA (speculative-config); min_split_kv bounds its split-KV workspace. + attention-config: '{"indexer_kv_dtype":"mxfp4","tokenspeed_mla_min_split_kv":32}' # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. mamba-ssm-cache-dtype: float32 + # KDA verify replays the accepted prefix instead of keeping one state per draft token. + use-replayssm: true + no-enable-flashinfer-autotune: true enable-prefix-caching: true max-num-batched-tokens: 16384 gpu-memory-utilization: 0.9 @@ -51,149 +59,72 @@ base: PYTHONHASHSEED: '42' # Outlast AIPerf's pooled-connection idle gaps. VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + # The specialized decode runs on the V2 model runner. + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ENABLE_GEMM_RS: '0' + TOKENSPEED_MLA_FP16_PARTIALS: '0' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC. DSpark drafts match the SGLang recipe's DFLASH drafts: 7 through c8, -# 6 at c16, 5 at c24-c40 and 4 from c48. The largest decode graph covers max-num-seqs x (1 + drafts), -# rounded up to a default capture size; throughput runs replace block rejection with the -# golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode -# collectives cost most; DCP8 from c4 for ~7x the KV tokens. -# Low-latency decode (the fused CuTe DSL path) runs only where every decode graph has at -# most 64 rows (c1-c4): this image launches its 65-320-row MoE experts on 152 CTAs that -# must all be resident, which a 148-SM B300 cannot hold. -override_tp8_c1: - roles: - agg: - args: - additional-config: '{"decode_backend":"latency_specialized"}' - max-num-seqs: 2 - max-cudagraph-capture-size: 16 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '1' +# Admission is 2x CONC up to c8, then near the decode graphs. The fused Kimi-K3 decode +# (decode_backend latency_specialized) captures steps of up to 64 rows on B300: its wider-step +# MoE launches 152 CTAs that must all be co-resident, more than a B300 holds. A step above the +# captured sizes runs eager at about 10x the graphed step time, so admission stays near the 64-row cap. +# FULL_DECODE_ONLY graphs; throughput runs replace block rejection with the golden acceptance length. -override_tp8_c2: +override_c1: roles: agg: args: + max-num-seqs: 2 additional-config: '{"decode_backend":"latency_specialized"}' - max-num-seqs: 4 - max-cudagraph-capture-size: 32 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '2' + CONC: '1' -override_dcp8_c4: +override_c4: roles: agg: args: - additional-config: '{"decode_backend":"latency_specialized"}' - decode-context-parallel-size: 8 max-num-seqs: 8 - max-cudagraph-capture-size: 64 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '4' -override_dcp8_c8: +override_c8: roles: agg: args: - decode-context-parallel-size: 8 max-num-seqs: 16 - max-cudagraph-capture-size: 128 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '8' -override_dcp8_c16: +override_c16: roles: agg: args: - decode-context-parallel-size: 8 - max-num-seqs: 32 - max-cudagraph-capture-size: 224 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-num-seqs: 20 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: '16' - -override_dcp8_c24: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 48 - max-cudagraph-capture-size: 288 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '24' - -override_dcp8_c32: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 64 - max-cudagraph-capture-size: 384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '32' - -override_dcp8_c40: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 80 - max-cudagraph-capture-size: 480 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '40' - -override_dcp8_c48: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 96 - max-cudagraph-capture-size: 480 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '48' - -override_dcp8_c56: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 112 - max-cudagraph-capture-size: 560 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '56' - -override_dcp8_c70: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 140 - max-cudagraph-capture-size: 704 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '70' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index fe4f5aad00..d765fb33bb 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5944,7 +5944,7 @@ dsv41flash-fp4-b300-vllm-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } kimik3.1-fp4-b300-vllm-agentic-dspark: - image: vllm/vllm-openai:kimi-k31@sha256:904520f3fefc32414abcedf6ba8dadc9c5978db1bcdaf40f58d6b9036ef30a99 + image: vllm/vllm-openai:kimi-k31-mc@sha256:eb9fcd314d104da39cf7e8055b28662dfe799ced618b62c3d92becf8a149b88b model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:b300-dsxe @@ -5957,9 +5957,13 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # 0.75 resolves to 2249 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.75 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 7 tokens up to conc 8, then 6, 5 and 4. - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens (3 at c16). + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } + # 1.0 resolves to 2999 GB of host DRAM. + - dram-utilization: 1.0 + search-space: + # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens. + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 48, 56], srt-recipe: agentic-ep.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 86f1786a8b..3db5a2a66b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12329,3 +12329,11 @@ - "Update the vLLM image from nightly-dev-arm64-cu130-ac9126e58aa7 to the digest-pinned nightly-00b6c4c5." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4558 scenario-type: [agentic-coding] + +- config-keys: + - kimik3.1-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs: kimi-k31-mc image, fused Kimi-K3 decode on TP8 x DCP8 at every point, routed experts EP8 on MegaMoE from c32." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From 7720d49b2b4c6bb73f4d07af7d2e0ea2037cd761 Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Fri, 6 Nov 2026 10:29:11 -0800 Subject: [PATCH 02/12] Link the B300 vLLM changelog entry to its PR --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3db5a2a66b..eb35b129f3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12336,4 +12336,4 @@ - agentic-coding description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs: kimi-k31-mc image, fused Kimi-K3 decode on TP8 x DCP8 at every point, routed experts EP8 on MegaMoE from c32." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From 19d58647aac4920ad6cf5755d2d2c32d32d1379b Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Fri, 6 Nov 2026 11:26:31 -0800 Subject: [PATCH 03/12] Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs --- .../kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml | 8 ++++---- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 9 ++++----- inferencex-e2e/configs/nvidia-master.yaml | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml index 284d54bfc3..a0d1329986 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -79,10 +79,10 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2999' -# Hybrid decode: TP8 x DCP8 attention, routed experts EP8 on MegaMoE. This path keeps the -# fused Kimi-K3 decode at every batch size (no 152-CTA wide-step MoE), at the cost of about -# 20 GiB per GPU for the MegaMoE buffers. Admission is 2x CONC; FULL_DECODE_ONLY graphs -# cover every decode batch; throughput runs replace block rejection with the golden acceptance length. +# Hybrid decode: TP8 x DCP8 attention with routed experts EP8 on MegaMoE and the fused Kimi-K3 decode at every batch +# size. Sequence-parallel MoE replicates the shared expert on each rank (about 20 GiB more weights per GPU). +# Admission is 2x CONC; FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with +# the golden acceptance length. override_c32: roles: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 456fdb0006..8de6096cc7 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -75,11 +75,10 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC up to c8, then near the decode graphs. The fused Kimi-K3 decode -# (decode_backend latency_specialized) captures steps of up to 64 rows on B300: its wider-step -# MoE launches 152 CTAs that must all be co-resident, more than a B300 holds. A step above the -# captured sizes runs eager at about 10x the graphed step time, so admission stays near the 64-row cap. -# FULL_DECODE_ONLY graphs; throughput runs replace block rejection with the golden acceptance length. +# Admission is 2x CONC up to c8, then near the decode graphs. FULL_DECODE_ONLY graphs cover decode steps of up to +# 64 rows with the fused Kimi-K3 decode (decode_backend latency_specialized); a step above the captured sizes runs +# eager at about 10x the graphed step time, so admission stays near that cap. Throughput runs replace block rejection +# with the golden acceptance length. override_c1: roles: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d765fb33bb..b9fced4c55 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5944,7 +5944,7 @@ dsv41flash-fp4-b300-vllm-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } kimik3.1-fp4-b300-vllm-agentic-dspark: - image: vllm/vllm-openai:kimi-k31-mc@sha256:eb9fcd314d104da39cf7e8055b28662dfe799ced618b62c3d92becf8a149b88b + image: vllm/vllm-openai:kimi-k31-mc@sha256:f81998764e8594d71b40788c5285ae112acf9230129d5d0d238b5df83da5acf6 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:b300-dsxe From f04e86b6b7293d595129bb132ff8f385ae3268f3 Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Fri, 6 Nov 2026 18:10:14 -0800 Subject: [PATCH 04/12] Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs --- .../vllm/b300-fp4-mtp/agentic-ep.yaml | 25 ++++-------- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 38 ++++++++++++++++--- inferencex-e2e/configs/nvidia-master.yaml | 7 ++-- inferencex-e2e/perf-changelog.yaml | 2 +- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml index a0d1329986..e02c236c30 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -47,7 +47,7 @@ base: no-enable-flashinfer-autotune: true enable-prefix-caching: true max-num-batched-tokens: 8192 - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.97 stream-interval: 10 disable-uvicorn-access-log: true # TOTAL_CPU_DRAM_GB (dram-utilization 1.0) split across the 8 TP ranks. @@ -81,26 +81,15 @@ base: # Hybrid decode: TP8 x DCP8 attention with routed experts EP8 on MegaMoE and the fused Kimi-K3 decode at every batch # size. Sequence-parallel MoE replicates the shared expert on each rank (about 20 GiB more weights per GPU). -# Admission is 2x CONC; FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with +# c48 admits 2x CONC with max-num-batched-tokens 16384; c64 admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with # the golden acceptance length. -override_c32: - roles: - agg: - args: - max-num-seqs: 64 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '32' - override_c48: roles: agg: args: max-num-seqs: 96 + max-num-batched-tokens: 16384 additional-config: '{"decode_backend":"latency_specialized"}' compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350,360,370,380,390,400,410,420,430,440,450,460,470,480]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' @@ -108,14 +97,14 @@ override_c48: env: CONC: '48' -override_c56: +override_c64: roles: agg: args: - max-num-seqs: 112 + max-num-seqs: 64 additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350,360,370,380,390,400,410,420,430,440,450,460,470,480,490,500,510,520,530,540,550,560]}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '56' + CONC: '64' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 8de6096cc7..3186e5fc3b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -75,6 +75,8 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' +# c24 and c32 capture decode steps of up to 240 and 320 rows (wider steps), admission 2x CONC; c32 uses +# dram-utilization 1.0 (its offload size and TOTAL_CPU_DRAM_GB are set in its override). # Admission is 2x CONC up to c8, then near the decode graphs. FULL_DECODE_ONLY graphs cover decode steps of up to # 64 rows with the fused Kimi-K3 decode (decode_backend latency_specialized); a step above the captured sizes runs # eager at about 10x the graphed step time, so admission stays near that cap. Throughput runs replace block rejection @@ -116,14 +118,40 @@ override_c8: env: CONC: '8' -override_c16: +override_c12: roles: agg: args: - max-num-seqs: 20 + max-num-seqs: 16 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '12' + +override_c24: + roles: + agg: + args: + max-num-seqs: 48 additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '24' + +override_c32: + roles: + agg: + args: + max-num-seqs: 64 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":374875000000,"lazy_offload":false}}' + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '16' + CONC: '32' + TOTAL_CPU_DRAM_GB: '2999' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9a58cab170..9f24d4aaa7 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5957,13 +5957,14 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # 0.75 resolves to 2249 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.75 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens (3 at c16). - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 12, 24], srt-recipe: agentic.yaml } # 1.0 resolves to 2999 GB of host DRAM. - dram-utilization: 1.0 search-space: + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agentic.yaml } # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens. - - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 48, 56], srt-recipe: agentic-ep.yaml } + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64], srt-recipe: agentic-ep.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a138c34186..5ea39e1545 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12358,5 +12358,5 @@ scenario-type: - agentic-coding description: - - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs: kimi-k31-mc image, fused Kimi-K3 decode on TP8 x DCP8 at every point, routed experts EP8 on MegaMoE from c32." + - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From 9ecae5a2465eede80d6fd89cc6040baa148d6f38 Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Fri, 6 Nov 2026 18:20:47 -0800 Subject: [PATCH 05/12] Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs --- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 17 +---------------- inferencex-e2e/configs/nvidia-master.yaml | 1 - 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 3186e5fc3b..99c553d6f2 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -75,8 +75,7 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# c24 and c32 capture decode steps of up to 240 and 320 rows (wider steps), admission 2x CONC; c32 uses -# dram-utilization 1.0 (its offload size and TOTAL_CPU_DRAM_GB are set in its override). +# c24 captures decode steps of up to 240 rows (wider steps) with admission 2x CONC. # Admission is 2x CONC up to c8, then near the decode graphs. FULL_DECODE_ONLY graphs cover decode steps of up to # 64 rows with the fused Kimi-K3 decode (decode_backend latency_specialized); a step above the captured sizes runs # eager at about 10x the graphed step time, so admission stays near that cap. Throughput runs replace block rejection @@ -141,17 +140,3 @@ override_c24: benchmark: env: CONC: '24' - -override_c32: - roles: - agg: - args: - max-num-seqs: 64 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":374875000000,"lazy_offload":false}}' - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '32' - TOTAL_CPU_DRAM_GB: '2999' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 512ad59115..3bf03e5015 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5964,7 +5964,6 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # 1.0 resolves to 2999 GB of host DRAM. - dram-utilization: 1.0 search-space: - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agentic.yaml } # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens. - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64], srt-recipe: agentic-ep.yaml } From cc6bab2c04f3a4ed9277723fedd18f16cb847e34 Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Sat, 7 Nov 2026 16:08:48 -0800 Subject: [PATCH 06/12] Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs Add concurrency 6 and 16 to the TP8 x DCP8 recipe. Run concurrency 24, 40 and 48 on the #4878 recipe, which also graphs prefill and mixed steps up to 4096 tokens, without the TEP shared-expert variable. Keep concurrency 64 on EP8 MegaMoE with max-num-batched-tokens 16384. Co-authored-by: Bowen Wang --- .../vllm/b300-fp4-mtp/agentic-ep.yaml | 17 +-- .../b300-fp4-mtp/agentic-mixed-graphs.yaml | 127 ++++++++++++++++++ .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 29 ++-- inferencex-e2e/configs/nvidia-master.yaml | 8 +- inferencex-e2e/perf-changelog.yaml | 3 + 5 files changed, 158 insertions(+), 26 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml index e02c236c30..1f9c67a90d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -46,7 +46,7 @@ base: use-replayssm: true no-enable-flashinfer-autotune: true enable-prefix-caching: true - max-num-batched-tokens: 8192 + max-num-batched-tokens: 16384 gpu-memory-utilization: 0.97 stream-interval: 10 disable-uvicorn-access-log: true @@ -81,22 +81,9 @@ base: # Hybrid decode: TP8 x DCP8 attention with routed experts EP8 on MegaMoE and the fused Kimi-K3 decode at every batch # size. Sequence-parallel MoE replicates the shared expert on each rank (about 20 GiB more weights per GPU). -# c48 admits 2x CONC with max-num-batched-tokens 16384; c64 admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with +# c64 admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with # the golden acceptance length. -override_c48: - roles: - agg: - args: - max-num-seqs: 96 - max-num-batched-tokens: 16384 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350,360,370,380,390,400,410,420,430,440,450,460,470,480]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '48' - override_c64: roles: agg: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml new file mode 100644 index 0000000000..dbfa1de650 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml @@ -0,0 +1,127 @@ +base: + schema: 2 + name: kimik3.1-fp4-b300-vllm-agentic-mixed-graphs + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take + # most of VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + tensor-parallel-size: 8 + trust-remote-code: true + language-model-only: true + load-format: fastsafetensors + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + kv-cache-dtype: fp8 + attention-config: '{"indexer_kv_dtype":"mxfp4"}' + # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. + mamba-ssm-cache-dtype: float32 + enable-prefix-caching: true + additional-config: '{"decode_backend":"latency_specialized"}' + max-num-batched-tokens: 16384 + # Prefill and mixed steps up to 4096 tokens replay graphs instead of running eager + # (~150 ms of launch overhead per step); larger steps are GPU-bound. + max-cudagraph-capture-size: 4096 + # Prefix-cache hits are KV-bound on B300. + gpu-memory-utilization: 0.95 + stream-interval: 10 + disable-uvicorn-access-log: true + # TOTAL_CPU_DRAM_GB split across the 8 TP ranks. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":281125000000,"lazy_offload":false}}' + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + # Outlast AIPerf's pooled-connection idle gaps. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2249' + +# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c24 runs TP8 x DCP8 with DSpark +# drafting 3 tokens; c40 and c48 add routed experts EP8 on MegaMoE with the one-sided NVLink all-to-all and draft 4. +# Throughput runs replace block rejection with the golden acceptance length. + +override_dcp8_c24: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + use-replayssm: true + max-num-seqs: 96 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276, 280, 284, 288, 292, 296, 300, 304, 308, 312, 316, 320, 324, 328, 332, 336, 340, 344, 348, 352, 356, 360, 364, 368, 372, 376, 380, 384, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + benchmark: + env: + CONC: '24' + +override_dcp8_ep8_c40: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + use-replayssm: true + enable-expert-parallel: true + moe-backend: deep_gemm_mega_moe + all2all-backend: flashinfer_nvlink_one_sided + max-num-seqs: 96 + cudagraph-capture-sizes: [1, 2, 4, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110, 115, 120, 125, 130, 135, 140, 145, 150, 155, 160, 165, 170, 175, 180, 185, 190, 195, 200, 205, 210, 215, 220, 225, 230, 235, 240, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + benchmark: + env: + CONC: '40' + +override_dcp8_ep8_c48: + roles: + agg: + args: + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + use-replayssm: true + enable-expert-parallel: true + moe-backend: deep_gemm_mega_moe + all2all-backend: flashinfer_nvlink_one_sided + max-num-seqs: 96 + cudagraph-capture-sizes: [1, 2, 4, 5, 8, 10, 15, 16, 20, 24, 25, 30, 32, 35, 40, 45, 48, 50, 55, 56, 60, 64, 65, 70, 72, 75, 80, 85, 88, 90, 95, 96, 100, 104, 105, 110, 112, 115, 120, 125, 128, 130, 135, 136, 140, 144, 145, 150, 152, 155, 160, 165, 168, 170, 175, 176, 180, 184, 185, 190, 192, 195, 200, 205, 208, 210, 215, 216, 220, 224, 225, 230, 232, 235, 240, 245, 248, 250, 255, 256, 260, 265, 270, 272, 275, 280, 285, 288, 290, 295, 300, 304, 305, 310, 315, 320, 325, 330, 335, 336, 340, 345, 350, 352, 355, 360, 365, 368, 370, 375, 380, 384, 385, 390, 395, 400, 405, 410, 415, 416, 420, 425, 430, 432, 435, 440, 445, 448, 450, 455, 460, 464, 465, 470, 475, 480, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + benchmark: + env: + CONC: '48' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 99c553d6f2..05b61b9024 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -75,11 +75,10 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# c24 captures decode steps of up to 240 rows (wider steps) with admission 2x CONC. -# Admission is 2x CONC up to c8, then near the decode graphs. FULL_DECODE_ONLY graphs cover decode steps of up to -# 64 rows with the fused Kimi-K3 decode (decode_backend latency_specialized); a step above the captured sizes runs -# eager at about 10x the graphed step time, so admission stays near that cap. Throughput runs replace block rejection -# with the golden acceptance length. +# Admission is 2x CONC, except CONC at c6 and 16 at c12. FULL_DECODE_ONLY graphs cover decode steps of 5 rows +# (1 + 4 draft tokens) per admitted request (up to 60 rows at c12) with the fused Kimi-K3 decode (decode_backend +# latency_specialized); a step above the captured sizes runs eager at about 10x the graphed step time. Throughput +# runs replace block rejection with the golden acceptance length. override_c1: roles: @@ -105,6 +104,18 @@ override_c4: env: CONC: '4' +override_c6: + roles: + agg: + args: + max-num-seqs: 6 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '6' + override_c8: roles: agg: @@ -129,14 +140,14 @@ override_c12: env: CONC: '12' -override_c24: +override_c16: roles: agg: args: - max-num-seqs: 48 + max-num-seqs: 32 additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240]}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '24' + CONC: '16' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 3bf03e5015..4bc79d8fb7 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5960,12 +5960,16 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: - dram-utilization: 0.75 search-space: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 12, 24], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 6, 8, 12, 16], srt-recipe: agentic.yaml } + # CUDA graphs also cover prefill and mixed steps up to 4096 tokens: TP8 x DCP8 with DSpark drafting 3 tokens at + # c24, routed experts EP8 on MegaMoE drafting 4 at c40 and c48. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: agentic-mixed-graphs.yaml } + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40, 48], srt-recipe: agentic-mixed-graphs.yaml } # 1.0 resolves to 2999 GB of host DRAM. - dram-utilization: 1.0 search-space: # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens. - - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64], srt-recipe: agentic-ep.yaml } + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [64], srt-recipe: agentic-ep.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ca037bc666..95a739d346 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12413,4 +12413,7 @@ - agentic-coding description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." + - "Concurrency 1-16: TP8 x DCP8 with FULL_DECODE_ONLY graphs; add concurrency 6 and 16." + - "Concurrency 24, 40 and 48: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens at 24, EP8 MegaMoE at 40 and 48." + - "Concurrency 64: EP8 MegaMoE with max-num-batched-tokens 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From eb118b9b661d8ca111f8dc29f19496e2a30ccb18 Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Sun, 8 Nov 2026 15:03:09 -0800 Subject: [PATCH 07/12] Tune Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs Admit prefill once every 8 engine steps (prefill-schedule-interval 8) at concurrency 24, 40, 48, 64 and 72, so more steps run the fused decode. Move concurrency 48 to the EP8 MegaMoE recipe with FULL_DECODE_ONLY graphs, raise gpu-memory-utilization to 0.98 at concurrency 64, and add concurrency 72. Signed-off-by: Mi Sun Park --- .../vllm/b300-fp4-mtp/agentic-ep.yaml | 35 +++++++++++++++++-- .../b300-fp4-mtp/agentic-mixed-graphs.yaml | 27 +++----------- inferencex-e2e/configs/nvidia-master.yaml | 9 ++--- inferencex-e2e/perf-changelog.yaml | 4 +-- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml index 1f9c67a90d..c1a12da482 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -47,7 +47,7 @@ base: no-enable-flashinfer-autotune: true enable-prefix-caching: true max-num-batched-tokens: 16384 - gpu-memory-utilization: 0.97 + gpu-memory-utilization: 0.98 stream-interval: 10 disable-uvicorn-access-log: true # TOTAL_CPU_DRAM_GB (dram-utilization 1.0) split across the 8 TP ranks. @@ -81,13 +81,30 @@ base: # Hybrid decode: TP8 x DCP8 attention with routed experts EP8 on MegaMoE and the fused Kimi-K3 decode at every batch # size. Sequence-parallel MoE replicates the shared expert on each rank (about 20 GiB more weights per GPU). -# c64 admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with +# Each point admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with # the golden acceptance length. +override_c48: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + max-num-seqs: 48 + gpu-memory-utilization: 0.97 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '48' + override_c64: roles: agg: args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 max-num-seqs: 64 additional-config: '{"decode_backend":"latency_specialized"}' compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' @@ -95,3 +112,17 @@ override_c64: benchmark: env: CONC: '64' + +override_c72: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + max-num-seqs: 72 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320,325,330,335,340,345,350,355,360]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '72' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml index dbfa1de650..2ca3a084b8 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml @@ -63,13 +63,15 @@ base: TOTAL_CPU_DRAM_GB: '2249' # CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c24 runs TP8 x DCP8 with DSpark -# drafting 3 tokens; c40 and c48 add routed experts EP8 on MegaMoE with the one-sided NVLink all-to-all and draft 4. +# drafting 3 tokens; c40 adds routed experts EP8 on MegaMoE with the one-sided NVLink all-to-all and drafts 4. # Throughput runs replace block rejection with the golden acceptance length. override_dcp8_c24: roles: agg: args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 decode-context-parallel-size: 8 dcp-comm-backend: a2a use-replayssm: true @@ -88,6 +90,8 @@ override_dcp8_ep8_c40: roles: agg: args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 decode-context-parallel-size: 8 dcp-comm-backend: a2a use-replayssm: true @@ -104,24 +108,3 @@ override_dcp8_ep8_c40: benchmark: env: CONC: '40' - -override_dcp8_ep8_c48: - roles: - agg: - args: - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - use-replayssm: true - enable-expert-parallel: true - moe-backend: deep_gemm_mega_moe - all2all-backend: flashinfer_nvlink_one_sided - max-num-seqs: 96 - cudagraph-capture-sizes: [1, 2, 4, 5, 8, 10, 15, 16, 20, 24, 25, 30, 32, 35, 40, 45, 48, 50, 55, 56, 60, 64, 65, 70, 72, 75, 80, 85, 88, 90, 95, 96, 100, 104, 105, 110, 112, 115, 120, 125, 128, 130, 135, 136, 140, 144, 145, 150, 152, 155, 160, 165, 168, 170, 175, 176, 180, 184, 185, 190, 192, 195, 200, 205, 208, 210, 215, 216, 220, 224, 225, 230, 232, 235, 240, 245, 248, 250, 255, 256, 260, 265, 270, 272, 275, 280, 285, 288, 290, 295, 300, 304, 305, 310, 315, 320, 325, 330, 335, 336, 340, 345, 350, 352, 355, 360, 365, 368, 370, 375, 380, 384, 385, 390, 395, 400, 405, 410, 415, 416, 420, 425, 430, 432, 435, 440, 445, 448, 450, 455, 460, 464, 465, 470, 475, 480, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - env: - VLLM_USE_DIRECT_DCP_A2A: '1' - VLLM_USE_DIRECT_DCP_Q_GATHER: '1' - VLLM_USE_DIRECT_DCP_KV_GATHER: '1' - benchmark: - env: - CONC: '48' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9947d13d30..a9b0d0c7b8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5962,14 +5962,15 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens. - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 6, 8, 12, 16], srt-recipe: agentic.yaml } # CUDA graphs also cover prefill and mixed steps up to 4096 tokens: TP8 x DCP8 with DSpark drafting 3 tokens at - # c24, routed experts EP8 on MegaMoE drafting 4 at c40 and c48. + # c24, routed experts EP8 on MegaMoE drafting 4 at c40. Prefill is admitted once every 8 engine steps. - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: agentic-mixed-graphs.yaml } - - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40, 48], srt-recipe: agentic-mixed-graphs.yaml } + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: agentic-mixed-graphs.yaml } # 1.0 resolves to 2999 GB of host DRAM. - dram-utilization: 1.0 search-space: - # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens. - - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [64], srt-recipe: agentic-ep.yaml } + # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens and prefill is + # admitted once every 8 engine steps. + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64, 72], srt-recipe: agentic-ep.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 0977042d6d..8a8bde3367 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12512,6 +12512,6 @@ description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." - "Concurrency 1-16: TP8 x DCP8 with FULL_DECODE_ONLY graphs; add concurrency 6 and 16." - - "Concurrency 24, 40 and 48: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens at 24, EP8 MegaMoE at 40 and 48." - - "Concurrency 64: EP8 MegaMoE with max-num-batched-tokens 16384." + - "Concurrency 24 and 40: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens at 24, EP8 MegaMoE at 40; prefill-schedule-interval 8." + - "Concurrency 48, 64 and 72: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384 and prefill-schedule-interval 8; gpu-memory-utilization 0.98 at 64 and 72; add concurrency 72." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From e8994358cefb5aa34dd86a1f4cf790b53f2bd0ad Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Sun, 8 Nov 2026 16:13:15 -0800 Subject: [PATCH 08/12] Add concurrency 20 to the Kimi-K3.1-Pro B300 vLLM AgentX configs Run concurrency 20 on the mixed-graph recipe with the concurrency-24 settings: TP8 x DCP8, DSpark drafting 3 tokens, CUDA graphs for prefill and mixed steps up to 4096 tokens, and prefill-schedule-interval 8. Signed-off-by: Mi Sun Park --- .../b300-fp4-mtp/agentic-mixed-graphs.yaml | 22 ++++++++++++++++++- inferencex-e2e/configs/nvidia-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml index 2ca3a084b8..73686590d3 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml @@ -62,10 +62,30 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c24 runs TP8 x DCP8 with DSpark +# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c20 and c24 run TP8 x DCP8 with DSpark # drafting 3 tokens; c40 adds routed experts EP8 on MegaMoE with the one-sided NVLink all-to-all and drafts 4. # Throughput runs replace block rejection with the golden acceptance length. +override_dcp8_c20: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + use-replayssm: true + max-num-seqs: 96 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276, 280, 284, 288, 292, 296, 300, 304, 308, 312, 316, 320, 324, 328, 332, 336, 340, 344, 348, 352, 356, 360, 364, 368, 372, 376, 380, 384, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + benchmark: + env: + CONC: '20' + override_dcp8_c24: roles: agg: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index a9b0d0c7b8..c141425ee5 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5962,8 +5962,8 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens. - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 6, 8, 12, 16], srt-recipe: agentic.yaml } # CUDA graphs also cover prefill and mixed steps up to 4096 tokens: TP8 x DCP8 with DSpark drafting 3 tokens at - # c24, routed experts EP8 on MegaMoE drafting 4 at c40. Prefill is admitted once every 8 engine steps. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: agentic-mixed-graphs.yaml } + # c20 and c24, routed experts EP8 on MegaMoE drafting 4 at c40. Prefill is admitted once every 8 engine steps. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 24], srt-recipe: agentic-mixed-graphs.yaml } - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: agentic-mixed-graphs.yaml } # 1.0 resolves to 2999 GB of host DRAM. - dram-utilization: 1.0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8a8bde3367..69a2db3283 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12512,6 +12512,6 @@ description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." - "Concurrency 1-16: TP8 x DCP8 with FULL_DECODE_ONLY graphs; add concurrency 6 and 16." - - "Concurrency 24 and 40: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens at 24, EP8 MegaMoE at 40; prefill-schedule-interval 8." + - "Concurrency 20, 24 and 40: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens at 20 and 24, EP8 MegaMoE at 40; prefill-schedule-interval 8; add concurrency 20." - "Concurrency 48, 64 and 72: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384 and prefill-schedule-interval 8; gpu-memory-utilization 0.98 at 64 and 72; add concurrency 72." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From e46f75b7ca3b9be51df23fe2ceee5abd3fb4310e Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Mon, 9 Nov 2026 04:51:33 -0800 Subject: [PATCH 09/12] Narrow the Kimi-K3.1-Pro B300 vLLM AgentX sweep to seven frontier points Sweep concurrency 1, 6, 12, 24, 48, 64 and 72. The dropped points (4, 8, 16, 20 and 40) sit on or below the line between their neighbours, so the curve keeps its shape with fewer runs. Concurrency 12 moves to the mixed-graph recipe with the concurrency-24 settings: TP8 x DCP8, DSpark drafting 3 tokens, CUDA graphs for prefill and mixed steps up to 4096 tokens, and prefill-schedule-interval 8. On the same image this raised P90 interactivity by about 10 % at the same throughput per GPU as the FULL_DECODE_ONLY recipe at concurrency 12. Signed-off-by: Mi Sun Park --- .../b300-fp4-mtp/agentic-mixed-graphs.yaml | 31 ++--------- .../kimik3.1/vllm/b300-fp4-mtp/agentic.yaml | 52 +------------------ inferencex-e2e/configs/nvidia-master.yaml | 7 ++- inferencex-e2e/perf-changelog.yaml | 5 +- 4 files changed, 12 insertions(+), 83 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml index 73686590d3..9a7b1b242e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml @@ -62,11 +62,11 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c20 and c24 run TP8 x DCP8 with DSpark -# drafting 3 tokens; c40 adds routed experts EP8 on MegaMoE with the one-sided NVLink all-to-all and drafts 4. +# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c12 and c24 run TP8 x DCP8 +# with DSpark drafting 3 tokens. # Throughput runs replace block rejection with the golden acceptance length. -override_dcp8_c20: +override_dcp8_c12: roles: agg: args: @@ -84,7 +84,7 @@ override_dcp8_c20: VLLM_USE_DIRECT_DCP_KV_GATHER: '1' benchmark: env: - CONC: '20' + CONC: '12' override_dcp8_c24: roles: @@ -105,26 +105,3 @@ override_dcp8_c24: benchmark: env: CONC: '24' - -override_dcp8_ep8_c40: - roles: - agg: - args: - # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. - prefill-schedule-interval: 8 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - use-replayssm: true - enable-expert-parallel: true - moe-backend: deep_gemm_mega_moe - all2all-backend: flashinfer_nvlink_one_sided - max-num-seqs: 96 - cudagraph-capture-sizes: [1, 2, 4, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110, 115, 120, 125, 130, 135, 140, 145, 150, 155, 160, 165, 170, 175, 180, 185, 190, 195, 200, 205, 210, 215, 220, 225, 230, 235, 240, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - env: - VLLM_USE_DIRECT_DCP_A2A: '1' - VLLM_USE_DIRECT_DCP_Q_GATHER: '1' - VLLM_USE_DIRECT_DCP_KV_GATHER: '1' - benchmark: - env: - CONC: '40' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 05b61b9024..c2b5949775 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -75,8 +75,8 @@ base: KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC, except CONC at c6 and 16 at c12. FULL_DECODE_ONLY graphs cover decode steps of 5 rows -# (1 + 4 draft tokens) per admitted request (up to 60 rows at c12) with the fused Kimi-K3 decode (decode_backend +# Admission is 2 at c1 and 6 at c6. FULL_DECODE_ONLY graphs cover decode steps of 5 rows +# (1 + 4 draft tokens) per admitted request (up to 30 rows at c6) with the fused Kimi-K3 decode (decode_backend # latency_specialized); a step above the captured sizes runs eager at about 10x the graphed step time. Throughput # runs replace block rejection with the golden acceptance length. @@ -92,18 +92,6 @@ override_c1: env: CONC: '1' -override_c4: - roles: - agg: - args: - max-num-seqs: 8 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '4' - override_c6: roles: agg: @@ -115,39 +103,3 @@ override_c6: benchmark: env: CONC: '6' - -override_c8: - roles: - agg: - args: - max-num-seqs: 16 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '8' - -override_c12: - roles: - agg: - args: - max-num-seqs: 16 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '12' - -override_c16: - roles: - agg: - args: - max-num-seqs: 32 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '16' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index c141425ee5..24e8b46db2 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5960,11 +5960,10 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: - dram-utilization: 0.75 search-space: # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 6, 8, 12, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 6], srt-recipe: agentic.yaml } # CUDA graphs also cover prefill and mixed steps up to 4096 tokens: TP8 x DCP8 with DSpark drafting 3 tokens at - # c20 and c24, routed experts EP8 on MegaMoE drafting 4 at c40. Prefill is admitted once every 8 engine steps. - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 24], srt-recipe: agentic-mixed-graphs.yaml } - - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: agentic-mixed-graphs.yaml } + # c12 and c24. Prefill is admitted once every 8 engine steps. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [12, 24], srt-recipe: agentic-mixed-graphs.yaml } # 1.0 resolves to 2999 GB of host DRAM. - dram-utilization: 1.0 search-space: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 69a2db3283..e5c78033ec 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12511,7 +12511,8 @@ - agentic-coding description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." - - "Concurrency 1-16: TP8 x DCP8 with FULL_DECODE_ONLY graphs; add concurrency 6 and 16." - - "Concurrency 20, 24 and 40: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens at 20 and 24, EP8 MegaMoE at 40; prefill-schedule-interval 8; add concurrency 20." + - "Sweep seven frontier points: concurrency 1, 6, 12, 24, 48, 64 and 72." + - "Concurrency 1 and 6: TP8 x DCP8 with FULL_DECODE_ONLY graphs." + - "Concurrency 12 and 24: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens; prefill-schedule-interval 8." - "Concurrency 48, 64 and 72: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384 and prefill-schedule-interval 8; gpu-memory-utilization 0.98 at 64 and 72; add concurrency 72." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From 84e415859593c6e9d9876146b3e0f7f8b95499b3 Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Mon, 9 Nov 2026 18:03:11 -0800 Subject: [PATCH 10/12] Lower Kimi-K3.1-Pro B300 concurrency-72 gpu-memory-utilization to 0.97 The concurrency-72 eval job's acceptance-length gate sends 80 requests at once. At gpu-memory-utilization 0.98 the resulting prefill step overflowed the direct DCP workspace, and the fallback combine ran out of GPU memory on every rank, so the eval could not run. Use 0.97, as at concurrency 48. Signed-off-by: Mi Sun Park --- .../kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml index c1a12da482..05d51c106f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -120,6 +120,8 @@ override_c72: # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. prefill-schedule-interval: 8 max-num-seqs: 72 + # 0.97 as at 48: at 0.98 an 80-request prefill burst ran out of memory in the DCP combine fallback. + gpu-memory-utilization: 0.97 additional-config: '{"decode_backend":"latency_specialized"}' compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320,325,330,335,340,345,350,355,360]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 0d2dcb0a7b..0673886060 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12572,5 +12572,5 @@ - "Sweep seven frontier points: concurrency 1, 6, 12, 24, 48, 64 and 72." - "Concurrency 1 and 6: TP8 x DCP8 with FULL_DECODE_ONLY graphs." - "Concurrency 12 and 24: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens; prefill-schedule-interval 8." - - "Concurrency 48, 64 and 72: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384 and prefill-schedule-interval 8; gpu-memory-utilization 0.98 at 64 and 72; add concurrency 72." + - "Concurrency 48, 64 and 72: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384 and prefill-schedule-interval 8; gpu-memory-utilization 0.98 at 64 and 0.97 at 72; add concurrency 72." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From ec76df1cbceb90c1d7fdf16f688b2528076f2f1d Mon Sep 17 00:00:00 2001 From: Mi Sun Park Date: Tue, 10 Nov 2026 07:44:27 -0800 Subject: [PATCH 11/12] Drop Kimi-K3.1-Pro B300 concurrency 72; run concurrency 64 at 0.97 Concurrency 72 had a P90 TTFT of 10.7 s, about four times the other points, for 2 % more throughput per GPU than concurrency 64. Its TTFT tail is engine queue time while requests wait for GPU KV cache, not prefill or preemption, so no setting within this recipe brings it under a few seconds. Concurrency 64 becomes the highest point. The agentic eval now runs at concurrency 64, so it takes gpu-memory-utilization 0.97 like concurrency 48: at 0.98 the eval's 80-request acceptance-length burst ran out of memory in the DCP combine fallback. At 0.97, concurrency 64 measured the same throughput per GPU as at 0.98. Signed-off-by: Mi Sun Park --- .../kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml | 18 ++---------------- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml index 05d51c106f..8bcf97a1ab 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -106,25 +106,11 @@ override_c64: # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. prefill-schedule-interval: 8 max-num-seqs: 64 - additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '64' - -override_c72: - roles: - agg: - args: - # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. - prefill-schedule-interval: 8 - max-num-seqs: 72 # 0.97 as at 48: at 0.98 an 80-request prefill burst ran out of memory in the DCP combine fallback. gpu-memory-utilization: 0.97 additional-config: '{"decode_backend":"latency_specialized"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320,325,330,335,340,345,350,355,360]}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '72' + CONC: '64' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 06e4b8a957..aa21eed1f9 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5977,7 +5977,7 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: search-space: # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens and prefill is # admitted once every 8 engine steps. - - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64, 72], srt-recipe: agentic-ep.yaml } + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64], srt-recipe: agentic-ep.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 0673886060..d56ff16f7a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12569,8 +12569,8 @@ - agentic-coding description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." - - "Sweep seven frontier points: concurrency 1, 6, 12, 24, 48, 64 and 72." + - "Sweep six frontier points: concurrency 1, 6, 12, 24, 48 and 64." - "Concurrency 1 and 6: TP8 x DCP8 with FULL_DECODE_ONLY graphs." - "Concurrency 12 and 24: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens; prefill-schedule-interval 8." - - "Concurrency 48, 64 and 72: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384 and prefill-schedule-interval 8; gpu-memory-utilization 0.98 at 64 and 0.97 at 72; add concurrency 72." + - "Concurrency 48 and 64: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384, prefill-schedule-interval 8 and gpu-memory-utilization 0.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871 From 0211db699b83ffc9eaf212aee3b36c3cf0da1df1 Mon Sep 17 00:00:00 2001 From: Ankur Singh Date: Wed, 11 Nov 2026 15:36:53 -0800 Subject: [PATCH 12/12] perf-changelog: describe the Kimi-K3.1-Pro B300 sweep as six points [skip-sweep] Concurrency 1 is not on the throughput/E2EL frontier, so drop 'frontier' from the entry. Signed-off-by: Ankur Singh Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 607579bf18..e4aefc5587 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12613,7 +12613,7 @@ - agentic-coding description: - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." - - "Sweep six frontier points: concurrency 1, 6, 12, 24, 48 and 64." + - "Sweep six points: concurrency 1, 6, 12, 24, 48 and 64." - "Concurrency 1 and 6: TP8 x DCP8 with FULL_DECODE_ONLY graphs." - "Concurrency 12 and 24: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens; prefill-schedule-interval 8." - "Concurrency 48 and 64: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384, prefill-schedule-interval 8 and gpu-memory-utilization 0.97."