diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml new file mode 100644 index 0000000000..8bcf97a1ab --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-ep.yaml @@ -0,0 +1,116 @@ +base: + schema: 2 + name: kimik3.1-fp4-b300-vllm-agentic-ep + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode JIT can take + # most of VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + tensor-parallel-size: 8 + # Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8. + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + cp-kv-cache-interleave-size: 1 + trust-remote-code: true + language-model-only: true + load-format: fastsafetensors + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + kv-cache-dtype: fp8 + # The DSpark draft layers run TOKENSPEED_MLA (speculative-config); min_split_kv bounds its split-KV workspace. + attention-config: '{"indexer_kv_dtype":"mxfp4","tokenspeed_mla_min_split_kv":16}' + # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. + mamba-ssm-cache-dtype: float32 + # KDA verify replays the accepted prefix instead of keeping one state per draft token. + use-replayssm: true + no-enable-flashinfer-autotune: true + enable-prefix-caching: true + max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.98 + stream-interval: 10 + disable-uvicorn-access-log: true + # TOTAL_CPU_DRAM_GB (dram-utilization 1.0) split across the 8 TP ranks. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":374875000000,"lazy_offload":false}}' + # Hybrid decode: TP8 x DCP8 attention, routed experts EP8 on MegaMoE. + # Routed experts EP8 on MegaMoE (see the note below the base). + enable-expert-parallel: true + moe-backend: deep_gemm_mega_moe + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + # Outlast AIPerf's pooled-connection idle gaps. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + # The specialized decode runs on the V2 model runner. + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ENABLE_GEMM_RS: '0' + TOKENSPEED_MLA_FP16_PARTIALS: '0' + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2999' + +# Hybrid decode: TP8 x DCP8 attention with routed experts EP8 on MegaMoE and the fused Kimi-K3 decode at every batch +# size. Sequence-parallel MoE replicates the shared expert on each rank (about 20 GiB more weights per GPU). +# Each point admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with +# the golden acceptance length. + +override_c48: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + max-num-seqs: 48 + gpu-memory-utilization: 0.97 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '48' + +override_c64: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + max-num-seqs: 64 + # 0.97 as at 48: at 0.98 an 80-request prefill burst ran out of memory in the DCP combine fallback. + gpu-memory-utilization: 0.97 + additional-config: '{"decode_backend":"latency_specialized"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + benchmark: + env: + CONC: '64' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml new file mode 100644 index 0000000000..9a7b1b242e --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic-mixed-graphs.yaml @@ -0,0 +1,107 @@ +base: + schema: 2 + name: kimik3.1-fp4-b300-vllm-agentic-mixed-graphs + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take + # most of VLLM_ENGINE_READY_TIMEOUT_S. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + tensor-parallel-size: 8 + trust-remote-code: true + language-model-only: true + load-format: fastsafetensors + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + kv-cache-dtype: fp8 + attention-config: '{"indexer_kv_dtype":"mxfp4"}' + # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. + mamba-ssm-cache-dtype: float32 + enable-prefix-caching: true + additional-config: '{"decode_backend":"latency_specialized"}' + max-num-batched-tokens: 16384 + # Prefill and mixed steps up to 4096 tokens replay graphs instead of running eager + # (~150 ms of launch overhead per step); larger steps are GPU-bound. + max-cudagraph-capture-size: 4096 + # Prefix-cache hits are KV-bound on B300. + gpu-memory-utilization: 0.95 + stream-interval: 10 + disable-uvicorn-access-log: true + # TOTAL_CPU_DRAM_GB split across the 8 TP ranks. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":281125000000,"lazy_offload":false}}' + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + # Identical prefixes must hash to identical block keys run-to-run. + PYTHONHASHSEED: '42' + # Outlast AIPerf's pooled-connection idle gaps. + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '2249' + +# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c12 and c24 run TP8 x DCP8 +# with DSpark drafting 3 tokens. +# Throughput runs replace block rejection with the golden acceptance length. + +override_dcp8_c12: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + use-replayssm: true + max-num-seqs: 96 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276, 280, 284, 288, 292, 296, 300, 304, 308, 312, 316, 320, 324, 328, 332, 336, 340, 344, 348, 352, 356, 360, 364, 368, 372, 376, 380, 384, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + benchmark: + env: + CONC: '12' + +override_dcp8_c24: + roles: + agg: + args: + # Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only. + prefill-schedule-interval: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + use-replayssm: true + max-num-seqs: 96 + cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276, 280, 284, 288, 292, 296, 300, 304, 308, 312, 316, 320, 324, 328, 332, 336, 340, 344, 348, 352, 356, 360, 364, 368, 372, 376, 380, 384, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096] + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + benchmark: + env: + CONC: '24' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml index 21413562a5..c2b5949775 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/vllm/b300-fp4-mtp/agentic.yaml @@ -14,7 +14,7 @@ base: engine: type: vllm connector: null - # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take + # Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode JIT can take # most of VLLM_ENGINE_READY_TIMEOUT_S. health_check: interval_seconds: 10 @@ -27,6 +27,10 @@ base: args: served-model-name: moonshotai/Kimi-K3.1-Pro tensor-parallel-size: 8 + # Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8. + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + cp-kv-cache-interleave-size: 1 trust-remote-code: true language-model-only: true load-format: fastsafetensors @@ -34,9 +38,13 @@ base: tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 kv-cache-dtype: fp8 - attention-config: '{"indexer_kv_dtype":"mxfp4"}' + # The DSpark draft layers run TOKENSPEED_MLA (speculative-config); min_split_kv bounds its split-KV workspace. + attention-config: '{"indexer_kv_dtype":"mxfp4","tokenspeed_mla_min_split_kv":32}' # The K3.1 week-1 precision policy requires an FP32 KDA recurrent state. mamba-ssm-cache-dtype: float32 + # KDA verify replays the accepted prefix instead of keeping one state per draft token. + use-replayssm: true + no-enable-flashinfer-autotune: true enable-prefix-caching: true max-num-batched-tokens: 16384 gpu-memory-utilization: 0.9 @@ -51,149 +59,47 @@ base: PYTHONHASHSEED: '42' # Outlast AIPerf's pooled-connection idle gaps. VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900' + # The specialized decode runs on the V2 model runner. + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + VLLM_USE_NCCL_SYMM_MEM: '0' + VLLM_ENABLE_GEMM_RS: '0' + TOKENSPEED_MLA_FP16_PARTIALS: '0' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '2249' -# Admission is 2x CONC. DSpark drafts match the SGLang recipe's DFLASH drafts: 7 through c8, -# 6 at c16, 5 at c24-c40 and 4 from c48. The largest decode graph covers max-num-seqs x (1 + drafts), -# rounded up to a default capture size; throughput runs replace block rejection with the -# golden acceptance length. Plain TP8 at c1-c2, where the KV fits on GPU and DCP's decode -# collectives cost most; DCP8 from c4 for ~7x the KV tokens. -# Low-latency decode (the fused CuTe DSL path) runs only where every decode graph has at -# most 64 rows (c1-c4): this image launches its 65-320-row MoE experts on 152 CTAs that -# must all be resident, which a 148-SM B300 cannot hold. -override_tp8_c1: - roles: - agg: - args: - additional-config: '{"decode_backend":"latency_specialized"}' - max-num-seqs: 2 - max-cudagraph-capture-size: 16 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '1' +# Admission is 2 at c1 and 6 at c6. FULL_DECODE_ONLY graphs cover decode steps of 5 rows +# (1 + 4 draft tokens) per admitted request (up to 30 rows at c6) with the fused Kimi-K3 decode (decode_backend +# latency_specialized); a step above the captured sizes runs eager at about 10x the graphed step time. Throughput +# runs replace block rejection with the golden acceptance length. -override_tp8_c2: +override_c1: roles: agg: args: + max-num-seqs: 2 additional-config: '{"decode_backend":"latency_specialized"}' - max-num-seqs: 4 - max-cudagraph-capture-size: 32 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '2' + CONC: '1' -override_dcp8_c4: +override_c6: roles: agg: args: + max-num-seqs: 6 additional-config: '{"decode_backend":"latency_specialized"}' - decode-context-parallel-size: 8 - max-num-seqs: 8 - max-cudagraph-capture-size: 64 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '4' - -override_dcp8_c8: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 16 - max-cudagraph-capture-size: 128 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '8' - -override_dcp8_c16: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 32 - max-cudagraph-capture-size: 224 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '16' - -override_dcp8_c24: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 48 - max-cudagraph-capture-size: 288 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '24' - -override_dcp8_c32: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 64 - max-cudagraph-capture-size: 384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '32' - -override_dcp8_c40: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 80 - max-cudagraph-capture-size: 480 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '40' - -override_dcp8_c48: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 96 - max-cudagraph-capture-size: 480 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '48' - -override_dcp8_c56: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 112 - max-cudagraph-capture-size: 560 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - benchmark: - env: - CONC: '56' - -override_dcp8_c70: - roles: - agg: - args: - decode-context-parallel-size: 8 - max-num-seqs: 140 - max-cudagraph-capture-size: 704 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30]}' + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: '70' + CONC: '6' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index aeae3e0a27..0b0c99c10b 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5914,7 +5914,7 @@ dsv41flash-fp4-b300-vllm-agentic-dspark: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [224, 256, 384, 544, 768, 1024, 1280], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } kimik3.1-fp4-b300-vllm-agentic-dspark: - image: vllm/vllm-openai:kimi-k31@sha256:904520f3fefc32414abcedf6ba8dadc9c5978db1bcdaf40f58d6b9036ef30a99 + image: vllm/vllm-openai:kimi-k31-mc@sha256:f81998764e8594d71b40788c5285ae112acf9230129d5d0d238b5df83da5acf6 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:b300-dsxe @@ -5927,9 +5927,17 @@ kimik3.1-fp4-b300-vllm-agentic-dspark: # 0.75 resolves to 2249 GB of host DRAM, which the recipe's Simple CPU offload splits across the 8 TP ranks. - dram-utilization: 0.75 search-space: - # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. DSpark drafts 7 tokens up to conc 8, then 6, 5 and 4. - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: agentic.yaml } + # The ~1.56 TB MXFP4 checkpoint needs all 8 GPUs. Fused Kimi-K3 decode on TP8 x DCP8; DSpark drafts 4 tokens. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 6], srt-recipe: agentic.yaml } + # CUDA graphs also cover prefill and mixed steps up to 4096 tokens: TP8 x DCP8 with DSpark drafting 3 tokens at + # c12 and c24. Prefill is admitted once every 8 engine steps. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [12, 24], srt-recipe: agentic-mixed-graphs.yaml } + # 1.0 resolves to 2999 GB of host DRAM. + - dram-utilization: 1.0 + search-space: + # Routed experts EP8 on MegaMoE keep the fused decode at every batch size; DSpark drafts 4 tokens and prefill is + # admitted once every 8 engine steps. + - { tp: 8, ep: 8, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48, 64], srt-recipe: agentic-ep.yaml } dsv41flash-fp4-b200-vllm-agentic-dspark: image: vllm/vllm-openai:cu134-nightly-40941f2c4ba96fba0a7e282a5355e89cd74d9ac5@sha256:13e92fcd97268013d13d07fb97d80803227cf0beae0161f8dce95a8200eaafb0 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9c6e3f1316..0900ede884 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12763,3 +12763,15 @@ description: - "Lower the TP8 concurrency 6 stream interval from 128 to 50 and rerun every point on the same lmsysorg/sglang-rocm:muse-spark-open-mi355-1106 image; the rest of the recipe and the search space are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4983 + +- config-keys: + - kimik3.1-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs." + - "Sweep six points: concurrency 1, 6, 12, 24, 48 and 64." + - "Concurrency 1 and 6: TP8 x DCP8 with FULL_DECODE_ONLY graphs." + - "Concurrency 12 and 24: CUDA graphs also for prefill and mixed steps up to 4096 tokens (#4878 recipe without the TEP shared-expert variable); DSpark drafts 3 tokens; prefill-schedule-interval 8." + - "Concurrency 48 and 64: EP8 MegaMoE with FULL_DECODE_ONLY graphs, max-num-batched-tokens 16384, prefill-schedule-interval 8 and gpu-memory-utilization 0.97." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4871