Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
01c0414
Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs
mispa-ms Nov 6, 2026
7720d49
Link the B300 vLLM changelog entry to its PR
mispa-ms Nov 6, 2026
19d5864
Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs
mispa-ms Nov 6, 2026
e6a093a
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 6, 2026
99f1413
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 6, 2026
1d3935c
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 6, 2026
f04e86b
Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs
mispa-ms Nov 7, 2026
8c7bfde
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 7, 2026
9ecae5a
Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs
mispa-ms Nov 7, 2026
cc6bab2
Update Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs
mispa-ms Nov 8, 2026
c983761
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 8, 2026
8c1b725
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 8, 2026
eb118b9
Tune Kimi-K3.1-Pro B300 vLLM aggregated AgentX configs
mispa-ms Nov 8, 2026
e899435
Add concurrency 20 to the Kimi-K3.1-Pro B300 vLLM AgentX configs
mispa-ms Nov 9, 2026
e46f75b
Narrow the Kimi-K3.1-Pro B300 vLLM AgentX sweep to seven frontier points
mispa-ms Nov 9, 2026
a1d04c2
Merge branch 'main' into kimik31-b300-vllm-fused-decode
mispa-ms Nov 9, 2026
84e4158
Lower Kimi-K3.1-Pro B300 concurrency-72 gpu-memory-utilization to 0.97
mispa-ms Nov 10, 2026
ec76df1
Drop Kimi-K3.1-Pro B300 concurrency 72; run concurrency 64 at 0.97
mispa-ms Nov 10, 2026
75b1a3c
Merge remote-tracking branch 'origin/main' into kimik31-b300-vllm-fus…
mispa-ms Nov 10, 2026
0211db6
perf-changelog: describe the Kimi-K3.1-Pro B300 sweep as six points […
Ankur-singh Nov 11, 2026
02893c1
Merge remote-tracking branch 'origin/main' into pr-4871-reuse-20537
Ankur-singh Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
base:
schema: 2
name: kimik3.1-fp4-b300-vllm-agentic-ep
resources:
gpu_type: b300
gpus_per_node: 8
frontend:
type: vllm
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine:
type: vllm
connector: null
# Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode JIT can take
# most of VLLM_ENGINE_READY_TIMEOUT_S.
health_check:
interval_seconds: 10
max_attempts: 720
roles:
agg:
nodes: 1
workers: 1
gpus: 8
args:
served-model-name: moonshotai/Kimi-K3.1-Pro
tensor-parallel-size: 8
# Shards the MLA latent and indexer-key caches across the TP group: ~7x the KV tokens of plain TP8.
decode-context-parallel-size: 8
dcp-comm-backend: a2a
cp-kv-cache-interleave-size: 1
trust-remote-code: true
language-model-only: true
load-format: fastsafetensors
enable-auto-tool-choice: true
tool-call-parser: kimi_k3
reasoning-parser: kimi_k3
kv-cache-dtype: fp8
# The DSpark draft layers run TOKENSPEED_MLA (speculative-config); min_split_kv bounds its split-KV workspace.
attention-config: '{"indexer_kv_dtype":"mxfp4","tokenspeed_mla_min_split_kv":16}'
# The K3.1 week-1 precision policy requires an FP32 KDA recurrent state.
mamba-ssm-cache-dtype: float32
# KDA verify replays the accepted prefix instead of keeping one state per draft token.
use-replayssm: true
no-enable-flashinfer-autotune: true
enable-prefix-caching: true
max-num-batched-tokens: 16384
gpu-memory-utilization: 0.98
stream-interval: 10
disable-uvicorn-access-log: true
# TOTAL_CPU_DRAM_GB (dram-utilization 1.0) split across the 8 TP ranks.
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":374875000000,"lazy_offload":false}}'
# Hybrid decode: TP8 x DCP8 attention, routed experts EP8 on MegaMoE.
# Routed experts EP8 on MegaMoE (see the note below the base).
enable-expert-parallel: true
moe-backend: deep_gemm_mega_moe
env:
VLLM_ENGINE_READY_TIMEOUT_S: '7200'
PYTHONUNBUFFERED: '1'
# Identical prefixes must hash to identical block keys run-to-run.
PYTHONHASHSEED: '42'
# Outlast AIPerf's pooled-connection idle gaps.
VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900'
# The specialized decode runs on the V2 model runner.
VLLM_USE_V2_MODEL_RUNNER: '1'
VLLM_USE_DIRECT_DCP_A2A: '1'
VLLM_USE_DIRECT_DCP_Q_GATHER: '1'
VLLM_USE_DIRECT_DCP_KV_GATHER: '1'
VLLM_ALLREDUCE_USE_FLASHINFER: '1'
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0'
VLLM_USE_NCCL_SYMM_MEM: '0'
VLLM_ENABLE_GEMM_RS: '0'
TOKENSPEED_MLA_FP16_PARTIALS: '0'
benchmark:
env:
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '2999'

# Hybrid decode: TP8 x DCP8 attention with routed experts EP8 on MegaMoE and the fused Kimi-K3 decode at every batch
# size. Sequence-parallel MoE replicates the shared expert on each rank (about 20 GiB more weights per GPU).
# Each point admits CONC. FULL_DECODE_ONLY graphs cover every decode batch; throughput runs replace block rejection with
# the golden acceptance length.

override_c48:
roles:
agg:
args:
# Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only.
prefill-schedule-interval: 8
max-num-seqs: 48
gpu-memory-utilization: 0.97
additional-config: '{"decode_backend":"latency_specialized"}'
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240]}'
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '48'

override_c64:
roles:
agg:
args:
# Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only.
prefill-schedule-interval: 8
max-num-seqs: 64
# 0.97 as at 48: at 0.98 an 80-request prefill burst ran out of memory in the DCP combine fallback.
gpu-memory-utilization: 0.97
additional-config: '{"decode_backend":"latency_specialized"}'
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60,65,70,75,80,85,90,95,100,105,110,115,120,125,130,135,140,145,150,155,160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270,275,280,285,290,295,300,305,310,315,320]}'
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","attention_backend":"TOKENSPEED_MLA","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
benchmark:
env:
CONC: '64'
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
base:
schema: 2
name: kimik3.1-fp4-b300-vllm-agentic-mixed-graphs
resources:
gpu_type: b300
gpus_per_node: 8
frontend:
type: vllm
enable_multiple_frontends: false
observability:
enabled: false
tachometer:
enabled: false
engine:
type: vllm
connector: null
# Loading ~1.56 TB of MXFP4 shards plus the latency_specialized decode autotune can take
# most of VLLM_ENGINE_READY_TIMEOUT_S.
health_check:
interval_seconds: 10
max_attempts: 720
roles:
agg:
nodes: 1
workers: 1
gpus: 8
args:
served-model-name: moonshotai/Kimi-K3.1-Pro
tensor-parallel-size: 8
trust-remote-code: true
language-model-only: true
load-format: fastsafetensors
enable-auto-tool-choice: true
tool-call-parser: kimi_k3
reasoning-parser: kimi_k3
kv-cache-dtype: fp8
attention-config: '{"indexer_kv_dtype":"mxfp4"}'
# The K3.1 week-1 precision policy requires an FP32 KDA recurrent state.
mamba-ssm-cache-dtype: float32
enable-prefix-caching: true
additional-config: '{"decode_backend":"latency_specialized"}'
max-num-batched-tokens: 16384
# Prefill and mixed steps up to 4096 tokens replay graphs instead of running eager
# (~150 ms of launch overhead per step); larger steps are GPU-bound.
max-cudagraph-capture-size: 4096
# Prefix-cache hits are KV-bound on B300.
gpu-memory-utilization: 0.95
stream-interval: 10
disable-uvicorn-access-log: true
# TOTAL_CPU_DRAM_GB split across the 8 TP ranks.
kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":281125000000,"lazy_offload":false}}'
env:
VLLM_ENGINE_READY_TIMEOUT_S: '7200'
PYTHONUNBUFFERED: '1'
# Identical prefixes must hash to identical block keys run-to-run.
PYTHONHASHSEED: '42'
# Outlast AIPerf's pooled-connection idle gaps.
VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '900'
benchmark:
env:
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '2249'

# CUDA graphs cover prefill and mixed steps up to 4096 tokens as well as decode. c12 and c24 run TP8 x DCP8
# with DSpark drafting 3 tokens.
# Throughput runs replace block rejection with the golden acceptance length.

override_dcp8_c12:
roles:
agg:
args:
# Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only.
prefill-schedule-interval: 8
decode-context-parallel-size: 8
dcp-comm-backend: a2a
use-replayssm: true
max-num-seqs: 96
cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276, 280, 284, 288, 292, 296, 300, 304, 308, 312, 316, 320, 324, 328, 332, 336, 340, 344, 348, 352, 356, 360, 364, 368, 372, 376, 380, 384, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096]
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
env:
VLLM_USE_DIRECT_DCP_A2A: '1'
VLLM_USE_DIRECT_DCP_Q_GATHER: '1'
VLLM_USE_DIRECT_DCP_KV_GATHER: '1'
benchmark:
env:
CONC: '12'

override_dcp8_c24:
roles:
agg:
args:
# Admit prefill once every 8 engine steps; the fused decode serves pure decode steps only.
prefill-schedule-interval: 8
decode-context-parallel-size: 8
dcp-comm-backend: a2a
use-replayssm: true
max-num-seqs: 96
cudagraph-capture-sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276, 280, 284, 288, 292, 296, 300, 304, 308, 312, 316, 320, 324, 328, 332, 336, 340, 344, 348, 352, 356, 360, 364, 368, 372, 376, 380, 384, 512, 640, 768, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096]
speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3.1-Pro-DSpark-preview","revision":"6a2ef20a05efe88aabbf9ecaa12d020c3b0edf94","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
env:
VLLM_USE_DIRECT_DCP_A2A: '1'
VLLM_USE_DIRECT_DCP_Q_GATHER: '1'
VLLM_USE_DIRECT_DCP_KV_GATHER: '1'
benchmark:
env:
CONC: '24'
Loading
Loading