Skip to content

Commit fe35c3c

Browse files
committed
fix(kimik3.1): keep only documented or common recipe settings
Remove the idle-GPU reaper job comment, the NCCL/UCX/Dynamo env overrides, the FlashInfer all-reduce and execute-timeout env vars, and the latency-specialized decode on the disaggregated pools, which vLLM documents only for single-node TP8 decode; the c24-c64 1P2D points now share one variant.
1 parent ccdd30e commit fe35c3c

4 files changed

Lines changed: 3 additions & 49 deletions

File tree

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb300-fp4/agentx/agg-tp8-lowlat-dspark7.yaml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ roles:
2424
HF_HUB_CACHE: /hf_hub_cache
2525
VLLM_ENGINE_READY_TIMEOUT_S: '3600'
2626
VLLM_RPC_TIMEOUT: '600000'
27-
VLLM_ALLREDUCE_USE_FLASHINFER: '1'
2827
args:
2928
served-model-name: moonshotai/Kimi-K3.1-Pro
3029
enable-auto-tool-choice: true
@@ -48,7 +47,6 @@ roles:
4847
sbatch_directives:
4948
mem: '0'
5049
cpus-per-task: '72'
51-
comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'''
5250
srun_options:
5351
mem: '0'
5452
container-remap-root: ''

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb300-fp4/agentx/disagg-dep16.yaml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ frontend:
3636
DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600'
3737
DYN_TOKENIZER_CACHE_BYTES: '8589934592'
3838
DYN_TCP_REQUEST_TIMEOUT: '120'
39-
DYN_DISABLE_OPENAI_STRICT_SCHEMA_VALIDATION: '1'
4039
engine:
4140
type: vllm
4241
connector: null
@@ -54,8 +53,6 @@ roles:
5453
VLLM_SSM_CONV_STATE_LAYOUT: DS
5554
VLLM_ENGINE_READY_TIMEOUT_S: '3600'
5655
VLLM_RPC_TIMEOUT: '600000'
57-
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800'
58-
VLLM_ALLREDUCE_USE_FLASHINFER: '1'
5956
DYN_REQUEST_PLANE: tcp
6057
DYN_TCP_REQUEST_TIMEOUT: '120'
6158
ETCD_LEASE_TTL: '600'
@@ -64,12 +61,8 @@ roles:
6461
NCCL_CUMEM_ENABLE: '1'
6562
NCCL_MNNVL_ENABLE: '1'
6663
NCCL_NVLS_ENABLE: '1'
67-
NCCL_P2P_LEVEL: NVL
68-
NCCL_NET_PLUGIN: none
6964
UCX_MEMTYPE_CACHE: 'n'
7065
UCX_MEMTYPE_REG_WHOLE: 'n'
71-
UCX_RCACHE_MAX_UNRELEASED: '1024'
72-
UCX_TCP_AF_PRIO: inet
7366
args:
7467
served-model-name: moonshotai/Kimi-K3.1-Pro
7568
trust-remote-code: true
@@ -124,7 +117,6 @@ roles:
124117
sbatch_directives:
125118
mem: '0'
126119
cpus-per-task: '72'
127-
comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'''
128120
srun_options:
129121
mem: '0'
130122
container-remap-root: ''

‎inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/vllm/gb300-fp4/agentx/disagg-variants.yaml‎

Lines changed: 1 addition & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ base:
3838
DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600'
3939
DYN_TOKENIZER_CACHE_BYTES: '8589934592'
4040
DYN_TCP_REQUEST_TIMEOUT: '120'
41-
DYN_DISABLE_OPENAI_STRICT_SCHEMA_VALIDATION: '1'
4241
engine:
4342
type: vllm
4443
connector: null
@@ -56,8 +55,6 @@ base:
5655
VLLM_SSM_CONV_STATE_LAYOUT: DS
5756
VLLM_ENGINE_READY_TIMEOUT_S: '3600'
5857
VLLM_RPC_TIMEOUT: '600000'
59-
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800'
60-
VLLM_ALLREDUCE_USE_FLASHINFER: '1'
6158
DYN_REQUEST_PLANE: tcp
6259
DYN_TCP_REQUEST_TIMEOUT: '120'
6360
ETCD_LEASE_TTL: '600'
@@ -66,12 +63,8 @@ base:
6663
NCCL_CUMEM_ENABLE: '1'
6764
NCCL_MNNVL_ENABLE: '1'
6865
NCCL_NVLS_ENABLE: '1'
69-
NCCL_P2P_LEVEL: NVL
70-
NCCL_NET_PLUGIN: none
7166
UCX_MEMTYPE_CACHE: 'n'
7267
UCX_MEMTYPE_REG_WHOLE: 'n'
73-
UCX_RCACHE_MAX_UNRELEASED: '1024'
74-
UCX_TCP_AF_PRIO: inet
7568
args:
7669
served-model-name: moonshotai/Kimi-K3.1-Pro
7770
trust-remote-code: true
@@ -120,7 +113,6 @@ base:
120113
sbatch_directives:
121114
mem: '0'
122115
cpus-per-task: '72'
123-
comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'''
124116
srun_options:
125117
mem: '0'
126118
container-remap-root: ''
@@ -135,31 +127,11 @@ base:
135127
AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400'
136128
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
137129

138-
override_1p3d_lowlat:
130+
override_1p3d:
139131
roles:
140-
prefill:
141-
args:
142-
cp-kv-cache-interleave-size: 1
143132
decode:
144133
nodes: 6
145134
workers: 3
146-
args: &lowlat_decode_args
147-
# Kept at 1 on both roles: NIXL otherwise raises it to the block size after the fused decode reads it.
148-
cp-kv-cache-interleave-size: 1
149-
additional-config: '{"decode_backend":"latency_specialized"}'
150-
# The fused decode path covers batches up to 64 tokens: 64 // (1 + 4 drafts).
151-
max-num-seqs: 12
152-
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[5,10,15,20,25,30,35,40,45,50,55,60]}'
153-
154-
override_1p2d_lowlat:
155-
roles:
156-
prefill:
157-
args:
158-
cp-kv-cache-interleave-size: 1
159-
decode:
160-
nodes: 4
161-
workers: 2
162-
args: *lowlat_decode_args
163135

164136
override_1p2d:
165137
roles:

‎inferencex-e2e/configs/nvidia-master.yaml‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7129,22 +7129,14 @@ kimik3.1-fp4-gb300-dynamo-vllm-agentic-dspark-disagg:
71297129
kv-offloading: dram
71307130
kv-offload-backend: { name: vllm-simple }
71317131
conc-list: [16]
7132-
srt-recipe: disagg-variants.yaml:override_1p3d_lowlat
7132+
srt-recipe: disagg-variants.yaml:override_1p3d
71337133
power: true
71347134
prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 8, dp-attn: false }
71357135
decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
71367136
- spec-decoding: mtp
71377137
kv-offloading: dram
71387138
kv-offload-backend: { name: vllm-simple }
7139-
conc-list: [24, 32]
7140-
srt-recipe: disagg-variants.yaml:override_1p2d_lowlat
7141-
power: true
7142-
prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 8, dp-attn: false }
7143-
decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
7144-
- spec-decoding: mtp
7145-
kv-offloading: dram
7146-
kv-offload-backend: { name: vllm-simple }
7147-
conc-list: [48, 64]
7139+
conc-list: [24, 32, 48, 64]
71487140
srt-recipe: disagg-variants.yaml:override_1p2d
71497141
power: true
71507142
prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 8, dp-attn: false }

0 commit comments

Comments
 (0)