Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
schema: 2

base:
name: kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg
slurm:
time_limit: '08:00:00'
sbatch_directives:
mem: '0'
srun_options:
container-remap-root: ''
container-writable: ''
mem: '0'
container_mounts:
/dev/infiniband: /dev/infiniband
extra_mount:
# The image has no Ionic verbs provider; libibverbs loads the host's only through its driver entry.
- /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro
- /etc/libibverbs.d/ionic.driver:/etc/libibverbs.d/ionic.driver:ro
resources:
gpu_type: mi355x
gpus_per_node: 8
frontend:
type: sglang-router
enable_multiple_frontends: false
placement:
node: head
args:
request-timeout-secs: 3600
observability:
enabled: false
tachometer:
enabled: false
# Loading the 1.56 TB checkpoint plus decode and prefill graph capture.
health_check:
max_attempts: 720
interval_seconds: 10
engine: sglang
roles:
prefill:
nodes: 1
workers: 1
gpus: 8
args:
served-model-name: moonshotai/Kimi-K3.1-Pro
trust-remote-code: true
tensor-parallel-size: 8
dcp-size: 8
dcp-comm-backend: a2a
kv-cache-dtype: fp8_e4m3
enable-dsa-fp4-indexer: true
mem-fraction-static: 0.85
# 32k chunks put the long-prefill sparse MLA on a Triton path that hangs below ~34 GB of free VRAM.
chunked-prefill-size: 16384
max-prefill-tokens: 16384
schedule-policy: shortest-prefill-first
mamba-radix-cache-strategy: extra_buffer
mamba-full-memory-ratio: 0.65
mamba-max-states-per-path: 1
# Checkpoints a KDA state every 64 tokens, so a resumed turn misses at most 63 cached tokens.
mamba-track-interval: 64
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-ratio: 4.0
cuda-graph-backend-prefill: breakable
cuda-graph-bs-prefill: [256, 512, 768, 1024, 1536, 2048, 3072, 4096]
disaggregation-transfer-backend: mooncake
disaggregation-ib-device: '@fabric.rdma-devices'
disaggregation-decode-allocation-policy: early
weight-loader-prefetch-checkpoints: true
model-loader-extra-config: '{"enable_multithread_load": true}'
reasoning-parser: kimi_k3
tool-call-parser: kimi_k3
enable-metrics: true
speculative-algorithm: DFLASH
speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview
speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1
speculative-draft-attention-backend: triton
speculative-draft-kv-cache-dtype: fp8_e4m3
# The golden AL curve was collected with block verification.
speculative-use-block-verification: true
# The current token plus 4 drafts; the prefill writes the draft KV ring the decode reads.
speculative-num-draft-tokens: 5
random-seed: 42
env: &common_env
PYTHONNOUSERSITE: '1'
PYTHONUNBUFFERED: '1'
HF_HOME: /hf_hub_cache
HF_HUB_CACHE: /hf_hub_cache/hub
HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub
NCCL_SOCKET_IFNAME: '@fabric.socket-ifname'
GLOO_SOCKET_IFNAME: '@fabric.socket-ifname'
MC_GID_INDEX: '@fabric.rdma-gid-index'
MC_MAX_CONCURRENT_REG_MR: '8'
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600'
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600'
# Outlasts the router's pooled connections, idle through the warmup's long tail.
SGLANG_TIMEOUT_KEEP_ALIVE: '7200'
SGLANG_USE_AITER: '1'
SGLANG_AITER_K3_OPT: '1'
AITER_FLYDSL_FORCE: '1'
AITER_SITUV2_A8W4: '1'
# BF16 MoE combine and all-reduce.
AITER_OPUS_STAGE2_FP8: '0'
ROCM_QUICK_REDUCE_QUANTIZATION: NONE
PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.95
SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1'
# Split-K HIP sparse MLA decode and verify for 16 to 512 rows.
SGLANG_ROCM_SPARSE_MLA_DECODE_HIP: '1'
SGLANG_ROCM_RUNTIME_RESERVE_GB: '10'
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
# Every rank commits rank 0's greedy accept; a rank that diverges hangs the batch.
SGLANG_SPEC_TP_SYNC: init,dflash-accept-greedy
# Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one.
RUSTUP_TOOLCHAIN: stable
# Pinned host pools; registered pageable pools stall every GPU queue on page migration.
HSA_USERPTR_FOR_PAGED_MEM: '0'
# The MI355X nodes report MEC firmware 38, below the 177 that fixes the
# RCCL memory reclaim issue.
HSA_NO_SCRATCH_RECLAIM: '1'
decode:
nodes: 1
workers: 1
gpus: 8
args:
served-model-name: moonshotai/Kimi-K3.1-Pro
trust-remote-code: true
tensor-parallel-size: 8
dcp-size: 8
dcp-comm-backend: a2a
kv-cache-dtype: fp8_e4m3
enable-dsa-fp4-indexer: true
mem-fraction-static: 0.92
mamba-radix-cache-strategy: extra_buffer
disaggregation-transfer-backend: mooncake
disaggregation-ib-device: '@fabric.rdma-devices'
disaggregation-decode-extra-slots: 16
disaggregation-decode-enable-radix-cache: true
weight-loader-prefetch-checkpoints: true
model-loader-extra-config: '{"enable_multithread_load": true}'
reasoning-parser: kimi_k3
tool-call-parser: kimi_k3
enable-metrics: true
speculative-algorithm: DFLASH
speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview
speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1
speculative-draft-attention-backend: triton
speculative-draft-kv-cache-dtype: fp8_e4m3
speculative-use-block-verification: true
speculative-num-draft-tokens: 5
# Without ReplaySSM, DFLASH keeps one KDA state per draft token and the KV pool collapses.
enable-linear-replayssm-spec: true
random-seed: 42
env:
<<: *common_env
# Decode graphs past ~96 at mem 0.92 abort when the allocator collects during capture.
PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.99
benchmark:
placement:
node: head
env:
# Warmup can leave bytes unacknowledged past AIPerf's 30 s default.
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

override_1p1d_c128:
roles:
prefill: &prefill_1p1d
env:
# 190 GB of pinned host pools per GPU (KV, KDA states, indexer keys) under the node's 1511 GiB GTT.
SGLANG_HICACHE_MAMBA_HOST_RATIO: '5'
SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: '0.1'
decode:
args:
max-running-requests: 144
cuda-graph-max-bs-decode: 144

override_1p1d_c144:
roles:
prefill: *prefill_1p1d
decode:
args:
max-running-requests: 160
cuda-graph-max-bs-decode: 160
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,21 @@ override_tp8_c1:
agg:
args:
enable-amd-monokernel: true
max-running-requests: 4
# One running request: agent fan-out queues instead of sharing decode steps.
max-running-requests: 1
cuda-graph-max-bs-decode: 4
# 150 state slots keep agent sessions' prefixes cached.
max-mamba-cache-size: 150
# The current token plus 7 drafts.
speculative-num-draft-tokens: 8
env:
SGLANG_AMD_MONO_K31_WIDE: '1'
SGLANG_ROCM_RUNTIME_RESERVE_GB: '6'
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
benchmark:
env:
CONC: '1'
Expand All @@ -102,9 +110,16 @@ override_tp8_c4:
cuda-graph-max-bs-decode: 4
# 150 state slots keep agent sessions' prefixes cached.
max-mamba-cache-size: 150
# The current token plus 7 drafts.
speculative-num-draft-tokens: 8
env:
SGLANG_AMD_MONO_K31_WIDE: '1'
SGLANG_ROCM_RUNTIME_RESERVE_GB: '6'
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
benchmark:
env:
CONC: '4'
Expand All @@ -120,14 +135,22 @@ override_tp8_dcp8_c8:
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-ratio: 2.5
max-running-requests: 8
cuda-graph-max-bs-decode: 8
# 6 running requests: the rest of the 8 sessions queue, which keeps decode steps short.
max-running-requests: 6
cuda-graph-max-bs-decode: 6
# Prefill every 32 decode steps: fewer stalls in running requests' token streams.
prefill-decode-interval: 32
env:
SGLANG_ROCM_RUNTIME_RESERVE_GB: '10'
# 170-185 GB of pinned host pools per GPU: the 8 ranks share the node's 1511 GiB GTT.
SGLANG_HICACHE_MAMBA_HOST_RATIO: '10'
# Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one.
RUSTUP_TOOLCHAIN: stable
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
benchmark:
env:
CONC: '8'
Expand All @@ -143,17 +166,58 @@ override_tp8_dcp8_c16:
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-ratio: 2.5
max-running-requests: 16
cuda-graph-max-bs-decode: 16
# 8 running requests: the rest of the 16 sessions queue, which keeps decode steps short.
max-running-requests: 8
cuda-graph-max-bs-decode: 8
# Prefill every 32 decode steps: fewer stalls in running requests' token streams.
prefill-decode-interval: 32

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does e2el improve tho

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be flat on most cc

# The current token plus 4 drafts.
speculative-num-draft-tokens: 5
env:
SGLANG_ROCM_RUNTIME_RESERVE_GB: '10'
SGLANG_HICACHE_MAMBA_HOST_RATIO: '10'
RUSTUP_TOOLCHAIN: stable
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
benchmark:
env:
CONC: '16'
KV_OFFLOADING: dram

override_tp8_dcp8_c24:
roles:
agg:
args:
dcp-size: 8
dcp-comm-backend: a2a
mamba-full-memory-ratio: 0.65
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-ratio: 2.5
# 13 running requests: the rest of the 24 sessions queue, which keeps decode steps short.
max-running-requests: 13
cuda-graph-max-bs-decode: 13
# Prefill every 32 decode steps: fewer stalls in running requests' token streams.
prefill-decode-interval: 32
# The current token plus 4 drafts.
speculative-num-draft-tokens: 5
env:
SGLANG_ROCM_RUNTIME_RESERVE_GB: '10'
SGLANG_HICACHE_MAMBA_HOST_RATIO: '10'
RUSTUP_TOOLCHAIN: stable
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
benchmark:
env:
CONC: '24'
KV_OFFLOADING: dram

override_tp8_dcp8_c32:
roles:
agg:
Expand All @@ -164,12 +228,22 @@ override_tp8_dcp8_c32:
enable-hierarchical-cache: true
hicache-write-policy: write_back
hicache-ratio: 2.5
max-running-requests: 32
cuda-graph-max-bs-decode: 32
# 20 running requests: the rest of the 32 sessions queue, which keeps decode steps short.
max-running-requests: 20
cuda-graph-max-bs-decode: 20
# Prefill every 32 decode steps: fewer stalls in running requests' token streams.
prefill-decode-interval: 32
# The current token plus 4 drafts.
speculative-num-draft-tokens: 5
env:
SGLANG_ROCM_RUNTIME_RESERVE_GB: '10'
SGLANG_HICACHE_MAMBA_HOST_RATIO: '10'
RUSTUP_TOOLCHAIN: stable
# MoE front and residual stream on row shards in prefill (no extra memory).
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1'
SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1'
# A draft block's ring locations in one launch.
SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1'
benchmark:
env:
CONC: '32'
Expand All @@ -188,7 +262,8 @@ override_tp8_dcp8_c48:
max-running-requests: 48
cuda-graph-max-bs-decode: 48
chunked-prefill-size: 32768
prefill-decode-interval: 8
# Prefill every 32 decode steps: fewer stalls in running requests' token streams.
prefill-decode-interval: 32
# The current token plus 4 drafts: from 48 requests the shorter verify outweighs its lower acceptance.
speculative-num-draft-tokens: 5
env:
Expand Down Expand Up @@ -218,7 +293,8 @@ override_tp8_dcp8_c56:
max-running-requests: 56
cuda-graph-max-bs-decode: 56
chunked-prefill-size: 32768
prefill-decode-interval: 8
# Prefill every 32 decode steps: fewer stalls in running requests' token streams.
prefill-decode-interval: 32
# The current token plus 4 drafts: from 48 requests the shorter verify outweighs its lower acceptance.
speculative-num-draft-tokens: 5
env:
Expand Down
Loading
Loading