diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..f7496f6cf8 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,187 @@ +schema: 2 + +base: + name: kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg + slurm: + time_limit: '08:00:00' + sbatch_directives: + mem: '0' + srun_options: + container-remap-root: '' + container-writable: '' + mem: '0' + container_mounts: + /dev/infiniband: /dev/infiniband + extra_mount: + # The image has no Ionic verbs provider; libibverbs loads the host's only through its driver entry. + - /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro + - /etc/libibverbs.d/ionic.driver:/etc/libibverbs.d/ionic.driver:ro + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + placement: + node: head + args: + request-timeout-secs: 3600 + observability: + enabled: false + tachometer: + enabled: false + # Loading the 1.56 TB checkpoint plus decode and prefill graph capture. + health_check: + max_attempts: 720 + interval_seconds: 10 + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: a2a + kv-cache-dtype: fp8_e4m3 + enable-dsa-fp4-indexer: true + mem-fraction-static: 0.85 + # 32k chunks put the long-prefill sparse MLA on a Triton path that hangs below ~34 GB of free VRAM. + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-radix-cache-strategy: extra_buffer + mamba-full-memory-ratio: 0.65 + mamba-max-states-per-path: 1 + # Checkpoints a KDA state every 64 tokens, so a resumed turn misses at most 63 cached tokens. + mamba-track-interval: 64 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 4.0 + cuda-graph-backend-prefill: breakable + cuda-graph-bs-prefill: [256, 512, 768, 1024, 1536, 2048, 3072, 4096] + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: '@fabric.rdma-devices' + disaggregation-decode-allocation-policy: early + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + reasoning-parser: kimi_k3 + tool-call-parser: kimi_k3 + enable-metrics: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: triton + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # The current token plus 4 drafts; the prefill writes the draft KV ring the decode reads. + speculative-num-draft-tokens: 5 + random-seed: 42 + env: &common_env + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + NCCL_SOCKET_IFNAME: '@fabric.socket-ifname' + GLOO_SOCKET_IFNAME: '@fabric.socket-ifname' + MC_GID_INDEX: '@fabric.rdma-gid-index' + MC_MAX_CONCURRENT_REG_MR: '8' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' + # Outlasts the router's pooled connections, idle through the warmup's long tail. + SGLANG_TIMEOUT_KEEP_ALIVE: '7200' + SGLANG_USE_AITER: '1' + SGLANG_AITER_K3_OPT: '1' + AITER_FLYDSL_FORCE: '1' + AITER_SITUV2_A8W4: '1' + # BF16 MoE combine and all-reduce. + AITER_OPUS_STAGE2_FP8: '0' + ROCM_QUICK_REDUCE_QUANTIZATION: NONE + PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.95 + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + # Split-K HIP sparse MLA decode and verify for 16 to 512 rows. + SGLANG_ROCM_SPARSE_MLA_DECODE_HIP: '1' + SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' + # Every rank commits rank 0's greedy accept; a rank that diverges hangs the batch. + SGLANG_SPEC_TP_SYNC: init,dflash-accept-greedy + # Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one. + RUSTUP_TOOLCHAIN: stable + # Pinned host pools; registered pageable pools stall every GPU queue on page migration. + HSA_USERPTR_FOR_PAGED_MEM: '0' + # The MI355X nodes report MEC firmware 38, below the 177 that fixes the + # RCCL memory reclaim issue. + HSA_NO_SCRATCH_RECLAIM: '1' + decode: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: a2a + kv-cache-dtype: fp8_e4m3 + enable-dsa-fp4-indexer: true + mem-fraction-static: 0.92 + mamba-radix-cache-strategy: extra_buffer + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: '@fabric.rdma-devices' + disaggregation-decode-extra-slots: 16 + disaggregation-decode-enable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + reasoning-parser: kimi_k3 + tool-call-parser: kimi_k3 + enable-metrics: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: triton + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 5 + # Without ReplaySSM, DFLASH keeps one KDA state per draft token and the KV pool collapses. + enable-linear-replayssm-spec: true + random-seed: 42 + env: + <<: *common_env + # Decode graphs past ~96 at mem 0.92 abort when the allocator collects during capture. + PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.99 + benchmark: + placement: + node: head + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_1p1d_c128: + roles: + prefill: &prefill_1p1d + env: + # 190 GB of pinned host pools per GPU (KV, KDA states, indexer keys) under the node's 1511 GiB GTT. + SGLANG_HICACHE_MAMBA_HOST_RATIO: '5' + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: '0.1' + decode: + args: + max-running-requests: 144 + cuda-graph-max-bs-decode: 144 + +override_1p1d_c144: + roles: + prefill: *prefill_1p1d + decode: + args: + max-running-requests: 160 + cuda-graph-max-bs-decode: 160 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml index 42286764a2..644242f33e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml @@ -81,13 +81,21 @@ override_tp8_c1: agg: args: enable-amd-monokernel: true - max-running-requests: 4 + # One running request: agent fan-out queues instead of sharing decode steps. + max-running-requests: 1 cuda-graph-max-bs-decode: 4 # 150 state slots keep agent sessions' prefixes cached. max-mamba-cache-size: 150 + # The current token plus 7 drafts. + speculative-num-draft-tokens: 8 env: SGLANG_AMD_MONO_K31_WIDE: '1' SGLANG_ROCM_RUNTIME_RESERVE_GB: '6' + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '1' @@ -102,9 +110,16 @@ override_tp8_c4: cuda-graph-max-bs-decode: 4 # 150 state slots keep agent sessions' prefixes cached. max-mamba-cache-size: 150 + # The current token plus 7 drafts. + speculative-num-draft-tokens: 8 env: SGLANG_AMD_MONO_K31_WIDE: '1' SGLANG_ROCM_RUNTIME_RESERVE_GB: '6' + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '4' @@ -120,14 +135,22 @@ override_tp8_dcp8_c8: enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 2.5 - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 + # 6 running requests: the rest of the 8 sessions queue, which keeps decode steps short. + max-running-requests: 6 + cuda-graph-max-bs-decode: 6 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 env: SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' # 170-185 GB of pinned host pools per GPU: the 8 ranks share the node's 1511 GiB GTT. SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' # Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one. RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '8' @@ -143,17 +166,58 @@ override_tp8_dcp8_c16: enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 2.5 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 + # 8 running requests: the rest of the 16 sessions queue, which keeps decode steps short. + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 + # The current token plus 4 drafts. + speculative-num-draft-tokens: 5 env: SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '16' KV_OFFLOADING: dram +override_tp8_dcp8_c24: + roles: + agg: + args: + dcp-size: 8 + dcp-comm-backend: a2a + mamba-full-memory-ratio: 0.65 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 2.5 + # 13 running requests: the rest of the 24 sessions queue, which keeps decode steps short. + max-running-requests: 13 + cuda-graph-max-bs-decode: 13 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 + # The current token plus 4 drafts. + speculative-num-draft-tokens: 5 + env: + SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' + SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' + RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + override_tp8_dcp8_c32: roles: agg: @@ -164,12 +228,22 @@ override_tp8_dcp8_c32: enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 2.5 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 + # 20 running requests: the rest of the 32 sessions queue, which keeps decode steps short. + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 + # The current token plus 4 drafts. + speculative-num-draft-tokens: 5 env: SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '32' @@ -188,7 +262,8 @@ override_tp8_dcp8_c48: max-running-requests: 48 cuda-graph-max-bs-decode: 48 chunked-prefill-size: 32768 - prefill-decode-interval: 8 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 # The current token plus 4 drafts: from 48 requests the shorter verify outweighs its lower acceptance. speculative-num-draft-tokens: 5 env: @@ -218,7 +293,8 @@ override_tp8_dcp8_c56: max-running-requests: 56 cuda-graph-max-bs-decode: 56 chunked-prefill-size: 32768 - prefill-decode-interval: 8 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 # The current token plus 4 drafts: from 48 requests the shorter verify outweighs its lower acceptance. speculative-num-draft-tokens: 5 env: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 737cf6ebe0..31a35450d2 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1115,7 +1115,7 @@ musespark-fp4-mi355x-sglang-agentic-dflash: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [28, 36, 40], srt-recipe: agentic.yaml } kimik3.1-fp4-mi355x-sglang-agentic-dflash: - image: lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:346a6d832bbfcb5776d8d938154870682a891278a8adc264ed996ccf3d3b6762 + image: lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db1401ad7b8bc62ac67175f6f013b1b868e2a2dab063ee8848e7290fa9a7 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:mi355x-amds @@ -1128,7 +1128,38 @@ kimik3.1-fp4-mi355x-sglang-agentic-dflash: - dram-utilization: 0.513 search-space: - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 16, 32, 48, 56], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 16, 24, 32, 48, 56], srt-recipe: agentic.yaml } + +kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg: + image: lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db1401ad7b8bc62ac67175f6f013b1b868e2a2dab063ee8848e7290fa9a7 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3.1/sglang/mi355x-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.513 + search-space: + - spec-decoding: draft_model + conc-list: [128] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c128 + prefill: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [144] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c144 + prefill: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } glm5.3flash-fp8-mi325x-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.22-rocm10-mi30x@sha256:5658cd0d8de0c68de10d2e349296f55ceff98053182fa87b2a7e942e5e7f3c2a diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index fd14da3302..ea5d3a5f15 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -789,6 +789,7 @@ clusters: MORI_RDMA_TC: '@fabric.rdma-tc' models: entries: + Kimi-K3.1-Pro: {root: hf-hub-cache, dir: models--moonshotai--Kimi-K3.1-Pro/snapshots/d9e99b0792a5e93b78ea9bea62a68167e36ddd9e} DeepSeek-R1-0528-MXFP4-v2: {root: it-share-data, dir: DeepSeek-R1-0528-MXFP4-v2} GLM-5.3: {root: it-share-data, dir: GLM-5.3} Qwen3.5-397B-A17B-FP8: {root: it-share-data, dir: Qwen3.5-397B-A17B-FP8} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 492866dae2..debff40a78 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12720,3 +12720,16 @@ - "Update the TensorRT-LLM image from 1.3.0rc28.dev202609210001 to 1.4.0rc2." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4626 scenario-type: [agentic-coding] + +- config-keys: + - kimik3.1-fp4-mi355x-sglang-agentic-dflash + - kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg + scenario-type: + - agentic-coding + description: + - "Image lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14 (was sha256:346a6d83)." + - "Single-node: prefill every 32 decode steps at concurrency 8 to 56 (was 16, and 8 at 48 and 56)." + - "Single-node: cap running requests at 1, 6, 8 and 20 at concurrency 1, 8, 16 and 32 (was 4, 8, 16 and 32); verify 8 tokens at concurrency 1 and 4 and 5 tokens at 16 and 32 (was 6)." + - "Single-node: row-sharded prefill MoE and residual stream and fused DFLASH ring locations at every concurrency (was 48 and 56 only); add concurrency 24 with 13 running requests and 5-token verify." + - "Add 1P1D PD at concurrency 128 and 144: TP8 DCP8 prefill and decode, KV over Mooncake RDMA, prefill HiCache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928