From 80811c6e1758328f5ad8b25b5c7b7b3ef22c9d0c Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Sun, 8 Nov 2026 19:50:03 +0000 Subject: [PATCH 1/9] [AMD][MI35X] Kimi K3.1 Pro sglang dflash agentic: tune every concurrency for P90 interactivity Recipe only, same image. A prefill every 32 decode steps at concurrency 8 to 56, fewer running requests and 4 drafts in the middle of the range, a new concurrency 24 point, and 7 drafts at concurrency 1 and 4. Co-Authored-By: Claude Opus 5.5 --- .../sglang/mi355x-fp4-mtp/agentic.yaml | 91 +++++++++++++++++-- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 13 +++ 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml index 42286764a2..4579181140 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml @@ -85,9 +85,16 @@ override_tp8_c1: cuda-graph-max-bs-decode: 4 # 150 state slots keep agent sessions' prefixes cached. max-mamba-cache-size: 150 + # The current token plus 7 drafts. + speculative-num-draft-tokens: 8 env: SGLANG_AMD_MONO_K31_WIDE: '1' SGLANG_ROCM_RUNTIME_RESERVE_GB: '6' + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '1' @@ -102,9 +109,16 @@ override_tp8_c4: cuda-graph-max-bs-decode: 4 # 150 state slots keep agent sessions' prefixes cached. max-mamba-cache-size: 150 + # The current token plus 7 drafts. + speculative-num-draft-tokens: 8 env: SGLANG_AMD_MONO_K31_WIDE: '1' SGLANG_ROCM_RUNTIME_RESERVE_GB: '6' + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '4' @@ -120,14 +134,22 @@ override_tp8_dcp8_c8: enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 2.5 - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 + # 6 running requests: the rest of the 8 sessions queue, which keeps decode steps short. + max-running-requests: 6 + cuda-graph-max-bs-decode: 6 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 env: SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' # 170-185 GB of pinned host pools per GPU: the 8 ranks share the node's 1511 GiB GTT. SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' # Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one. RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '8' @@ -143,17 +165,58 @@ override_tp8_dcp8_c16: enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 2.5 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 + # 8 running requests: the rest of the 16 sessions queue, which keeps decode steps short. + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 + # The current token plus 4 drafts. + speculative-num-draft-tokens: 5 env: SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '16' KV_OFFLOADING: dram +override_tp8_dcp8_c24: + roles: + agg: + args: + dcp-size: 8 + dcp-comm-backend: a2a + mamba-full-memory-ratio: 0.65 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 2.5 + # 13 running requests: the rest of the 24 sessions queue, which keeps decode steps short. + max-running-requests: 13 + cuda-graph-max-bs-decode: 13 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 + # The current token plus 4 drafts. + speculative-num-draft-tokens: 5 + env: + SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' + SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' + RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + override_tp8_dcp8_c32: roles: agg: @@ -164,12 +227,22 @@ override_tp8_dcp8_c32: enable-hierarchical-cache: true hicache-write-policy: write_back hicache-ratio: 2.5 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 + # 20 running requests: the rest of the 32 sessions queue, which keeps decode steps short. + max-running-requests: 20 + cuda-graph-max-bs-decode: 20 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 + # The current token plus 4 drafts. + speculative-num-draft-tokens: 5 env: SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' SGLANG_HICACHE_MAMBA_HOST_RATIO: '10' RUSTUP_TOOLCHAIN: stable + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' benchmark: env: CONC: '32' @@ -188,7 +261,8 @@ override_tp8_dcp8_c48: max-running-requests: 48 cuda-graph-max-bs-decode: 48 chunked-prefill-size: 32768 - prefill-decode-interval: 8 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 # The current token plus 4 drafts: from 48 requests the shorter verify outweighs its lower acceptance. speculative-num-draft-tokens: 5 env: @@ -218,7 +292,8 @@ override_tp8_dcp8_c56: max-running-requests: 56 cuda-graph-max-bs-decode: 56 chunked-prefill-size: 32768 - prefill-decode-interval: 8 + # Prefill every 32 decode steps: fewer stalls in running requests' token streams. + prefill-decode-interval: 32 # The current token plus 4 drafts: from 48 requests the shorter verify outweighs its lower acceptance. speculative-num-draft-tokens: 5 env: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 08e27bff23..5e894f2c3a 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1121,7 +1121,7 @@ kimik3.1-fp4-mi355x-sglang-agentic-dflash: - dram-utilization: 0.513 search-space: - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4], srt-recipe: agentic.yaml } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 16, 32, 48, 56], srt-recipe: agentic.yaml } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 16, 24, 32, 48, 56], srt-recipe: agentic.yaml } glm5.3flash-fp8-mi325x-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.22-rocm10-mi30x@sha256:5658cd0d8de0c68de10d2e349296f55ceff98053182fa87b2a7e942e5e7f3c2a diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 49c3625b4f..9d5287f7ea 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12471,3 +12471,16 @@ description: - "The H3 client polls job status every 0.1 s instead of every 0.5 s. A job counts as done only at the next poll, so each published latency carried up to 0.5 s of polling delay; it now carries up to 0.1 s." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4904 + +- config-keys: + - kimik3.1-fp4-mi355x-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Recipe only, same image (lmsysorg/sglang:dev-kimi-k31-pro-mi35x sha256:346a6d83)." + - "Concurrency 8 to 56: a prefill every 32 decode steps instead of 16 (8 at concurrency 48 and 56), so prefills interrupt running requests' token streams half as often." + - "Concurrency 8, 16 and 32: 6, 8 and 20 running requests; concurrency 16 and 32 verify the current token plus 4 drafts." + - "Add concurrency 24: 13 running requests, current token plus 4 drafts." + - "Concurrency 1 and 4: verify the current token plus 7 drafts." + - "All concurrencies: MoE front and residual stream on row shards in prefill and fused DFLASH ring locations (already on at 48 and 56)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From a50a174869eed91dc5ca12d1431efc79bc882b40 Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Sun, 8 Nov 2026 19:57:55 +0000 Subject: [PATCH 2/9] Link the perf-changelog entry to #4928 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9d5287f7ea..561ae1e804 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12483,4 +12483,4 @@ - "Add concurrency 24: 13 running requests, current token plus 4 drafts." - "Concurrency 1 and 4: verify the current token plus 7 drafts." - "All concurrencies: MoE front and residual stream on row shards in prefill and fused DFLASH ring locations (already on at 48 and 56)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928 From c4e644e1721011d5b4f6c6c4acefb2ac24074b1e Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Mon, 9 Nov 2026 09:27:46 +0000 Subject: [PATCH 3/9] [AMD][MI35X] Kimi K3.1 Pro sglang dflash agentic: one image for every point, one running request at concurrency 1 The image moves to lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14: SGLang de0f491836 (the #48427 branch with sgl-project/sglang#49152 and the PD commits) on AITER ROCm/aiter#7620 (flydsl 0.4.2) and sgl-amd-mono 219f40be9a, the image the MI355X PD points use too. Concurrency 1 runs one request at a time, so an agent's parallel tool calls queue instead of sharing decode steps. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml | 3 ++- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml index 4579181140..644242f33e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4-mtp/agentic.yaml @@ -81,7 +81,8 @@ override_tp8_c1: agg: args: enable-amd-monokernel: true - max-running-requests: 4 + # One running request: agent fan-out queues instead of sharing decode steps. + max-running-requests: 1 cuda-graph-max-bs-decode: 4 # 150 state slots keep agent sessions' prefixes cached. max-mamba-cache-size: 150 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 5bd12d56f2..bb966aac35 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1121,7 +1121,7 @@ musespark-fp4-mi355x-sglang-agentic-dflash: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [28, 36, 40], srt-recipe: agentic.yaml } kimik3.1-fp4-mi355x-sglang-agentic-dflash: - image: lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:346a6d832bbfcb5776d8d938154870682a891278a8adc264ed996ccf3d3b6762 + image: lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db1401ad7b8bc62ac67175f6f013b1b868e2a2dab063ee8848e7290fa9a7 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fee0382d07..c99ef2d788 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12568,7 +12568,8 @@ scenario-type: - agentic-coding description: - - "Recipe only, same image (lmsysorg/sglang:dev-kimi-k31-pro-mi35x sha256:346a6d83)." + - "Image lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14 (was sha256:346a6d83): SGLang de0f491836, the #48427 branch with the SiTU gate fix (sgl-project/sglang#49152) and the PD commits, on AITER ROCm/aiter#7620 (5405ee0026, flydsl 0.4.2) and sgl-amd-mono 219f40be9a." + - "Concurrency 1: one running request instead of 4, so an agent's parallel tool calls queue instead of sharing decode steps." - "Concurrency 8 to 56: a prefill every 32 decode steps instead of 16 (8 at concurrency 48 and 56), so prefills interrupt running requests' token streams half as often." - "Concurrency 8, 16 and 32: 6, 8 and 20 running requests; concurrency 16 and 32 verify the current token plus 4 drafts." - "Add concurrency 24: 13 running requests, current token plus 4 drafts." From 7b64e5ff5e146107d15b105028e435c01a8ccde3 Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Mon, 9 Nov 2026 09:32:01 +0000 Subject: [PATCH 4/9] [AMD][MI35X] Kimi K3.1 Pro sglang dflash agentic: add the 1P1D PD recipe at concurrency 128 and 144 kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg on the same image: one TP8 DCP8 prefill node and one TP8 DCP8 decode node, KV and the MXFP4 DSA index keys over Mooncake RDMA, five-token DFLASH on both roles. Concurrency 144 (decode cap 160) is the highest point that beat the one below it by more than 1% over 3600 s; concurrency 128 (cap 144) is the point below it. Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 184 ++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 31 +++ inferencex-e2e/configs/runners.yaml | 1 + inferencex-e2e/perf-changelog.yaml | 10 + 4 files changed, 226 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..6bd61b97a5 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,184 @@ +schema: 2 + +base: + name: kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg + slurm: + time_limit: '08:00:00' + sbatch_directives: + mem: '0' + srun_options: + container-remap-root: '' + container-writable: '' + mem: '0' + container_mounts: + /dev/infiniband: /dev/infiniband + extra_mount: + # Match the host's Ionic provider without replacing the image's verbs core. + - /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + placement: + node: head + args: + request-timeout-secs: 3600 + observability: + enabled: false + tachometer: + enabled: false + # Loading the 1.56 TB checkpoint plus decode and prefill graph capture. + health_check: + max_attempts: 720 + interval_seconds: 10 + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: a2a + kv-cache-dtype: fp8_e4m3 + enable-dsa-fp4-indexer: true + mem-fraction-static: 0.85 + # 32k chunks put the long-prefill sparse MLA on a Triton path that hangs below ~34 GB of free VRAM. + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-radix-cache-strategy: extra_buffer + mamba-full-memory-ratio: 0.65 + mamba-max-states-per-path: 1 + # Checkpoints a KDA state every 64 tokens, so a resumed turn misses at most 63 cached tokens. + mamba-track-interval: 64 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 4.0 + cuda-graph-backend-prefill: breakable + cuda-graph-bs-prefill: [256, 512, 768, 1024, 1536, 2048, 3072, 4096] + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: '@fabric.rdma-devices' + disaggregation-decode-allocation-policy: early + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + reasoning-parser: kimi_k3 + tool-call-parser: kimi_k3 + enable-metrics: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: triton + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # The current token plus 4 drafts; the prefill writes the draft KV ring the decode reads. + speculative-num-draft-tokens: 5 + random-seed: 42 + env: &common_env + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + NCCL_SOCKET_IFNAME: '@fabric.socket-ifname' + GLOO_SOCKET_IFNAME: '@fabric.socket-ifname' + MC_GID_INDEX: '@fabric.rdma-gid-index' + MC_MAX_CONCURRENT_REG_MR: '8' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_USE_AITER: '1' + SGLANG_AITER_K3_OPT: '1' + AITER_FLYDSL_FORCE: '1' + AITER_SITUV2_A8W4: '1' + # BF16 MoE combine and all-reduce. + AITER_OPUS_STAGE2_FP8: '0' + ROCM_QUICK_REDUCE_QUANTIZATION: NONE + PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.95 + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + # Split-K HIP sparse MLA decode and verify for 16 to 512 rows. + SGLANG_ROCM_SPARSE_MLA_DECODE_HIP: '1' + SGLANG_ROCM_RUNTIME_RESERVE_GB: '10' + # MoE front and residual stream on row shards in prefill (no extra memory). + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_ROUTED_FRONT: '1' + SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' + # A draft block's ring locations in one launch. + SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' + # Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one. + RUSTUP_TOOLCHAIN: stable + # Pinned host pools; registered pageable pools stall every GPU queue on page migration. + HSA_USERPTR_FOR_PAGED_MEM: '0' + # The MI355X nodes report MEC firmware 38, below the 177 that fixes the + # RCCL memory reclaim issue. + HSA_NO_SCRATCH_RECLAIM: '1' + decode: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: a2a + kv-cache-dtype: fp8_e4m3 + enable-dsa-fp4-indexer: true + mem-fraction-static: 0.92 + mamba-radix-cache-strategy: extra_buffer + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: '@fabric.rdma-devices' + disaggregation-decode-extra-slots: 16 + disaggregation-decode-enable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + reasoning-parser: kimi_k3 + tool-call-parser: kimi_k3 + enable-metrics: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: triton + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 5 + # Without ReplaySSM, DFLASH keeps one KDA state per draft token and the KV pool collapses. + enable-linear-replayssm-spec: true + random-seed: 42 + env: + <<: *common_env + # Decode graphs past ~96 at mem 0.92 abort when the allocator collects during capture. + PYTORCH_HIP_ALLOC_CONF: garbage_collection_threshold:0.99 + benchmark: + placement: + node: head + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_1p1d_c128: + roles: + prefill: &prefill_1p1d + env: + # 190 GB of pinned host pools per GPU (KV, KDA states, indexer keys) under the node's 1511 GiB GTT. + SGLANG_HICACHE_MAMBA_HOST_RATIO: '5' + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: '0.1' + decode: + args: + max-running-requests: 144 + cuda-graph-max-bs-decode: 144 + +override_1p1d_c144: + roles: + prefill: *prefill_1p1d + decode: + args: + max-running-requests: 160 + cuda-graph-max-bs-decode: 160 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index bb966aac35..8eefdf1556 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1136,6 +1136,37 @@ kimik3.1-fp4-mi355x-sglang-agentic-dflash: - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4], srt-recipe: agentic.yaml } - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [8, 16, 24, 32, 48, 56], srt-recipe: agentic.yaml } +kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg: + image: lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db1401ad7b8bc62ac67175f6f013b1b868e2a2dab063ee8848e7290fa9a7 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3.1/sglang/mi355x-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.513 + search-space: + - spec-decoding: draft_model + conc-list: [128] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c128 + prefill: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [144] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_c144 + prefill: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 8, ep: 1, dp-attn: false, dcp-size: 8 } + glm5.3flash-fp8-mi325x-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.22-rocm10-mi30x@sha256:5658cd0d8de0c68de10d2e349296f55ceff98053182fa87b2a7e942e5e7f3c2a model: zai-org/GLM-5.3-Flash diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index fd14da3302..f31a49e3d5 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -789,6 +789,7 @@ clusters: MORI_RDMA_TC: '@fabric.rdma-tc' models: entries: + Kimi-K3.1-Pro: {root: it-share-data, dir: Kimi-K3.1-Pro} DeepSeek-R1-0528-MXFP4-v2: {root: it-share-data, dir: DeepSeek-R1-0528-MXFP4-v2} GLM-5.3: {root: it-share-data, dir: GLM-5.3} Qwen3.5-397B-A17B-FP8: {root: it-share-data, dir: Qwen3.5-397B-A17B-FP8} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c99ef2d788..0940b45a64 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12576,3 +12576,13 @@ - "Concurrency 1 and 4: verify the current token plus 7 drafts." - "All concurrencies: MoE front and residual stream on row shards in prefill and fused DFLASH ring locations (already on at 48 and 56)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928 + +- config-keys: + - kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3.1-Pro MI355X SGLang PD AgentX on lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14: 1P1D with TP8 DCP8 on both roles, KV over Mooncake RDMA, five-token DFLASH (DFlash2) on both roles, and MXFP4 DSA index keys sent with the KV." + - "Prefill: 16k chunks, shortest-prefill-first, a KDA checkpoint every 64 tokens, and a write-back HiCache host tier (ratio 4.0, KDA host ratio 5) under the node's pinned-memory limit. Decode: up to 144 (concurrency 128) or 160 (concurrency 144) running requests with matching decode graphs and a decode radix cache." + - "Measured on two MI355X nodes over 3600 s with golden acceptance 3.25, with these settings and SGLang commits on the previous image's AITER (285579b): concurrency 144 at 21,435 tok/s/GPU (p90 interactivity 31.7, TTFT p50 3.75 s) and concurrency 128 at 21,124 (32.8, 3.67 s); GSM8K 97.88% at native acceptance on the same configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928 From 697d4c77c3f1f257d482f4346a69dba8c76d812e Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Mon, 9 Nov 2026 12:41:24 +0000 Subject: [PATCH 5/9] [AMD][MI35X] Kimi K3.1 Pro sglang dflash agentic: serve PD from the node-local hub snapshot The mi355x-amds Kimi-K3.1-Pro entry pointed at /it-share/data/Kimi-K3.1-Pro, which does not exist, so the 1P1D job failed before launch. Point it at the node-local HF cache snapshot the single-node points already serve. Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/configs/runners.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index f31a49e3d5..ea5d3a5f15 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -789,7 +789,7 @@ clusters: MORI_RDMA_TC: '@fabric.rdma-tc' models: entries: - Kimi-K3.1-Pro: {root: it-share-data, dir: Kimi-K3.1-Pro} + Kimi-K3.1-Pro: {root: hf-hub-cache, dir: models--moonshotai--Kimi-K3.1-Pro/snapshots/d9e99b0792a5e93b78ea9bea62a68167e36ddd9e} DeepSeek-R1-0528-MXFP4-v2: {root: it-share-data, dir: DeepSeek-R1-0528-MXFP4-v2} GLM-5.3: {root: it-share-data, dir: GLM-5.3} Qwen3.5-397B-A17B-FP8: {root: it-share-data, dir: Qwen3.5-397B-A17B-FP8} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 0940b45a64..81d86ae6f9 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12568,7 +12568,7 @@ scenario-type: - agentic-coding description: - - "Image lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14 (was sha256:346a6d83): SGLang de0f491836, the #48427 branch with the SiTU gate fix (sgl-project/sglang#49152) and the PD commits, on AITER ROCm/aiter#7620 (5405ee0026, flydsl 0.4.2) and sgl-amd-mono 219f40be9a." + - "Image lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14 (was sha256:346a6d83): SGLang de0f491836 with the SiTU gate fix and the PD commits, on AITER 5405ee0026 (flydsl 0.4.2) and sgl-amd-mono 219f40be9a." - "Concurrency 1: one running request instead of 4, so an agent's parallel tool calls queue instead of sharing decode steps." - "Concurrency 8 to 56: a prefill every 32 decode steps instead of 16 (8 at concurrency 48 and 56), so prefills interrupt running requests' token streams half as often." - "Concurrency 8, 16 and 32: 6, 8 and 20 running requests; concurrency 16 and 32 verify the current token plus 4 drafts." From 7768eee2d9e6e80b92a5576bbd88e5f976311002 Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Mon, 9 Nov 2026 15:42:15 +0000 Subject: [PATCH 6/9] [AMD][MI35X] Kimi K3.1 Pro sglang dflash agentic: keep PD's router connections and decode ranks in step The 1P1D c144 warmup reused router-to-prefill connections that had sat idle past the 900 s keep-alive, and the c128 GSM8K run hung after one decode rank dropped a request the other seven kept. Raise the keep-alive to 7200 s and have rank 0 broadcast the greedy DFLASH accept on both roles. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml | 6 ++++-- inferencex-e2e/perf-changelog.yaml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml index 6bd61b97a5..98f4aa3508 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -92,8 +92,8 @@ base: MC_MAX_CONCURRENT_REG_MR: '8' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' - # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. - SGLANG_TIMEOUT_KEEP_ALIVE: '900' + # Outlasts the router's pooled connections, idle through the warmup's long tail. + SGLANG_TIMEOUT_KEEP_ALIVE: '7200' SGLANG_USE_AITER: '1' SGLANG_AITER_K3_OPT: '1' AITER_FLYDSL_FORCE: '1' @@ -111,6 +111,8 @@ base: SGLANG_ENABLE_ROCM_K3_ROW_SHARDED_STREAM: '1' # A draft block's ring locations in one launch. SGLANG_OPT_DFLASH_RING_LOCS_FUSED: '1' + # Every rank commits rank 0's greedy accept; a rank that diverges hangs the batch. + SGLANG_SPEC_TP_SYNC: init,dflash-accept-greedy # Keeps every rank on the Rust TreeCore; write-back asserts on a mix with the Python one. RUSTUP_TOOLCHAIN: stable # Pinned host pools; registered pageable pools stall every GPU queue on page migration. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 81d86ae6f9..4455cb3812 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12584,5 +12584,6 @@ description: - "Add Kimi-K3.1-Pro MI355X SGLang PD AgentX on lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14: 1P1D with TP8 DCP8 on both roles, KV over Mooncake RDMA, five-token DFLASH (DFlash2) on both roles, and MXFP4 DSA index keys sent with the KV." - "Prefill: 16k chunks, shortest-prefill-first, a KDA checkpoint every 64 tokens, and a write-back HiCache host tier (ratio 4.0, KDA host ratio 5) under the node's pinned-memory limit. Decode: up to 144 (concurrency 128) or 160 (concurrency 144) running requests with matching decode graphs and a decode radix cache." + - "Both roles: a 7200 s HTTP keep-alive, so the router's idle pooled connections stay open, and rank 0's greedy DFLASH accept broadcast to every rank." - "Measured on two MI355X nodes over 3600 s with golden acceptance 3.25, with these settings and SGLang commits on the previous image's AITER (285579b): concurrency 144 at 21,435 tok/s/GPU (p90 interactivity 31.7, TTFT p50 3.75 s) and concurrency 128 at 21,124 (32.8, 3.67 s); GSM8K 97.88% at native acceptance on the same configuration." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928 From d96c510d0f1120180abb98704415254abbd092fd Mon Sep 17 00:00:00 2001 From: Kevin Mi Date: Mon, 9 Nov 2026 20:51:40 +0000 Subject: [PATCH 7/9] [AMD][MI35X] Kimi K3.1 Pro sglang dflash agentic: mount the host's Ionic driver entry so PD moves KV over RDMA The image has no Ionic verbs provider, and libibverbs loads a provider only when /etc/libibverbs.d names it. With only libionic mounted, Mooncake found 0 HCAs on every rank and moved KV over TCP: the concurrency 128 PD point of run 42647381696 read 16,321 tok/s/GPU with a TTFT p50 of 18.2 s, while the prefill held 47-48 requests waiting on transfers and the decode 60-84. Co-Authored-By: Claude Opus 5.5 --- .../kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml index 98f4aa3508..f7496f6cf8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -13,8 +13,9 @@ base: container_mounts: /dev/infiniband: /dev/infiniband extra_mount: - # Match the host's Ionic provider without replacing the image's verbs core. + # The image has no Ionic verbs provider; libibverbs loads the host's only through its driver entry. - /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro + - /etc/libibverbs.d/ionic.driver:/etc/libibverbs.d/ionic.driver:ro resources: gpu_type: mi355x gpus_per_node: 8 From 67eed3b805367b6f744f49dab42b855ed3992e8a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 10 Nov 2026 11:41:22 -0600 Subject: [PATCH 8/9] chore(changelog): merge Kimi K3.1 MI355X entries into one block --- inferencex-e2e/perf-changelog.yaml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 92748ddb63..ecfb956570 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12609,25 +12609,13 @@ - config-keys: - kimik3.1-fp4-mi355x-sglang-agentic-dflash - scenario-type: - - agentic-coding - description: - - "Image lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14 (was sha256:346a6d83): SGLang de0f491836 with the SiTU gate fix and the PD commits, on AITER 5405ee0026 (flydsl 0.4.2) and sgl-amd-mono 219f40be9a." - - "Concurrency 1: one running request instead of 4, so an agent's parallel tool calls queue instead of sharing decode steps." - - "Concurrency 8 to 56: a prefill every 32 decode steps instead of 16 (8 at concurrency 48 and 56), so prefills interrupt running requests' token streams half as often." - - "Concurrency 8, 16 and 32: 6, 8 and 20 running requests; concurrency 16 and 32 verify the current token plus 4 drafts." - - "Add concurrency 24: 13 running requests, current token plus 4 drafts." - - "Concurrency 1 and 4: verify the current token plus 7 drafts." - - "All concurrencies: MoE front and residual stream on row shards in prefill and fused DFLASH ring locations (already on at 48 and 56)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928 - -- config-keys: - kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg scenario-type: - agentic-coding description: - - "Add Kimi-K3.1-Pro MI355X SGLang PD AgentX on lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14: 1P1D with TP8 DCP8 on both roles, KV over Mooncake RDMA, five-token DFLASH (DFlash2) on both roles, and MXFP4 DSA index keys sent with the KV." - - "Prefill: 16k chunks, shortest-prefill-first, a KDA checkpoint every 64 tokens, and a write-back HiCache host tier (ratio 4.0, KDA host ratio 5) under the node's pinned-memory limit. Decode: up to 144 (concurrency 128) or 160 (concurrency 144) running requests with matching decode graphs and a decode radix cache." - - "Both roles: a 7200 s HTTP keep-alive, so the router's idle pooled connections stay open, and rank 0's greedy DFLASH accept broadcast to every rank." - - "Measured on two MI355X nodes over 3600 s with golden acceptance 3.25, with these settings and SGLang commits on the previous image's AITER (285579b): concurrency 144 at 21,435 tok/s/GPU (p90 interactivity 31.7, TTFT p50 3.75 s) and concurrency 128 at 21,124 (32.8, 3.67 s); GSM8K 97.88% at native acceptance on the same configuration." + - "Image lmsysorg/sglang:dev-kimi-k31-pro-mi35x@sha256:5901db14 (was sha256:346a6d83)." + - "Single-node: prefill every 32 decode steps at concurrency 8 to 56 (was 16, and 8 at 48 and 56)." + - "Single-node: cap running requests at 1, 6, 8 and 20 at concurrency 1, 8, 16 and 32 (was 4, 8, 16 and 32); verify 8 tokens at concurrency 1 and 4 and 5 tokens at 16 and 32 (was 6)." + - "Single-node: row-sharded prefill MoE and residual stream and fused DFLASH ring locations at every concurrency (was 48 and 56 only); add concurrency 24 with 13 running requests and 5-token verify." + - "Add 1P1D PD at concurrency 128 and 144: TP8 DCP8 prefill and decode, KV over Mooncake RDMA, prefill HiCache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4928 From 020a6a07d544e7e976e993af0d9cb57b86d09c8e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 10 Nov 2026 11:51:02 -0600 Subject: [PATCH 9/9] fix(configs): publish Kimi K3.1 MI355X PD on the sglang curve --- inferencex-e2e/configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 8eefdf1556..8112441a2f 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1142,7 +1142,7 @@ kimik3.1-fp4-mi355x-sglang-agentic-dflash-disagg: model-prefix: kimik3.1 runner: cluster:mi355x-amds precision: fp4 - framework: sglang-disagg + framework: sglang router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mooncake multinode: true