From a2f3b1026e342e6403adc04171f4008ef8b6df85 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 15:05:54 -0600 Subject: [PATCH 1/5] feat(kimik3.1): add GB300 Dynamo SGLang DFLASH AgentX configs Aggregated TP8 DCP8 at c1, TP8 DCP8 1P3D/1P2D/2P3D/3P4D disaggregation with prefill HiCache, and 3P1D with a TP16 DCP8 attention-DP2 EP16 decode, all on the RadixArk DFLASH draft with block verification. --- .../agentx/agg-tp8-dcp8-dflash7.yaml | 115 ++++++++++ .../disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml | 187 +++++++++++++++ .../gb300-fp4/agentx/disagg-variants.yaml | 214 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 146 ++++++++++++ inferencex-e2e/perf-changelog.yaml | 11 + 5 files changed, 673 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml new file mode 100644 index 0000000000..5d902469c2 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-agg-tp8-dcp8-dflash7 +identity: + model: {} + container: {} +resources: + gpu_type: gb300 + gpus_per_node: 4 +# The K3.1 SGLang image ships no Dynamo. +dynamo: + install: true + source: + wheel: 1.6.0.dev20261016 +environment: + HF_HUB_CACHE: /hf_hub_cache +observability: + tachometer: + enabled: false +slurm: + time_limit: "4:00:00" +health_check: + max_attempts: 1440 + interval_seconds: 10 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + - name: otel-collector + type: otel-collector + enabled: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + # The Kimi tokenizer is remote code, so chat templating stays in SGLang. + dyn-chat-processor: sglang + dyn-preprocess-workers: 4 + trust-remote-code: true + router-mode: kv + router-session-affinity-ttl-secs: "14400" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + dcp-size: 8 + dcp-comm-backend: fi_a2a + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: float32 + page-size: 64 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + chunked-prefill-size: 16384 + mem-fraction-static: 0.88 + max-running-requests: 1 + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + # DFLASH counts the verification width: 7 drafts plus the current token. + speculative-num-draft-tokens: 8 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + enable-linear-replayssm-spec: true + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: true +sbatch_directives: + mem: "0" +srun_options: + mem: "0" + container-remap-root: "" +benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml new file mode 100644 index 0000000000..524f88a42f --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml @@ -0,0 +1,187 @@ +schema: 2 +name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-3p1d-p-tp8-d-tp16-dpa2-ep16 +identity: + model: {} + container: {} +resources: + gpu_type: gb300 + gpus_per_node: 4 +# The K3.1 SGLang image ships no Dynamo. +dynamo: + install: true + source: + wheel: 1.6.0.dev20261016 +environment: + HF_HUB_CACHE: /hf_hub_cache +observability: + tachometer: + enabled: false +slurm: + time_limit: "6:00:00" +health_check: + max_attempts: 1440 + interval_seconds: 10 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + - name: otel-collector + type: otel-collector + enabled: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + # The Kimi tokenizer is remote code, so chat templating stays in SGLang. + dyn-chat-processor: sglang + dyn-preprocess-workers: 4 + trust-remote-code: true + router-mode: kv + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" +engine: sglang +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 2 + dcp-size: 8 + dcp-comm-backend: fi_a2a + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: float32 + page-size: 64 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mem-fraction-static: 0.88 + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + # 120 GB per rank is 480 GB of each 4-GPU tray's host DRAM (dram-utilization 0.5). + enable-hierarchical-cache: true + hicache-size: 120 + hicache-write-policy: write_back + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + # DFLASH counts the verification width: 4 drafts plus the current token, as decode. + speculative-num-draft-tokens: 5 + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + router-mode: kv + router-session-affinity-ttl-secs: "14400" + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + PYTHONUNBUFFERED: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1536" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 16 + attn-dp-size: 2 + enable-dp-lm-head: true + expert-parallel-size: 16 + # MegaMoE deadlocks with --enable-symm-mem, so symmetric memory stays off. + moe-a2a-backend: megamoe + dcp-size: 8 + dcp-comm-backend: fi_a2a + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: float32 + page-size: 64 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + mem-fraction-static: 0.85 + max-running-requests: 384 + cuda-graph-max-bs-decode: 192 + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-num-draft-tokens: 5 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + enable-linear-replayssm-spec: true + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: false +sbatch_directives: + mem: "0" +srun_options: + mem: "0" + container-remap-root: "" +benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..0efb075006 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,214 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg + identity: + model: {} + container: {} + resources: + gpu_type: gb300 + gpus_per_node: 4 + # The K3.1 SGLang image ships no Dynamo. + dynamo: + install: true + source: + wheel: 1.6.0.dev20261016 + environment: + HF_HUB_CACHE: /hf_hub_cache + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + # The Kimi tokenizer is remote code, so chat templating stays in SGLang. + dyn-chat-processor: sglang + dyn-preprocess-workers: 4 + trust-remote-code: true + router-mode: kv + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 2 + dcp-size: 8 + dcp-comm-backend: fi_a2a + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: float32 + page-size: 64 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mem-fraction-static: 0.88 + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + # 120 GB per rank is 480 GB of each 4-GPU tray's host DRAM (dram-utilization 0.5). + enable-hierarchical-cache: true + hicache-size: 120 + hicache-write-policy: write_back + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + # DFLASH counts the verification width: 6 drafts plus the current token, as decode. + speculative-num-draft-tokens: 7 + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + router-mode: kv + router-session-affinity-ttl-secs: "14400" + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: true + decode: + nodes: 4 + workers: 2 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 2 + dcp-size: 8 + dcp-comm-backend: fi_a2a + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: float32 + page-size: 64 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + mem-fraction-static: 0.88 + max-running-requests: 64 + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-num-draft-tokens: 7 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + enable-linear-replayssm-spec: true + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: false + sbatch_directives: + mem: "0" + srun_options: + mem: "0" + container-remap-root: "" + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' + +override_1p3d_c16: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-1p3d-c16 + roles: + decode: + nodes: 6 + workers: 3 + +override_1p2d_c24_c64: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-1p2d-c24-c64 + +override_2p3d_c240: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-2p3d-c240 + roles: + prefill: + nodes: 4 + workers: 2 + decode: + nodes: 6 + workers: 3 + args: + max-running-requests: 96 + +override_3p4d_c240: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-3p4d-c240 + roles: + prefill: + nodes: 6 + workers: 3 + decode: + nodes: 8 + workers: 4 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1ca0feca1a..26ff2974f3 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6530,6 +6530,152 @@ musespark-fp4-gb300-dynamo-sglang-agentic-dflash: ep: 4 dp-attn: false +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0.dev20261016" } + multinode: true + disagg: false + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: draft_model + kv-offloading: none + conc-list: [1] + num-nodes: 2 + srt-recipe: agg-tp8-dcp8-dflash7.yaml + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0.dev20261016" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + # Prefill HiCache: 120 GB per rank, 480 GB of each 4-GPU tray. + - dram-utilization: 0.5 + search-space: + - spec-decoding: draft_model + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [16] + srt-recipe: disagg-variants.yaml:override_1p3d_c16 + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + decode: + num-worker: 3 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [24, 32, 48, 64] + srt-recipe: disagg-variants.yaml:override_1p2d_c24_c64 + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + decode: + num-worker: 2 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [240] + srt-recipe: disagg-variants.yaml:override_2p3d_c240 + prefill: + num-worker: 2 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + decode: + num-worker: 3 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + - spec-decoding: draft_model + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [240] + srt-recipe: disagg-variants.yaml:override_3p4d_c240 + prefill: + num-worker: 3 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + decode: + num-worker: 4 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-wideep-disagg: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0.dev20261016" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.5 + search-space: + - spec-decoding: draft_model + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [336] + srt-recipe: disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml + prefill: + num-worker: 3 + tp: 8 + dcp-size: 8 + ep: 2 + dp-attn: false + decode: + num-worker: 1 + tp: 16 + dcp-size: 8 + ep: 16 + dp-attn: true + musespark-fp4-gb300-trt-agentic-dflash: image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610221558 model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d4b52e5dc2..29d663599f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12012,3 +12012,14 @@ - "Keep historical results, SPEED-Bench collectors, golden AL curves, vendor evals and OperatorX shapes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4762 retired: true + +- config-keys: + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-wideep-disagg + scenario-type: + - agentic-coding + description: + - "Add Kimi K3.1 Pro day-0 GB300 Dynamo SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification." + - "Aggregated TP8 DCP8 at concurrency 1; disaggregated TP8 DCP8 prefill with HiCache and TP8 DCP8 decode (1P3D, 1P2D, 2P3D, 3P4D) plus 3P1D with a TP16 DCP8 attention-DP2 EP16 decode." + pr-link: TBD From 42eb4202b0d8aec0c9d1ae226be83f09a489ae9e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 15:06:29 -0600 Subject: [PATCH 2/5] chore(kimik3.1): link the GB300 SGLang changelog entry to its PR --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 29d663599f..31b962c0b7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12022,4 +12022,4 @@ description: - "Add Kimi K3.1 Pro day-0 GB300 Dynamo SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification." - "Aggregated TP8 DCP8 at concurrency 1; disaggregated TP8 DCP8 prefill with HiCache and TP8 DCP8 decode (1P3D, 1P2D, 2P3D, 3P4D) plus 3P1D with a TP16 DCP8 attention-DP2 EP16 decode." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4802 From 6ee8b1d5c79c72889f4d3978154ee7cac5ce54df Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 18:25:37 -0600 Subject: [PATCH 3/5] fix(kimik3.1): pin Dynamo #18124 for the GB300 SGLang workers Published Dynamo wheels cannot load the kimi_k31 tokenizer and their SGLang argument check rejects DCP decode for K3.1's DSA layers. ai-dynamo/dynamo#18124 fixes both; pin it and record it as the router version. --- .../sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml | 3 ++- .../agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml | 3 ++- .../kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml | 3 ++- inferencex-e2e/configs/nvidia-master.yaml | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml index 5d902469c2..732857515d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml @@ -10,7 +10,8 @@ resources: dynamo: install: true source: - wheel: 1.6.0.dev20261016 + # ai-dynamo/dynamo#18124: its frontend loads the kimi_k31 tokenizer and its SGLang args allow DSA DCP decode. + rev: "4b0850752a7370aa20706095558616488b5cef35" environment: HF_HUB_CACHE: /hf_hub_cache observability: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml index 524f88a42f..a6812fe8e7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml @@ -10,7 +10,8 @@ resources: dynamo: install: true source: - wheel: 1.6.0.dev20261016 + # ai-dynamo/dynamo#18124: its frontend loads the kimi_k31 tokenizer and its SGLang args allow DSA DCP decode. + rev: "4b0850752a7370aa20706095558616488b5cef35" environment: HF_HUB_CACHE: /hf_hub_cache observability: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml index 0efb075006..46ee5eda65 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -12,7 +12,8 @@ base: dynamo: install: true source: - wheel: 1.6.0.dev20261016 + # ai-dynamo/dynamo#18124: its frontend loads the kimi_k31 tokenizer and its SGLang args allow DSA DCP decode. + rev: "4b0850752a7370aa20706095558616488b5cef35" environment: HF_HUB_CACHE: /hf_hub_cache observability: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 26ff2974f3..a907a46a31 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6537,7 +6537,7 @@ kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0.dev20261016" } + router: { name: dynamo-router, version: "4b0850752a7370aa20706095558616488b5cef35" } multinode: true disagg: false srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx @@ -6563,7 +6563,7 @@ kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0.dev20261016" } + router: { name: dynamo-router, version: "4b0850752a7370aa20706095558616488b5cef35" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -6649,7 +6649,7 @@ kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-wideep-disagg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0.dev20261016" } + router: { name: dynamo-router, version: "4b0850752a7370aa20706095558616488b5cef35" } kv-p2p-transfer: mooncake multinode: true disagg: true From 17711cd94473fe6361a3e96186d0ad64b98c85e0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 19:25:17 -0600 Subject: [PATCH 4/5] fix(kimik3.1): serve the GB300 SGLang points behind the SGLang router Dynamo #18124 does not build, so the GB300 K3.1 SGLang workers run as sglang.launch_server: the disaggregated points behind the SGLang Model Gateway (cache-aware, one gateway) and the aggregated point as a direct server. Engine args, topologies and concurrencies are unchanged; Dynamo-only worker flags give way to SGLang's own kimi_k3 parsers, and the config keys drop dynamo from their names. --- .../agentx/agg-tp8-dcp8-dflash7.yaml | 44 +------------ .../disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml | 57 ++++------------ .../gb300-fp4/agentx/disagg-variants.yaml | 65 +++++-------------- inferencex-e2e/configs/nvidia-master.yaml | 17 +++-- inferencex-e2e/perf-changelog.yaml | 8 +-- 5 files changed, 41 insertions(+), 150 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml index 732857515d..5cb0d882cc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml @@ -1,17 +1,11 @@ schema: 2 -name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-agg-tp8-dcp8-dflash7 +name: kimik3.1-fp4-gb300-sglang-agentx-agg-tp8-dcp8-dflash7 identity: model: {} container: {} resources: gpu_type: gb300 gpus_per_node: 4 -# The K3.1 SGLang image ships no Dynamo. -dynamo: - install: true - source: - # ai-dynamo/dynamo#18124: its frontend loads the kimi_k31 tokenizer and its SGLang args allow DSA DCP decode. - rev: "4b0850752a7370aa20706095558616488b5cef35" environment: HF_HUB_CACHE: /hf_hub_cache observability: @@ -22,39 +16,9 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 - - name: otel-collector - type: otel-collector - enabled: false frontend: - type: dynamo + type: sglang enable_multiple_frontends: false - env: - DYN_ROUTER_TEMPERATURE: "0" - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - args: - # The Kimi tokenizer is remote code, so chat templating stays in SGLang. - dyn-chat-processor: sglang - dyn-preprocess-workers: 4 - trust-remote-code: true - router-mode: kv - router-session-affinity-ttl-secs: "14400" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" engine: sglang roles: agg: @@ -95,13 +59,11 @@ roles: speculative-draft-attention-backend: cutedsl_mla enable-linear-replayssm-spec: true reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - dyn-tool-call-parser: kimi_k3 + tool-call-parser: kimi_k3 watchdog-timeout: 3600 enable-metrics: true weight-loader-prefetch-checkpoints: true model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: true sbatch_directives: mem: "0" srun_options: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml index a6812fe8e7..06dfb46e81 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml @@ -1,17 +1,11 @@ schema: 2 -name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-3p1d-p-tp8-d-tp16-dpa2-ep16 +name: kimik3.1-fp4-gb300-sglang-agentx-disagg-3p1d-p-tp8-d-tp16-dpa2-ep16 identity: model: {} container: {} resources: gpu_type: gb300 gpus_per_node: 4 -# The K3.1 SGLang image ships no Dynamo. -dynamo: - install: true - source: - # ai-dynamo/dynamo#18124: its frontend loads the kimi_k31 tokenizer and its SGLang args allow DSA DCP decode. - rev: "4b0850752a7370aa20706095558616488b5cef35" environment: HF_HUB_CACHE: /hf_hub_cache observability: @@ -22,38 +16,17 @@ slurm: health_check: max_attempts: 1440 interval_seconds: 10 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 - - name: otel-collector - type: otel-collector - enabled: false frontend: - type: dynamo + type: sglang-router + # One gateway keeps a single prefix tree, so cache-aware routing sees every turn of a session. enable_multiple_frontends: false - env: - DYN_ROUTER_TEMPERATURE: "0" - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" args: - # The Kimi tokenizer is remote code, so chat templating stays in SGLang. - dyn-chat-processor: sglang - dyn-preprocess-workers: 4 - trust-remote-code: true - router-mode: kv - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" + policy: cache_aware + request-id-headers: x-correlation-id + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true engine: sglang roles: prefill: @@ -107,18 +80,12 @@ roles: speculative-num-draft-tokens: 5 speculative-draft-kv-cache-dtype: fp8_e4m3 speculative-draft-attention-backend: cutedsl_mla - router-mode: kv - router-session-affinity-ttl-secs: "14400" - router-host-cache-hit-weight: 1 - router-kv-overlap-score-credit: 4 reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - dyn-tool-call-parser: kimi_k3 + tool-call-parser: kimi_k3 watchdog-timeout: 3600 enable-metrics: true weight-loader-prefetch-checkpoints: true model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: true decode: nodes: 4 workers: 1 @@ -167,13 +134,11 @@ roles: speculative-draft-attention-backend: cutedsl_mla enable-linear-replayssm-spec: true reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - dyn-tool-call-parser: kimi_k3 + tool-call-parser: kimi_k3 watchdog-timeout: 3600 enable-metrics: true weight-loader-prefetch-checkpoints: true model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: false sbatch_directives: mem: "0" srun_options: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml index 46ee5eda65..78279d80d9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -1,19 +1,13 @@ schema: 2 base: - name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg + name: kimik3.1-fp4-gb300-sglang-agentx-disagg identity: model: {} container: {} resources: gpu_type: gb300 gpus_per_node: 4 - # The K3.1 SGLang image ships no Dynamo. - dynamo: - install: true - source: - # ai-dynamo/dynamo#18124: its frontend loads the kimi_k31 tokenizer and its SGLang args allow DSA DCP decode. - rev: "4b0850752a7370aa20706095558616488b5cef35" environment: HF_HUB_CACHE: /hf_hub_cache observability: @@ -24,38 +18,17 @@ base: health_check: max_attempts: 1440 interval_seconds: 10 - services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 64 - - name: otel-collector - type: otel-collector - enabled: false frontend: - type: dynamo + type: sglang-router + # One gateway keeps a single prefix tree, so cache-aware routing sees every turn of a session. enable_multiple_frontends: false - env: - DYN_ROUTER_TEMPERATURE: "0" - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" args: - # The Kimi tokenizer is remote code, so chat templating stays in SGLang. - dyn-chat-processor: sglang - dyn-preprocess-workers: 4 - trust-remote-code: true - router-mode: kv - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" + policy: cache_aware + request-id-headers: x-correlation-id + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true engine: sglang roles: prefill: @@ -109,18 +82,12 @@ base: speculative-num-draft-tokens: 7 speculative-draft-kv-cache-dtype: fp8_e4m3 speculative-draft-attention-backend: cutedsl_mla - router-mode: kv - router-session-affinity-ttl-secs: "14400" - router-host-cache-hit-weight: 1 - router-kv-overlap-score-credit: 4 reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - dyn-tool-call-parser: kimi_k3 + tool-call-parser: kimi_k3 watchdog-timeout: 3600 enable-metrics: true weight-loader-prefetch-checkpoints: true model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: true decode: nodes: 4 workers: 2 @@ -162,13 +129,11 @@ base: speculative-draft-attention-backend: cutedsl_mla enable-linear-replayssm-spec: true reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - dyn-tool-call-parser: kimi_k3 + tool-call-parser: kimi_k3 watchdog-timeout: 3600 enable-metrics: true weight-loader-prefetch-checkpoints: true model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: false sbatch_directives: mem: "0" srun_options: @@ -183,17 +148,17 @@ base: AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' override_1p3d_c16: - name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-1p3d-c16 + name: kimik3.1-fp4-gb300-sglang-agentx-disagg-1p3d-c16 roles: decode: nodes: 6 workers: 3 override_1p2d_c24_c64: - name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-1p2d-c24-c64 + name: kimik3.1-fp4-gb300-sglang-agentx-disagg-1p2d-c24-c64 override_2p3d_c240: - name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-2p3d-c240 + name: kimik3.1-fp4-gb300-sglang-agentx-disagg-2p3d-c240 roles: prefill: nodes: 4 @@ -205,7 +170,7 @@ override_2p3d_c240: max-running-requests: 96 override_3p4d_c240: - name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-3p4d-c240 + name: kimik3.1-fp4-gb300-sglang-agentx-disagg-3p4d-c240 roles: prefill: nodes: 6 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index a907a46a31..96347262e0 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6530,14 +6530,13 @@ musespark-fp4-gb300-dynamo-sglang-agentic-dflash: ep: 4 dp-attn: false -kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg: +kimik3.1-fp4-gb300-sglang-agentic-dflash-agg: image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:gb300-nv precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "4b0850752a7370aa20706095558616488b5cef35" } + framework: sglang multinode: true disagg: false srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx @@ -6556,14 +6555,14 @@ kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg: ep: 1 dp-attn: false -kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: +kimik3.1-fp4-gb300-sglang-agentic-dflash-disagg: image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:gb300-nv precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "4b0850752a7370aa20706095558616488b5cef35" } + framework: sglang + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -6642,14 +6641,14 @@ kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: ep: 2 dp-attn: false -kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-wideep-disagg: +kimik3.1-fp4-gb300-sglang-agentic-dflash-wideep-disagg: image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:gb300-nv precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "4b0850752a7370aa20706095558616488b5cef35" } + framework: sglang + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mooncake multinode: true disagg: true diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index df01911c74..6a603218c9 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12024,12 +12024,12 @@ scenario-type: [fixed-seq-len] - config-keys: - - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg - - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg - - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-wideep-disagg + - kimik3.1-fp4-gb300-sglang-agentic-dflash-agg + - kimik3.1-fp4-gb300-sglang-agentic-dflash-disagg + - kimik3.1-fp4-gb300-sglang-agentic-dflash-wideep-disagg scenario-type: - agentic-coding description: - - "Add Kimi K3.1 Pro day-0 GB300 Dynamo SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification." + - "Add Kimi K3.1 Pro day-0 GB300 SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification, behind the SGLang Model Gateway (disaggregated) or a direct server (aggregated)." - "Aggregated TP8 DCP8 at concurrency 1; disaggregated TP8 DCP8 prefill with HiCache and TP8 DCP8 decode (1P3D, 1P2D, 2P3D, 3P4D) plus 3P1D with a TP16 DCP8 attention-DP2 EP16 decode." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4802 From 0b73d2f0f1afab3d7a29bbaabb760147b0b0668e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 4 Nov 2026 23:03:10 -0600 Subject: [PATCH 5/5] feat(kimik3.1): run the GB300 SGLang disagg points on the image's Dynamo Port Po-Han Huang's GB300 Dynamo+SGLang DFLASH disaggregated points from #4811: 3P1D c300, 2P1D c180 and 1P1D c76 with a TP16 (DCP8 x attention DP2, EP16 MegaMoE) decode, and 1P3D c60, 1P3D c36 and 1P4D c32 with TP8 DCP8 decodes, served by the Dynamo 1.6.0 the image ships with its KV router and the SGLang chat processor. They replace the SGLang-router disaggregated and wide-EP points; the aggregated c1 point stays a direct SGLang server. Co-authored-by: Po-Han Huang --- .../disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml | 153 ------------ .../agentx/disagg-tp16-variants.yaml | 232 ++++++++++++++++++ ...variants.yaml => disagg-tp8-variants.yaml} | 202 ++++++++------- inferencex-e2e/configs/nvidia-master.yaml | 123 +++------- inferencex-e2e/perf-changelog.yaml | 8 +- 5 files changed, 386 insertions(+), 332 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml rename inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/{disagg-variants.yaml => disagg-tp8-variants.yaml} (54%) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml deleted file mode 100644 index 06dfb46e81..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml +++ /dev/null @@ -1,153 +0,0 @@ -schema: 2 -name: kimik3.1-fp4-gb300-sglang-agentx-disagg-3p1d-p-tp8-d-tp16-dpa2-ep16 -identity: - model: {} - container: {} -resources: - gpu_type: gb300 - gpus_per_node: 4 -environment: - HF_HUB_CACHE: /hf_hub_cache -observability: - tachometer: - enabled: false -slurm: - time_limit: "6:00:00" -health_check: - max_attempts: 1440 - interval_seconds: 10 -frontend: - type: sglang-router - # One gateway keeps a single prefix tree, so cache-aware routing sees every turn of a session. - enable_multiple_frontends: false - args: - policy: cache_aware - request-id-headers: x-correlation-id - connect-timeout-secs: 900 - request-timeout-secs: 14400 - disable-health-check: true - disable-retries: true -engine: sglang -roles: - prefill: - nodes: 6 - workers: 3 - gpus: 8 - env: - PYTHONUNBUFFERED: "1" - CUDA_MODULE_LOADING: EAGER - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" - SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" - SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" - SGLANG_NUMA_BIND_V2: "0" - args: - served-model-name: moonshotai/Kimi-K3.1-Pro - trust-remote-code: true - tensor-parallel-size: 8 - expert-parallel-size: 2 - dcp-size: 8 - dcp-comm-backend: fi_a2a - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: float32 - page-size: 64 - dsa-prefill-backend: prims_ts - dsa-decode-backend: prims_ts - disaggregation-transfer-backend: mooncake - disaggregation-decode-allocation-policy: early - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - schedule-policy: shortest-prefill-first - mem-fraction-static: 0.88 - mamba-full-memory-ratio: 1.2 - mamba-max-states-per-path: 1 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 256 - # 120 GB per rank is 480 GB of each 4-GPU tray's host DRAM (dram-utilization 0.5). - enable-hierarchical-cache: true - hicache-size: 120 - hicache-write-policy: write_back - speculative-algorithm: DFLASH - speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview - speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 - # DFLASH counts the verification width: 4 drafts plus the current token, as decode. - speculative-num-draft-tokens: 5 - speculative-draft-kv-cache-dtype: fp8_e4m3 - speculative-draft-attention-backend: cutedsl_mla - reasoning-parser: kimi_k3 - tool-call-parser: kimi_k3 - watchdog-timeout: 3600 - enable-metrics: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load":true}' - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - PYTHONUNBUFFERED: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" - SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" - SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1536" - args: - served-model-name: moonshotai/Kimi-K3.1-Pro - trust-remote-code: true - tensor-parallel-size: 16 - attn-dp-size: 2 - enable-dp-lm-head: true - expert-parallel-size: 16 - # MegaMoE deadlocks with --enable-symm-mem, so symmetric memory stays off. - moe-a2a-backend: megamoe - dcp-size: 8 - dcp-comm-backend: fi_a2a - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: float32 - page-size: 64 - dsa-prefill-backend: prims_ts - dsa-decode-backend: prims_ts - disaggregation-transfer-backend: mooncake - disaggregation-decode-enable-radix-cache: true - disaggregation-decode-extra-slots: 16 - mem-fraction-static: 0.85 - max-running-requests: 384 - cuda-graph-max-bs-decode: 192 - speculative-algorithm: DFLASH - speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview - speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 - speculative-num-draft-tokens: 5 - speculative-use-block-verification: true - speculative-draft-kv-cache-dtype: fp8_e4m3 - speculative-draft-attention-backend: cutedsl_mla - enable-linear-replayssm-spec: true - reasoning-parser: kimi_k3 - tool-call-parser: kimi_k3 - watchdog-timeout: 3600 - enable-metrics: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load":true}' -sbatch_directives: - mem: "0" -srun_options: - mem: "0" - container-remap-root: "" -benchmark: - env: - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # The client splits this value on whitespace, so it has none. - AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml new file mode 100644 index 0000000000..9b5283d514 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml @@ -0,0 +1,232 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16 + identity: + model: {} + container: {} + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + enable-hierarchical-cache: true + # 96 GB per rank, 384 GB per prefill node. + hicache-size: 96 + hicache-write-policy: write_back + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + # A session's turns stay on one prefill worker. + router-mode: kv + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + router-session-affinity-ttl-secs: "14400" + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + PYTHONUNBUFFERED: "1" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NVSHMEM_REMOTE_TRANSPORT: none + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + # DCP8 x attention DP2 over the 16 GPUs; MoE runs EP16. + tensor-parallel-size: 16 + dcp-size: 8 + attn-dp-size: 2 + dcp-comm-backend: fi_a2a + expert-parallel-size: 16 + moe-a2a-backend: megamoe + enable-dp-lm-head: true + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.85 + max-running-requests: 384 + cuda-graph-max-bs-decode: 192 + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + enable-linear-replayssm-spec: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +override_3p1d_c300: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-3p1d-c300 + roles: + prefill: + nodes: 6 + workers: 3 + args: + # Verifies the current token plus 5 drafts: at this batch the shorter block is faster. + speculative-num-draft-tokens: 6 + decode: + args: + speculative-num-draft-tokens: 6 + +override_2p1d_c180: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-2p1d-c180 + roles: + prefill: + nodes: 4 + workers: 2 + +override_1p1d_c76: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-1p1d-c76 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml similarity index 54% rename from inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml rename to inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml index 78279d80d9..29d46f9d5f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml @@ -1,15 +1,22 @@ schema: 2 base: - name: kimik3.1-fp4-gb300-sglang-agentx-disagg + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8 identity: model: {} container: {} - resources: - gpu_type: gb300 - gpus_per_node: 4 + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. environment: HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. observability: tachometer: enabled: false @@ -18,17 +25,42 @@ base: health_check: max_attempts: 1440 interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false frontend: - type: sglang-router - # One gateway keeps a single prefix tree, so cache-aware routing sees every turn of a session. + type: dynamo enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" args: - policy: cache_aware - request-id-headers: x-correlation-id - connect-timeout-secs: 900 - request-timeout-secs: 14400 - disable-health-check: true - disable-retries: true + router-mode: kv + # The active-request term dominates KV overlap, so each turn goes to the least-loaded + # decode worker. + router-decode-active-request-weight: "1000000" + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" engine: sglang roles: prefill: @@ -38,143 +70,147 @@ base: env: PYTHONUNBUFFERED: "1" CUDA_MODULE_LOADING: EAGER - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" SGLANG_NUMA_BIND_V2: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" args: served-model-name: moonshotai/Kimi-K3.1-Pro trust-remote-code: true tensor-parallel-size: 8 - expert-parallel-size: 2 dcp-size: 8 dcp-comm-backend: fi_a2a - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: float32 - page-size: 64 + expert-parallel-size: 2 dsa-prefill-backend: prims_ts dsa-decode-backend: prims_ts - disaggregation-transfer-backend: mooncake - disaggregation-decode-allocation-policy: early + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 chunked-prefill-size: 16384 max-prefill-tokens: 16384 schedule-policy: shortest-prefill-first - mem-fraction-static: 0.88 mamba-full-memory-ratio: 1.2 mamba-max-states-per-path: 1 mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 256 - # 120 GB per rank is 480 GB of each 4-GPU tray's host DRAM (dram-utilization 0.5). enable-hierarchical-cache: true - hicache-size: 120 + # 96 GB per rank, 384 GB per prefill node. + hicache-size: 96 hicache-write-policy: write_back + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early speculative-algorithm: DFLASH speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 - # DFLASH counts the verification width: 6 drafts plus the current token, as decode. - speculative-num-draft-tokens: 7 - speculative-draft-kv-cache-dtype: fp8_e4m3 speculative-draft-attention-backend: cutedsl_mla - reasoning-parser: kimi_k3 - tool-call-parser: kimi_k3 + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + router-mode: kv + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + router-session-affinity-ttl-secs: "14400" + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false watchdog-timeout: 3600 enable-metrics: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load":true}' + enable-cache-report: true + kv_events: false decode: - nodes: 4 - workers: 2 + nodes: 6 + workers: 3 gpus: 8 env: PYTHONUNBUFFERED: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" args: served-model-name: moonshotai/Kimi-K3.1-Pro trust-remote-code: true tensor-parallel-size: 8 - expert-parallel-size: 2 dcp-size: 8 dcp-comm-backend: fi_a2a - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: float32 - page-size: 64 + expert-parallel-size: 2 dsa-prefill-backend: prims_ts dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.88 + max-running-requests: 64 disaggregation-transfer-backend: mooncake disaggregation-decode-enable-radix-cache: true disaggregation-decode-extra-slots: 16 - mem-fraction-static: 0.88 - max-running-requests: 64 + enable-linear-replayssm-spec: true speculative-algorithm: DFLASH speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 - speculative-num-draft-tokens: 7 - speculative-use-block-verification: true - speculative-draft-kv-cache-dtype: fp8_e4m3 speculative-draft-attention-backend: cutedsl_mla - enable-linear-replayssm-spec: true - reasoning-parser: kimi_k3 - tool-call-parser: kimi_k3 + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false watchdog-timeout: 3600 enable-metrics: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load":true}' + enable-cache-report: true + kv_events: false sbatch_directives: mem: "0" + exclusive: "" srun_options: - mem: "0" container-remap-root: "" + mem: "0" benchmark: + placement: + node: dedicated + colocate_with_frontend: true env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" # The client splits this value on whitespace, so it has none. - AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" -override_1p3d_c16: - name: kimik3.1-fp4-gb300-sglang-agentx-disagg-1p3d-c16 - roles: - decode: - nodes: 6 - workers: 3 +override_1p3d_c60: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p3d-c60 -override_1p2d_c24_c64: - name: kimik3.1-fp4-gb300-sglang-agentx-disagg-1p2d-c24-c64 +override_1p3d_c36: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p3d-c36 -override_2p3d_c240: - name: kimik3.1-fp4-gb300-sglang-agentx-disagg-2p3d-c240 +override_1p4d_c32: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p4d-c32 roles: - prefill: - nodes: 4 - workers: 2 - decode: - nodes: 6 - workers: 3 - args: - max-running-requests: 96 - -override_3p4d_c240: - name: kimik3.1-fp4-gb300-sglang-agentx-disagg-3p4d-c240 - roles: - prefill: - nodes: 6 - workers: 3 decode: nodes: 8 workers: 4 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 96347262e0..039c61a778 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6555,125 +6555,64 @@ kimik3.1-fp4-gb300-sglang-agentic-dflash-agg: ep: 1 dp-attn: false -kimik3.1-fp4-gb300-sglang-agentic-dflash-disagg: +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 model: moonshotai/Kimi-K3.1-Pro model-prefix: kimik3.1 runner: cluster:gb300-nv precision: fp4 - framework: sglang - router: { name: sglang-router, version: "0.3.2" } + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0" } kv-p2p-transfer: mooncake multinode: true disagg: true srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx scenarios: agentic-coding: - # Prefill HiCache: 120 GB per rank, 480 GB of each 4-GPU tray. - - dram-utilization: 0.5 + - dram-utilization: 0.80 search-space: - spec-decoding: draft_model + conc-list: [300] + srt-recipe: disagg-tp16-variants.yaml:override_3p1d_c300 kv-offloading: dram kv-offload-backend: { name: hicache } - conc-list: [16] - srt-recipe: disagg-variants.yaml:override_1p3d_c16 - prefill: - num-worker: 1 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false - decode: - num-worker: 3 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false + prefill: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } - spec-decoding: draft_model + conc-list: [180] + srt-recipe: disagg-tp16-variants.yaml:override_2p1d_c180 kv-offloading: dram kv-offload-backend: { name: hicache } - conc-list: [24, 32, 48, 64] - srt-recipe: disagg-variants.yaml:override_1p2d_c24_c64 - prefill: - num-worker: 1 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false + prefill: { num-worker: 2, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } - spec-decoding: draft_model + conc-list: [76] + srt-recipe: disagg-tp16-variants.yaml:override_1p1d_c76 kv-offloading: dram kv-offload-backend: { name: hicache } - conc-list: [240] - srt-recipe: disagg-variants.yaml:override_2p3d_c240 - prefill: - num-worker: 2 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false - decode: - num-worker: 3 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } - spec-decoding: draft_model + conc-list: [60] + srt-recipe: disagg-tp8-variants.yaml:override_1p3d_c60 kv-offloading: dram kv-offload-backend: { name: hicache } - conc-list: [240] - srt-recipe: disagg-variants.yaml:override_3p4d_c240 - prefill: - num-worker: 3 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false - decode: - num-worker: 4 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false - -kimik3.1-fp4-gb300-sglang-agentic-dflash-wideep-disagg: - image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 - model: moonshotai/Kimi-K3.1-Pro - model-prefix: kimik3.1 - runner: cluster:gb300-nv - precision: fp4 - framework: sglang - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx - scenarios: - agentic-coding: - - dram-utilization: 0.5 - search-space: + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } - spec-decoding: draft_model + conc-list: [36] + srt-recipe: disagg-tp8-variants.yaml:override_1p3d_c36 kv-offloading: dram kv-offload-backend: { name: hicache } - conc-list: [336] - srt-recipe: disagg-3p1d-p-tp8-d-tp16-dpa2-ep16.yaml - prefill: - num-worker: 3 - tp: 8 - dcp-size: 8 - ep: 2 - dp-attn: false - decode: - num-worker: 1 - tp: 16 - dcp-size: 8 - ep: 16 - dp-attn: true + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [32] + srt-recipe: disagg-tp8-variants.yaml:override_1p4d_c32 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 4, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } musespark-fp4-gb300-trt-agentic-dflash: image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610221558 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 6a603218c9..1fde8ad311 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12025,11 +12025,11 @@ - config-keys: - kimik3.1-fp4-gb300-sglang-agentic-dflash-agg - - kimik3.1-fp4-gb300-sglang-agentic-dflash-disagg - - kimik3.1-fp4-gb300-sglang-agentic-dflash-wideep-disagg + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg scenario-type: - agentic-coding description: - - "Add Kimi K3.1 Pro day-0 GB300 SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification, behind the SGLang Model Gateway (disaggregated) or a direct server (aggregated)." - - "Aggregated TP8 DCP8 at concurrency 1; disaggregated TP8 DCP8 prefill with HiCache and TP8 DCP8 decode (1P3D, 1P2D, 2P3D, 3P4D) plus 3P1D with a TP16 DCP8 attention-DP2 EP16 decode." + - "Add Kimi K3.1 Pro day-0 GB300 SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification." + - "Aggregated TP8 DCP8 at concurrency 1 as a direct SGLang server." + - "Disaggregated Dynamo (the image's 1.6.0, KV router) with TP8 DCP8 HiCache prefill feeding one TP16 (DCP8 x attention DP2, EP16 MegaMoE) decode (3P1D c300, 2P1D c180, 1P1D c76) or TP8 DCP8 decodes (1P3D c60 and c36, 1P4D c32)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4802