diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml new file mode 100644 index 0000000000..5cb0d882cc --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp8-dcp8-dflash7.yaml @@ -0,0 +1,78 @@ +schema: 2 +name: kimik3.1-fp4-gb300-sglang-agentx-agg-tp8-dcp8-dflash7 +identity: + model: {} + container: {} +resources: + gpu_type: gb300 + gpus_per_node: 4 +environment: + HF_HUB_CACHE: /hf_hub_cache +observability: + tachometer: + enabled: false +slurm: + time_limit: "4:00:00" +health_check: + max_attempts: 1440 + interval_seconds: 10 +frontend: + type: sglang + enable_multiple_frontends: false +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: "0" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 1 + dcp-size: 8 + dcp-comm-backend: fi_a2a + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: float32 + page-size: 64 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + chunked-prefill-size: 16384 + mem-fraction-static: 0.88 + max-running-requests: 1 + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + # DFLASH counts the verification width: 7 drafts plus the current token. + speculative-num-draft-tokens: 8 + speculative-use-block-verification: true + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-draft-attention-backend: cutedsl_mla + enable-linear-replayssm-spec: true + reasoning-parser: kimi_k3 + tool-call-parser: kimi_k3 + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load":true}' +sbatch_directives: + mem: "0" +srun_options: + mem: "0" + container-remap-root: "" +benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"temperature":1,"top_p":0.95}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml new file mode 100644 index 0000000000..9b5283d514 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml @@ -0,0 +1,232 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16 + identity: + model: {} + container: {} + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + enable-hierarchical-cache: true + # 96 GB per rank, 384 GB per prefill node. + hicache-size: 96 + hicache-write-policy: write_back + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + # A session's turns stay on one prefill worker. + router-mode: kv + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + router-session-affinity-ttl-secs: "14400" + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + PYTHONUNBUFFERED: "1" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NVSHMEM_REMOTE_TRANSPORT: none + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + # DCP8 x attention DP2 over the 16 GPUs; MoE runs EP16. + tensor-parallel-size: 16 + dcp-size: 8 + attn-dp-size: 2 + dcp-comm-backend: fi_a2a + expert-parallel-size: 16 + moe-a2a-backend: megamoe + enable-dp-lm-head: true + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.85 + max-running-requests: 384 + cuda-graph-max-bs-decode: 192 + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + enable-linear-replayssm-spec: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +override_3p1d_c300: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-3p1d-c300 + roles: + prefill: + nodes: 6 + workers: 3 + args: + # Verifies the current token plus 5 drafts: at this batch the shorter block is faster. + speculative-num-draft-tokens: 6 + decode: + args: + speculative-num-draft-tokens: 6 + +override_2p1d_c180: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-2p1d-c180 + roles: + prefill: + nodes: 4 + workers: 2 + +override_1p1d_c76: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-1p1d-c76 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml new file mode 100644 index 0000000000..29d46f9d5f --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml @@ -0,0 +1,216 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8 + identity: + model: {} + container: {} + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + # The active-request term dominates KV overlap, so each turn goes to the least-loaded + # decode worker. + router-decode-active-request-weight: "1000000" + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + enable-hierarchical-cache: true + # 96 GB per rank, 384 GB per prefill node. + hicache-size: 96 + hicache-write-policy: write_back + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + router-mode: kv + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + router-session-affinity-ttl-secs: "14400" + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + decode: + nodes: 6 + workers: 3 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.88 + max-running-requests: 64 + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + enable-linear-replayssm-spec: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +override_1p3d_c60: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p3d-c60 + +override_1p3d_c36: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p3d-c36 + +override_1p4d_c32: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p4d-c32 + roles: + decode: + nodes: 8 + workers: 4 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1ca0feca1a..039c61a778 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6530,6 +6530,90 @@ musespark-fp4-gb300-dynamo-sglang-agentic-dflash: ep: 4 dp-attn: false +kimik3.1-fp4-gb300-sglang-agentic-dflash-agg: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: sglang + multinode: true + disagg: false + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: draft_model + kv-offloading: none + conc-list: [1] + num-nodes: 2 + srt-recipe: agg-tp8-dcp8-dflash7.yaml + worker: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: draft_model + conc-list: [300] + srt-recipe: disagg-tp16-variants.yaml:override_3p1d_c300 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [180] + srt-recipe: disagg-tp16-variants.yaml:override_2p1d_c180 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 2, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [76] + srt-recipe: disagg-tp16-variants.yaml:override_1p1d_c76 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [60] + srt-recipe: disagg-tp8-variants.yaml:override_1p3d_c60 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [36] + srt-recipe: disagg-tp8-variants.yaml:override_1p3d_c36 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [32] + srt-recipe: disagg-tp8-variants.yaml:override_1p4d_c32 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 4, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + musespark-fp4-gb300-trt-agentic-dflash: image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610221558 model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 93ad828ec6..1fde8ad311 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12022,3 +12022,14 @@ - "Integrate each package over the formal GPU measurement window and emit cpu_power_valid, avg_cpu_socket_power_w, avg_total_cpu_power_w, and total_cpu_energy_j with sensor_kind rapl_package; RAPL reports no component rails. Concurrency points and evaluation coverage are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4154 scenario-type: [fixed-seq-len] + +- config-keys: + - kimik3.1-fp4-gb300-sglang-agentic-dflash-agg + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg + scenario-type: + - agentic-coding + description: + - "Add Kimi K3.1 Pro day-0 GB300 SGLang AgentX on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk DFLASH draft and block verification." + - "Aggregated TP8 DCP8 at concurrency 1 as a direct SGLang server." + - "Disaggregated Dynamo (the image's 1.6.0, KV router) with TP8 DCP8 HiCache prefill feeding one TP16 (DCP8 x attention DP2, EP16 MegaMoE) decode (3P1D c300, 2P1D c180, 1P1D c76) or TP8 DCP8 decodes (1P3D c60 and c36, 1P4D c32)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4802