diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 7f9b8da792..dcd6abd1ea 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c + container_image: rocm/atom-dev@sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7 enable_multiple_frontends: false observability: enabled: false @@ -20,10 +20,9 @@ base: args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true - kv_cache_dtype: bf16 + kv_cache_dtype: fp8 index-cache-dtype: fp4 gpu-memory-utilization: 0.9 - max-num-seqs: 128 max-num-batched-tokens: 16384 attn-prefill-chunk-size: 16384 enable_prefix_caching: true @@ -38,6 +37,8 @@ base: method: dspark num-speculative-tokens: 5 tool-call-parser: dsml_v41 + # uvicorn's 5 s default resets AIPerf's pooled connections at high concurrency. + timeout-keep-alive: 900 env: OMP_NUM_THREADS: '4' ATOM_NUMA_BIND: '0' @@ -50,13 +51,18 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Every point admits 128 sequences and captures every -# decode size through 32 plus 48, 64 and 128. - override_tp2_c1: roles: agg: gpus: 2 + args: &c1_args + # DSpark's block-drafter warmup needs two sequences. + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + env: &c1_env + ATOM_ENABLE_PREFILL_DELAYER: '1' + # No step reaches this interval, so a new turn's prefill waits for the running decode. + ATOM_PREFILL_DECODE_INTERVAL: '1000000' benchmark: env: CONC: '1' @@ -65,86 +71,135 @@ override_tp2_c2: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: CONC: '2' -override_tp2_c4: +override_tp2_c16: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '96' benchmark: env: - CONC: '4' + CONC: '16' -override_tp2_c6: +override_tp2_c32: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '40' benchmark: env: - CONC: '6' + CONC: '32' -override_tp2_c8: +override_tp4_c1: roles: agg: - gpus: 2 + gpus: 4 + args: *c1_args + env: *c1_env benchmark: env: - CONC: '8' + CONC: '1' -override_tp2_c16: +override_tp4_c2: roles: agg: - gpus: 2 + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: - CONC: '16' + CONC: '2' -override_tp2_c32: +override_tp4_c4: roles: agg: - gpus: 2 + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: - CONC: '32' + CONC: '4' -override_tp2_c64: +override_tp4_c8: roles: agg: - gpus: 2 - # Run at least 4 decode passes after each prefill and batch the prefills - # that arrive meanwhile; at this load the batching also raises throughput. + gpus: 4 env: - ATOM_PREFILL_DECODE_INTERVAL: '4' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: - CONC: '64' + CONC: '8' -override_tp4_c1: +override_tp4_c32: roles: agg: gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '16' benchmark: env: - CONC: '1' + CONC: '32' -override_tp4_c8: +override_tp2_c64_dtp: roles: agg: - gpus: 4 + gpus: 2 + args: + # TBO stays off: it does not support SWA bounded replay. + enable-dp-attention: true + # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. + dp-load-balance: least_tokens + env: &dp_env + GPU_MAX_HW_QUEUES: '5' + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' benchmark: env: - CONC: '8' + CONC: '64' + # Session affinity keys on the AIPerf session id. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' -override_tp4_c32: +override_tp2_c128_dep: roles: agg: - gpus: 4 - # Run at least 16 decode passes after each prefill and batch the prefills - # that arrive meanwhile, so new turns stall decode less often. - env: - ATOM_PREFILL_DECODE_INTERVAL: '16' + gpus: 2 + args: + enable-dp-attention: true + enable-expert-parallel: true + dp-load-balance: least_tokens + env: *dp_env benchmark: env: - CONC: '32' + CONC: '128' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c256_dep: + roles: + agg: + gpus: 2 + args: + enable-dp-attention: true + enable-expert-parallel: true + dp-load-balance: least_tokens + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' + env: *dp_env + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 4b234ba7e4..bdba1eb4d7 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1017,10 +1017,9 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # High-Throughput: TP4 + HiCache, 180 GB of host KV per rank (720 of 735 GB). - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], srt-recipe: agentic.yaml } -# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610121516 + image: rocm/atom-dev:nightly_202611101510 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -1031,8 +1030,10 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 16, 32, 64], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 16, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 32], srt-recipe: agentic.yaml } # The golden acceptance curve is the GLM-5.3 NVFP4 SGLang curve in # infx/golden_al_distribution/glm-5.3-nvfp4__mtp__native__default-default.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 92a547f94d..7a99a1a1e3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12827,3 +12827,19 @@ - "Drop 3 prefill workers at c700 and c1228 (DEP16 decode): the DEP32 c3018 point has both higher interactivity and higher throughput per GPU than each of them. Drop 5 prefill workers at c2884 (DEP16 decode): it lies under the line between DEP32 c3018 and 7P c5540." - "Set stream-interval 32 on the agg role and on the disagg prefill role, which used the SGLang default of 1; disagg decode already used 32. Agg TP4 c1, c4 and c16 change otherwise not at all; disagg 7P c5540 and 9P c7480 change otherwise only by the image. The agg key is listed so the whole Dynamo + SGLang curve, agg points included, comes from this sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4997 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (ATOM e5b74bc86, ROCm 7.2.4) in the master config, and the AToMesh frontend container_image from sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c to sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, the same image." + - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8, 32]. TP2 c4, c6, c8 and c64 are dropped; TP4 c2 and c4 are new." + - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8) at every point; the index cache stays FP4." + - "Drop max-num-seqs 128, so every point except concurrency 1 runs ATOM's default admission. Concurrency 1 (TP2 and TP4) runs max-num-seqs 2, decode graphs [1, 2] and ATOM_PREFILL_DECODE_INTERVAL=1000000, the C1 recipe of ATOM's recipes/Kimi-K3.md (ROCm/ATOM#2659). The c256 point also captures graph size 256." + - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32, 16 at TP4 c32 (unchanged) and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." + - "The DP points add dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512 and GPU_MAX_HW_QUEUES=5, and AIPerf sends the correlation ID as the X-Dynamo-Session-Id header for session affinity." + - "Every point adds timeout-keep-alive 900 so the server keeps AIPerf's pooled connections open past uvicorn's 5 s default." + - "The new ATOM brings DeepSeek-V4.1 DP attention (ROCm/ATOM#2376), the V4.1 MXFP8 KV cache with DSpark on an FP8 cache (ROCm/ATOM#2725), DP-attention prefill balancing and graph replay (ROCm/ATOM#2551, #2696, #2697) and the default HIP Mega-MHC kernel on gfx950 (ROCm/ATOM#2568)." + - "No change to the DSpark draft's weight precision: ATOM loads the checkpoint's mtp.0-2 layers as shipped (FP8 E4M3 attention, shared experts and main_proj with 32x32 UE8M0 scales, MXFP4 routed experts, BF16 router, norms and heads) at both images. The draft shares the target's KV pool, so its KV cache follows the target from BF16 to MXFP8." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965