From 5a695ba78879a322dfb0e93e424ebb29d4b11bc2 Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 14 Oct 2026 00:49:03 +0800 Subject: [PATCH 01/19] perf(amd): DSv4.1-Flash ATOM DPA + prefill TBO from c32 on the 1013 nightly Move dsv41flash-fp4-mi355x-atom-agentic-dspark to rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661, with ROCm/ATOM#2376 V4.1 DPA and prefill TBO). Run TP2 c32/64/128/256 and TP4 c32 with DP attention, prefill TBO, DP session affinity and the prefill delayer; drop TP2 c6. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 94 ++++++++++++++----- inferencex-e2e/configs/amd-master.yaml | 11 ++- inferencex-e2e/perf-changelog.yaml | 10 ++ 3 files changed, 87 insertions(+), 28 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 39f78b50d4..ff4370edcb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,20 +1,21 @@ # DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), # following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's -# revision (a0527c9dbfb801e22cd29882454df6f6d21cdd91). TP2 and TP4, no expert -# parallelism; the KV cache is GPU-resident. +# revision (661ef7661eef0d804d2b52d81ed41d80aeaf9fdb). TP2 and TP4, no expert +# parallelism, DP attention with prefill TBO (ROCm/ATOM#2376) from concurrency +# 32; the KV cache is GPU-resident. base: schema: 2 name: dsv41flash-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610121516 + container: rocm/atom-dev:nightly_202610131505 precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c + container_image: rocm/atom-dev@sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884 enable_multiple_frontends: false observability: enabled: false @@ -66,7 +67,10 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' # One variant per point. Every point admits 128 sequences and captures every -# decode size through 32 plus 48, 64 and 128. +# decode size through 32 plus 48, 64 and 128. The _dpa points (concurrency 32 +# and above) run DP attention with prefill TBO, keep each AIPerf session on one +# DP rank and defer prefill so that at least 10 decode passes run between +# prefills. override_tp2_c1: roles: @@ -92,14 +96,6 @@ override_tp2_c4: env: CONC: '4' -override_tp2_c6: - roles: - agg: - gpus: 2 - benchmark: - env: - CONC: '6' - override_tp2_c8: roles: agg: @@ -116,25 +112,76 @@ override_tp2_c16: env: CONC: '16' -override_tp2_c32: +override_tp2_c32_dpa: roles: agg: gpus: 2 + args: &dpa_args + enable-dp-attention: true + enable-tbo: true + # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. + dp-load-balance: least_tokens + env: &dpa_env + GPU_MAX_HW_QUEUES: '5' + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' benchmark: env: CONC: '32' + # Session affinity keys on the AIPerf session id. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' -override_tp2_c64: +override_tp2_c64_dpa: roles: agg: gpus: 2 - # Run at least 4 decode passes after each prefill and batch the prefills - # that arrive meanwhile; at this load the batching also raises throughput. - env: - ATOM_PREFILL_DECODE_INTERVAL: '4' + args: *dpa_args + env: *dpa_env benchmark: env: CONC: '64' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c128_dpa: + roles: + agg: + gpus: 2 + args: *dpa_args + env: *dpa_env + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c256_dpa: + roles: + agg: + gpus: 2 + args: *dpa_args + env: *dpa_env + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +# Plain-TP companions of the DP-attention points above, at the same concurrency. +override_tp2_c128: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '128' + +override_tp2_c256: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '256' override_tp4_c1: roles: @@ -152,14 +199,13 @@ override_tp4_c8: env: CONC: '8' -override_tp4_c32: +override_tp4_c32_dpa: roles: agg: gpus: 4 - # Run at least 16 decode passes after each prefill and batch the prefills - # that arrive meanwhile, so new turns stall decode less often. - env: - ATOM_PREFILL_DECODE_INTERVAL: '16' + args: *dpa_args + env: *dpa_env benchmark: env: CONC: '32' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 48a89330eb..af44ff47f4 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1518,10 +1518,11 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } -# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. +# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe, +# with DP attention and prefill TBO from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610121516 + image: rocm/atom-dev:nightly_202610131505 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -1531,8 +1532,10 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [32, 64, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 17ff00f039..f77bbf063a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9646,3 +9646,13 @@ - "ATOM's mono decode path (on by default) now takes any decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, not only DSpark verify steps. Steps of 12 or more rows that are a multiple of TP run its sublayers token-parallel, with reduce-scatters instead of all-reduces. The draft runner never runs token-parallel." - "No data-type or precision change to the DSpark draft. The checkpoint's bundled mtp.0-2 stages are stored like the target's layers: FP8 E4M3 weights with 32x32 UE8M0 block scales for the attention projections, shared experts and main_proj; MXFP4 routed experts; BF16 norms, router gate, Markov head and confidence head; F32 attention sinks and hc mixers. ATOM builds the draft with the target's native_quant_config() and make_v4_quant_config(), with no online quantization in this recipe, so it runs as shipped: FP8 activations on the FP8 GEMMs and the target's BF16 KV cache. Between 73defd9b8 and a0527c9db only the rows the draft reads under bounded replay change." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3928 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610131505 (digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884; ATOM 0.1.8.dev44+g661ef7661, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 (V4 FP4 graph replay with fixed-capacity metadata) and ROCm/ATOM#2556 (FP4 decode bounds drop the DP-padded empty tail)." + - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." + - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 1273f8b54f95727418970403bafbf151072f5d62 Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 14 Oct 2026 00:59:25 +0800 Subject: [PATCH 02/19] chore: set perf-changelog pr-link to #3965 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f77bbf063a..32d7b4e684 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9655,4 +9655,4 @@ - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610131505 (digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884; ATOM 0.1.8.dev44+g661ef7661, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 (V4 FP4 graph replay with fixed-capacity metadata) and ROCm/ATOM#2556 (FP4 decode bounds drop the DP-padded empty tail)." - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From cd3daee00aed8492a7f8744020b62902f8238e5b Mon Sep 17 00:00:00 2001 From: Yi Date: Fri, 16 Oct 2026 01:06:22 +0800 Subject: [PATCH 03/19] perf(amd): move DSv4.1-Flash ATOM AgentX to the 1015 nightly Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 6 +++--- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index ff4370edcb..66531143da 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,6 +1,6 @@ # DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), # following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's -# revision (661ef7661eef0d804d2b52d81ed41d80aeaf9fdb). TP2 and TP4, no expert +# revision (e6182e08b85f0e58bc15f7393ef57899220bfbaf). TP2 and TP4, no expert # parallelism, DP attention with prefill TBO (ROCm/ATOM#2376) from concurrency # 32; the KV cache is GPU-resident. base: @@ -8,14 +8,14 @@ base: name: dsv41flash-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610131505 + container: rocm/atom-dev:nightly_202610151507 precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884 + container_image: rocm/atom-dev@sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index af44ff47f4..db858ff525 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1522,7 +1522,7 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention and prefill TBO from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610131505 + image: rocm/atom-dev:nightly_202610151507 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 32d7b4e684..24f853a332 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9652,7 +9652,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610131505 (digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884; ATOM 0.1.8.dev44+g661ef7661, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 (V4 FP4 graph replay with fixed-capacity metadata) and ROCm/ATOM#2556 (FP4 decode bounds drop the DP-padded empty tail)." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610151507 (digest sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf; ATOM 0.1.8.dev63+ge6182e08b, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention) and ROCm/ATOM#2588 (prompt hashing and the prefix walk off the GPU critical path)." - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 57f16ecf170f001cdd6f5d8987970fb981792f67 Mon Sep 17 00:00:00 2001 From: Yi Date: Thu, 22 Oct 2026 11:48:19 -0500 Subject: [PATCH 04/19] perf(amd): move DSv4.1-Flash ATOM AgentX to the 1022 nightly rocm/atom-dev:nightly_202610221638 (sha256:32c81e56e468...; ATOM 0.1.9.dev11+gce46f6877, ROCm 7.2.4). Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index eada9477a3..2d5be82958 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf + container_image: rocm/atom-dev@sha256:32c81e56e4680065b6ea3e0ca8363a00cd3ccb932d8990e83d799b8bbb58a4b3 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 02d7e5ccc0..bd2a75c0de 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1536,7 +1536,7 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention and prefill TBO from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610151507 + image: rocm/atom-dev:nightly_202610221638 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e3b51f12d1..1f12102387 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10812,7 +10812,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610151507 (digest sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf; ATOM 0.1.8.dev63+ge6182e08b, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention) and ROCm/ATOM#2588 (prompt hashing and the prefix walk off the GPU critical path)." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610221638 (digest sha256:32c81e56e4680065b6ea3e0ca8363a00cd3ccb932d8990e83d799b8bbb58a4b3; ATOM 0.1.9.dev11+gce46f6877, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 7a22fb85158e44fde35ec1f0dad03f4b0ba210e3 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 01:40:48 +0800 Subject: [PATCH 05/19] perf(amd): DSv4.1-Flash ATOM AgentX on nightly_202610231734 Use the ATOM 2c772a5 release image (ROCm/ATOM run 39986380362, sha256:01a19e8b...) instead of nightly_202610231458. Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 2f6bc4b048..71a1c4585a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:05322b8651303be041b7adf2ee7d562815688ebe88ae2a11b3f8f2fc89adde61 + container_image: rocm/atom-dev@sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 18e674c96b..a910e86b50 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1557,7 +1557,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention (no TBO) from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610231458 + image: rocm/atom-dev:nightly_202610231734 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e2ada942ca..c31643f7d8 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11003,7 +11003,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231458 (digest sha256:05322b8651303be041b7adf2ee7d562815688ebe88ae2a11b3f8f2fc89adde61) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231734 (ATOM 2c772a5, digest sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 67b40a6853fde6a14c750d843227418ca2414457 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 01:53:24 +0800 Subject: [PATCH 06/19] perf(amd): DSv4.1-Flash ATOM TP2 c256 plain TP admits 256 sequences max-num-seqs 128 queued half of the 256 sessions (TTFT 490 s); raise it to 256 and capture a 256 decode graph on that point. Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 10 ++++++++-- inferencex-e2e/perf-changelog.yaml | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 71a1c4585a..df4d4997b3 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -52,8 +52,9 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Every point admits 128 sequences and captures every -# decode size through 32 plus 48, 64 and 128. The _dpa points (concurrency 32 +# One variant per point. Every point admits 128 sequences (256 on plain-TP TP2 +# c256) and captures every decode size through 32 plus 48, 64 and 128 (and 256 +# there). The _dpa points (concurrency 32 # and above) run DP attention without TBO (TBO does not support SWA bounded # replay), keep each AIPerf session on one DP rank and defer prefill so that at # least 10 decode passes run between prefills. @@ -164,6 +165,11 @@ override_tp2_c256: roles: agg: gpus: 2 + # Admit every session: at 128 sequences half of the 256 queue behind the + # running batch. + args: + max-num-seqs: 256 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' benchmark: env: CONC: '256' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 82e682cb0f..977520059e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11014,6 +11014,6 @@ - agentic-coding description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231734 (ATOM 2c772a5, digest sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." + - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on these points." + - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points; plain-TP TP2 c256 raises max-num-seqs from 128 to 256 and adds 256 to cudagraph-capture-sizes so every session is admitted (at 128 the previous run queued half the sessions, TTFT 490 s). Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 94c8a190b8d4853528d255fdbb928ade28929f38 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 16:38:16 +0800 Subject: [PATCH 07/19] perf(amd): DSv4.1-Flash ATOM AgentX on nightly_202610240802, max-num-seqs as B300 vLLM Move to the ATOM 11a2d56 release image (ROCm/ATOM run 40072622970, sha256:42716923...) and set max-num-seqs per point (per DP rank) like the B300 vLLM AgentX recipe: 1 at c1, 2x concurrency for c2-c16, concurrency from c32 up; capture sizes stop at each point's max-num-seqs. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 73 +++++++++++++++---- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 7 +- 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index df4d4997b3..7df0878eab 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105 + container_image: rocm/atom-dev@sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048 enable_multiple_frontends: false observability: enabled: false @@ -23,7 +23,6 @@ base: kv_cache_dtype: bf16 index-cache-dtype: fp4 gpu-memory-utilization: 0.9 - max-num-seqs: 128 max-num-batched-tokens: 16384 attn-prefill-chunk-size: 16384 enable_prefix_caching: true @@ -31,7 +30,6 @@ base: state-checkpoint-interval-tokens: 8192 level: 3 cudagraph-mode: FULL - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' # The checkpoint's bundled DSpark head. Throughput runs get the golden # AL (dsv41flash_dspark thinking_on, K5) from infx/srt_slurm/synthetic_acceptance.py; # evals keep real acceptance. @@ -52,17 +50,22 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Every point admits 128 sequences (256 on plain-TP TP2 -# c256) and captures every decode size through 32 plus 48, 64 and 128 (and 256 -# there). The _dpa points (concurrency 32 -# and above) run DP attention without TBO (TBO does not support SWA bounded -# replay), keep each AIPerf session on one DP rank and defer prefill so that at -# least 10 decode passes run between prefills. +# One variant per point. max-num-seqs (per DP rank under DP attention) follows +# the B300 vLLM AgentX recipe: one sequence at concurrency 1, twice the +# concurrency from 2 to 16, and the concurrency from 32 up. Each point captures +# every decode size through min(32, max-num-seqs) plus 48, 64, 128 and 256 up to +# its max-num-seqs. The _dpa points (concurrency 32 and above) run DP attention +# without TBO (TBO does not support SWA bounded replay), keep each AIPerf +# session on one DP rank and defer prefill so that at least 10 decode passes run +# between prefills. override_tp2_c1: roles: agg: gpus: 2 + args: + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' benchmark: env: CONC: '1' @@ -71,6 +74,9 @@ override_tp2_c2: roles: agg: gpus: 2 + args: + max-num-seqs: 4 + cudagraph-capture-sizes: '[1,2,3,4]' benchmark: env: CONC: '2' @@ -79,6 +85,9 @@ override_tp2_c4: roles: agg: gpus: 2 + args: + max-num-seqs: 8 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8]' benchmark: env: CONC: '4' @@ -87,6 +96,9 @@ override_tp2_c8: roles: agg: gpus: 2 + args: + max-num-seqs: 16 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -95,6 +107,9 @@ override_tp2_c16: roles: agg: gpus: 2 + args: + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' benchmark: env: CONC: '16' @@ -103,10 +118,12 @@ override_tp2_c32_dpa: roles: agg: gpus: 2 - args: &dpa_args + args: enable-dp-attention: true # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. dp-load-balance: least_tokens + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: &dpa_env GPU_MAX_HW_QUEUES: '5' ATOM_DP_SESSION_AFFINITY: '1' @@ -123,7 +140,11 @@ override_tp2_c64_dpa: roles: agg: gpus: 2 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 64 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64]' env: *dpa_env benchmark: env: @@ -134,7 +155,11 @@ override_tp2_c128_dpa: roles: agg: gpus: 2 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 128 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' env: *dpa_env benchmark: env: @@ -145,7 +170,11 @@ override_tp2_c256_dpa: roles: agg: gpus: 2 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 256 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' env: *dpa_env benchmark: env: @@ -153,10 +182,14 @@ override_tp2_c256_dpa: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' # Plain-TP companions of the DP-attention points above, at the same concurrency. + override_tp2_c128: roles: agg: gpus: 2 + args: + max-num-seqs: 128 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' benchmark: env: CONC: '128' @@ -165,8 +198,6 @@ override_tp2_c256: roles: agg: gpus: 2 - # Admit every session: at 128 sequences half of the 256 queue behind the - # running batch. args: max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' @@ -178,6 +209,9 @@ override_tp4_c1: roles: agg: gpus: 4 + args: + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' benchmark: env: CONC: '1' @@ -186,6 +220,9 @@ override_tp4_c8: roles: agg: gpus: 4 + args: + max-num-seqs: 16 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -194,7 +231,11 @@ override_tp4_c32_dpa: roles: agg: gpus: 4 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: *dpa_env benchmark: env: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index a910e86b50..60ea330b6b 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1557,7 +1557,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention (no TBO) from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610231734 + image: rocm/atom-dev:nightly_202610240802 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 977520059e..eb7fc8111c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11013,7 +11013,8 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231734 (ATOM 2c772a5, digest sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on these points." - - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points; plain-TP TP2 c256 raises max-num-seqs from 128 to 256 and adds 256 to cudagraph-capture-sizes so every session is admitted (at 128 the previous run queued half the sessions, TTFT 490 s). Other server settings are unchanged." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610240802 (ATOM 11a2d56, digest sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." + - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32)." + - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." + - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): 1 at concurrency 1, twice the concurrency from 2 to 16 (TP2 c2/c4/c8/c16 -> 4/8/16/32, TP4 c8 -> 16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From c4815ef5c57f7c9d2d8deed9f2fde0011b65ec9b Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 17:24:42 +0800 Subject: [PATCH 08/19] perf(amd): DSv4.1-Flash ATOM c1 admits 2 sequences max-num-seqs is twice the concurrency at c1 too (TP2 and TP4). Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 12 ++++++------ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 7df0878eab..9be4cd78d4 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -51,8 +51,8 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' # One variant per point. max-num-seqs (per DP rank under DP attention) follows -# the B300 vLLM AgentX recipe: one sequence at concurrency 1, twice the -# concurrency from 2 to 16, and the concurrency from 32 up. Each point captures +# the B300 vLLM AgentX recipe: twice the concurrency from 1 to 16 and the +# concurrency from 32 up. Each point captures # every decode size through min(32, max-num-seqs) plus 48, 64, 128 and 256 up to # its max-num-seqs. The _dpa points (concurrency 32 and above) run DP attention # without TBO (TBO does not support SWA bounded replay), keep each AIPerf @@ -64,8 +64,8 @@ override_tp2_c1: agg: gpus: 2 args: - max-num-seqs: 1 - cudagraph-capture-sizes: '[1]' + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' @@ -210,8 +210,8 @@ override_tp4_c1: agg: gpus: 4 args: - max-num-seqs: 1 - cudagraph-capture-sizes: '[1]' + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c4838e3541..a48fa7adff 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11080,5 +11080,5 @@ - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610240802 (ATOM 11a2d56, digest sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32)." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." - - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): 1 at concurrency 1, twice the concurrency from 2 to 16 (TP2 c2/c4/c8/c16 -> 4/8/16/32, TP4 c8 -> 16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." + - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): twice the concurrency from 1 to 16 (TP2 c1/c2/c4/c8/c16 -> 2/4/8/16/32, TP4 c1/c8 -> 2/16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 9542f0d294a270eb92e7667e08c9c7deb9ad1b39 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 17:33:47 +0800 Subject: [PATCH 09/19] perf(amd): DSv4.1-Flash ATOM AgentX on ATOM's default max-num-seqs Drop every max-num-seqs setting; decode graphs go back to the base list (sizes through 32 plus 48, 64, 128), with 256 added on the c256 points. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 49 +++---------------- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 9be4cd78d4..176c830812 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -30,6 +30,7 @@ base: state-checkpoint-interval-tokens: 8192 level: 3 cudagraph-mode: FULL + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' # The checkpoint's bundled DSpark head. Throughput runs get the golden # AL (dsv41flash_dspark thinking_on, K5) from infx/srt_slurm/synthetic_acceptance.py; # evals keep real acceptance. @@ -50,22 +51,17 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. max-num-seqs (per DP rank under DP attention) follows -# the B300 vLLM AgentX recipe: twice the concurrency from 1 to 16 and the -# concurrency from 32 up. Each point captures -# every decode size through min(32, max-num-seqs) plus 48, 64, 128 and 256 up to -# its max-num-seqs. The _dpa points (concurrency 32 and above) run DP attention -# without TBO (TBO does not support SWA bounded replay), keep each AIPerf -# session on one DP rank and defer prefill so that at least 10 decode passes run -# between prefills. +# One variant per point. max-num-seqs is ATOM's default on every point; decode +# graphs cover every size through 32 plus 48, 64 and 128 (and 256 on the c256 +# points). The _dpa points (concurrency 32 and above) run DP attention without +# TBO (TBO does not support SWA bounded replay), keep each AIPerf session on +# one DP rank and defer prefill so that at least 10 decode passes run between +# prefills. override_tp2_c1: roles: agg: gpus: 2 - args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' @@ -74,9 +70,6 @@ override_tp2_c2: roles: agg: gpus: 2 - args: - max-num-seqs: 4 - cudagraph-capture-sizes: '[1,2,3,4]' benchmark: env: CONC: '2' @@ -85,9 +78,6 @@ override_tp2_c4: roles: agg: gpus: 2 - args: - max-num-seqs: 8 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8]' benchmark: env: CONC: '4' @@ -96,9 +86,6 @@ override_tp2_c8: roles: agg: gpus: 2 - args: - max-num-seqs: 16 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -107,9 +94,6 @@ override_tp2_c16: roles: agg: gpus: 2 - args: - max-num-seqs: 32 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' benchmark: env: CONC: '16' @@ -122,8 +106,6 @@ override_tp2_c32_dpa: enable-dp-attention: true # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. dp-load-balance: least_tokens - max-num-seqs: 32 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: &dpa_env GPU_MAX_HW_QUEUES: '5' ATOM_DP_SESSION_AFFINITY: '1' @@ -143,8 +125,6 @@ override_tp2_c64_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 64 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64]' env: *dpa_env benchmark: env: @@ -158,8 +138,6 @@ override_tp2_c128_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 128 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' env: *dpa_env benchmark: env: @@ -173,7 +151,6 @@ override_tp2_c256_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' env: *dpa_env benchmark: @@ -187,9 +164,6 @@ override_tp2_c128: roles: agg: gpus: 2 - args: - max-num-seqs: 128 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' benchmark: env: CONC: '128' @@ -199,7 +173,6 @@ override_tp2_c256: agg: gpus: 2 args: - max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' benchmark: env: @@ -209,9 +182,6 @@ override_tp4_c1: roles: agg: gpus: 4 - args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' @@ -220,9 +190,6 @@ override_tp4_c8: roles: agg: gpus: 4 - args: - max-num-seqs: 16 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -234,8 +201,6 @@ override_tp4_c32_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 32 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: *dpa_env benchmark: env: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a48fa7adff..9641067e48 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11080,5 +11080,5 @@ - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610240802 (ATOM 11a2d56, digest sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32)." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." - - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): twice the concurrency from 1 to 16 (TP2 c1/c2/c4/c8/c16 -> 2/4/8/16/32, TP4 c1/c8 -> 2/16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." + - "Drop the max-num-seqs 128 cap: every point runs ATOM's default max-num-seqs (at 128 the plain-TP TP2 c256 point queued half the sessions, TTFT 490 s). Decode graphs stay at every size through 32 plus 48, 64 and 128; the TP2 c256 points (plain TP and DP attention) also capture 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From ce20d5812a2b59c058e375e738ba2b8da2405775 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 31 Oct 2026 15:57:55 +0800 Subject: [PATCH 10/19] DSv4.1-Flash ATOM AgentX: tuned delayer TP points through c32, DTP2 at c64, DEP2 at c128/c256 Plain TP2 / TP4 at concurrency 1-32: c1 keeps the ROCm/ATOM#2659 recipe, c2-c32 turn on the prefill delayer (decode interval 192 at c2-c8, 96 at c16, 30 / 32 at c32). DTP2 (DP attention without TBO) takes c64 and DEP2 c128 and c256; plain TP at 64 and above is dropped. The server keeps idle connections for 900 s, past uvicorn's 5 s default. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 99 +++++++++++++------ inferencex-e2e/configs/amd-master.yaml | 13 +-- inferencex-e2e/perf-changelog.yaml | 6 +- 3 files changed, 80 insertions(+), 38 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 2d937e42e0..70afab695e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -37,6 +37,8 @@ base: method: dspark num-speculative-tokens: 5 tool-call-parser: dsml_v41 + # uvicorn's 5 s default resets AIPerf's pooled connections at high concurrency. + timeout-keep-alive: 900 env: OMP_NUM_THREADS: '4' ATOM_NUMA_BIND: '0' @@ -49,15 +51,19 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point, on an MXFP8 KV cache. Plain TP everywhere, plus DEP2 -# (DP attention and expert parallelism over the TP group) from concurrency 64. -# max-num-seqs is ATOM's default except at concurrency 1; decode graphs cover -# every size through 32 plus 48, 64 and 128 (and 256 on the c256 points). +# One variant per point, on an MXFP8 KV cache: plain TP through concurrency 32, +# DTP2 (DP attention, MoE over TP) at 64 and DEP2 (DP attention plus expert +# parallelism) at 128 and 256. max-num-seqs is ATOM's default except at +# concurrency 1; decode graphs cover every size through 32 plus 48, 64 and 128 +# (and 256 on the c256 point). # # Concurrency 1 follows the C1 recipe of ROCm/ATOM#2659: the prefill delayer # with a decode interval no step reaches, so a new turn's prefill waits for the # running decode instead of interleaving with it, and max-num-seqs 2 (DSpark's # block-drafter warmup needs two sequences) with decode graphs at 1 and 2. +# From concurrency 2 to 32 the delayer runs a decode interval that shrinks as +# the load grows (192 through c8, 96 at c16, about 32 at c32), so a new prefill +# does not stall the running decodes, at a small TTFT cost. override_tp2_c1: roles: @@ -77,6 +83,9 @@ override_tp2_c2: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: CONC: '2' @@ -85,6 +94,9 @@ override_tp2_c4: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: CONC: '4' @@ -93,6 +105,9 @@ override_tp2_c8: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: CONC: '8' @@ -101,6 +116,9 @@ override_tp2_c16: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '96' benchmark: env: CONC: '16' @@ -109,69 +127,92 @@ override_tp2_c32: roles: agg: gpus: 2 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '30' benchmark: env: CONC: '32' -override_tp2_c64: +override_tp4_c1: roles: agg: - gpus: 2 + gpus: 4 + args: *c1_args + env: *c1_env benchmark: env: - CONC: '64' + CONC: '1' -override_tp2_c128: +override_tp4_c2: roles: agg: - gpus: 2 + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: - CONC: '128' + CONC: '2' -override_tp2_c256: +override_tp4_c4: roles: agg: - gpus: 2 - args: - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: - CONC: '256' + CONC: '4' -override_tp4_c1: +override_tp4_c8: roles: agg: gpus: 4 - args: *c1_args - env: *c1_env + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '192' benchmark: env: - CONC: '1' + CONC: '8' -override_tp4_c8: +override_tp4_c16: roles: agg: gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '96' benchmark: env: - CONC: '8' + CONC: '16' -# DEP2: the DSv4.1 DEP2 cell of ROCm/ATOM#2784 (DP attention without TBO, which -# does not support SWA bounded replay, plus expert parallelism). Each AIPerf -# session stays on one DP rank and at least 10 decode passes run between +override_tp4_c32: + roles: + agg: + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '32' + benchmark: + env: + CONC: '32' + +# DTP2 / DEP2: DP attention without TBO (which does not support SWA bounded +# replay); DEP2 adds expert parallelism, the DEP2 cell of ROCm/ATOM#2784. Each +# AIPerf session stays on one DP rank and at least 10 decode passes run between # prefills. -override_tp2_c64_dep: +override_tp2_c64_dtp: roles: agg: gpus: 2 args: enable-dp-attention: true - enable-expert-parallel: true # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. dp-load-balance: least_tokens - env: &dep_env + env: &dp_env GPU_MAX_HW_QUEUES: '5' ATOM_DP_SESSION_AFFINITY: '1' ATOM_DP_LB_REQ_EQUIV: '512' @@ -191,7 +232,7 @@ override_tp2_c128_dep: enable-dp-attention: true enable-expert-parallel: true dp-load-balance: least_tokens - env: *dep_env + env: *dp_env benchmark: env: CONC: '128' @@ -206,7 +247,7 @@ override_tp2_c256_dep: enable-expert-parallel: true dp-load-balance: least_tokens cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' - env: *dep_env + env: *dp_env benchmark: env: CONC: '256' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 657b66f271..7c101abd1c 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1628,9 +1628,9 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # Low-Latency: TP4, GPU-resident. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 64], srt-recipe: agentic.yaml } -# DeepSeek-V4.1-Flash on ATOM with an MXFP8 KV cache, following the upstream -# TP2 / TP4 AgentX recipe, with DEP2 (DP attention, no TBO, plus expert -# parallelism) from concurrency 64. +# DeepSeek-V4.1-Flash on ATOM with an MXFP8 KV cache: plain TP2 / TP4 through +# concurrency 32, DTP2 (DP attention, no TBO) at 64 and DEP2 (plus expert +# parallelism) at 128 and 256. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: image: rocm/atom-dev:nightly_202610281457 @@ -1644,9 +1644,10 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256], srt-recipe: agentic.yaml } - - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 256], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8c4d66ca09..ea4b2fc195 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11507,7 +11507,7 @@ description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610281457 (digest sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af): nightly_202610271459 rebuilt on ATOM 646da00fe and AITER 11a9b4f5c. AITER includes ROCm/aiter#7085, which keeps the MX quant-sort and multi-phase MoE sort in bounds under num_local_tokens (plain TP2 hit an intermittent illegal memory access at concurrency 16 and above on nightly_202610261558), plus #7020 and #7068." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - - "Run two shapes. Plain TP: TP2 at concurrency [1, 2, 4, 8, 16, 32, 64, 128, 256] and TP4 at [1, 8]. DEP2 (DP attention without TBO plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at concurrency [64, 128, 256]: dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5." - - "Drop the DP-attention points without expert parallelism (TP2 concurrency 32-256, TP4 concurrency 32); TP2 concurrency 32 and 64 move to plain TP." - - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 points also capture 256." + - "Plain TP2 and TP4 at concurrency [1, 2, 4, 8, 16, 32]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. Plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." + - "Concurrency 2 to 32 (TP2 and TP4) turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16, 30 (TP2) / 32 (TP4) at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." + - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 point also captures 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From c93c7915beeecc27a77fdc926477593d65139a6f Mon Sep 17 00:00:00 2001 From: Yi Date: Fri, 6 Nov 2026 11:17:26 +0800 Subject: [PATCH 11/19] DSv4.1-Flash ATOM AgentX: drop TP2 c4 and TP4 c16/c32 Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 41 ++----------------- inferencex-e2e/configs/amd-master.yaml | 4 +- inferencex-e2e/perf-changelog.yaml | 4 +- 3 files changed, 8 insertions(+), 41 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 70afab695e..693cb9b506 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -51,9 +51,9 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point, on an MXFP8 KV cache: plain TP through concurrency 32, -# DTP2 (DP attention, MoE over TP) at 64 and DEP2 (DP attention plus expert -# parallelism) at 128 and 256. max-num-seqs is ATOM's default except at +# One variant per point, on an MXFP8 KV cache: plain TP2 at concurrency 1, 2, 8, +# 16 and 32, plain TP4 at 1, 2, 4 and 8, DTP2 (DP attention, MoE over TP) at 64 +# and DEP2 (DP attention plus expert parallelism) at 128 and 256. max-num-seqs is ATOM's default except at # concurrency 1; decode graphs cover every size through 32 plus 48, 64 and 128 # (and 256 on the c256 point). # @@ -62,7 +62,7 @@ base: # running decode instead of interleaving with it, and max-num-seqs 2 (DSpark's # block-drafter warmup needs two sequences) with decode graphs at 1 and 2. # From concurrency 2 to 32 the delayer runs a decode interval that shrinks as -# the load grows (192 through c8, 96 at c16, about 32 at c32), so a new prefill +# the load grows (192 through c8, 96 at c16, 30 at c32), so a new prefill # does not stall the running decodes, at a small TTFT cost. override_tp2_c1: @@ -90,17 +90,6 @@ override_tp2_c2: env: CONC: '2' -override_tp2_c4: - roles: - agg: - gpus: 2 - env: - ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '192' - benchmark: - env: - CONC: '4' - override_tp2_c8: roles: agg: @@ -177,28 +166,6 @@ override_tp4_c8: env: CONC: '8' -override_tp4_c16: - roles: - agg: - gpus: 4 - env: - ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '96' - benchmark: - env: - CONC: '16' - -override_tp4_c32: - roles: - agg: - gpus: 4 - env: - ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '32' - benchmark: - env: - CONC: '32' - # DTP2 / DEP2: DP attention without TBO (which does not support SWA bounded # replay); DEP2 adds expert parallelism, the DEP2 cell of ROCm/ATOM#2784. Each # AIPerf session stays on one DP rank and at least 10 decode passes run between diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 8f4c5de20b..89c4a957a1 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1279,10 +1279,10 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 8, 16, 32], srt-recipe: agentic.yaml } - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d28488d867..68153285f1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11732,7 +11732,7 @@ description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610281457 (digest sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af): nightly_202610271459 rebuilt on ATOM 646da00fe and AITER 11a9b4f5c. AITER includes ROCm/aiter#7085, which keeps the MX quant-sort and multi-phase MoE sort in bounds under num_local_tokens (plain TP2 hit an intermittent illegal memory access at concurrency 16 and above on nightly_202610261558), plus #7020 and #7068." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - - "Plain TP2 and TP4 at concurrency [1, 2, 4, 8, 16, 32]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. Plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." - - "Concurrency 2 to 32 (TP2 and TP4) turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16, 30 (TP2) / 32 (TP4) at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." + - "Plain TP2 at concurrency [1, 2, 8, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." + - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 30 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 point also captures 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 57c93cbf174ebff885b29052411377b7b4d4cb29 Mon Sep 17 00:00:00 2001 From: Yi Date: Fri, 6 Nov 2026 11:20:17 +0800 Subject: [PATCH 12/19] DSv4.1-Flash ATOM AgentX: move to nightly_202611051503 Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 693cb9b506..43bdd04243 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af + container_image: rocm/atom-dev@sha256:c615bbe35972d6a808fc4829a4d987242a3c2c599fd9d6238280e236e3d4cbd0 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 89c4a957a1..76742d41a4 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1268,7 +1268,7 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # parallelism) at 128 and 256. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610281457 + image: rocm/atom-dev:nightly_202611051503 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 68153285f1..ebe58e7af3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11730,7 +11730,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610281457 (digest sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af): nightly_202610271459 rebuilt on ATOM 646da00fe and AITER 11a9b4f5c. AITER includes ROCm/aiter#7085, which keeps the MX quant-sort and multi-phase MoE sort in bounds under num_local_tokens (plain TP2 hit an intermittent illegal memory access at concurrency 16 and above on nightly_202610261558), plus #7020 and #7068." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611051503 (digest sha256:c615bbe35972d6a808fc4829a4d987242a3c2c599fd9d6238280e236e3d4cbd0; ATOM 0.1.10.dev31+g64412f109, ROCm 7.2.4), in the master config and the AToMesh frontend image." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - "Plain TP2 at concurrency [1, 2, 8, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 30 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." From 0fb378646794878a55c1a002617a8280902cbf2e Mon Sep 17 00:00:00 2001 From: yhl-amd Date: Sat, 7 Nov 2026 21:17:59 +0800 Subject: [PATCH 13/19] DSv4.1-Flash ATOM AgentX: nightly_202611071057, drop TP2 c8 Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 17 +++-------------- inferencex-e2e/configs/amd-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 1ec8a0c1f4..b7c1443b37 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:c615bbe35972d6a808fc4829a4d987242a3c2c599fd9d6238280e236e3d4cbd0 + container_image: rocm/atom-dev@sha256:e01f202b325049d3674c588273ff0000f14bd5f757cb15cfac769e8aa6332dec enable_multiple_frontends: false observability: enabled: false @@ -51,8 +51,8 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point, on an MXFP8 KV cache: plain TP2 at concurrency 1, 2, 8, -# 16 and 32, plain TP4 at 1, 2, 4 and 8, DTP2 (DP attention, MoE over TP) at 64 +# One variant per point, on an MXFP8 KV cache: plain TP2 at concurrency 1, 2, 16 +# and 32, plain TP4 at 1, 2, 4 and 8, DTP2 (DP attention, MoE over TP) at 64 # and DEP2 (DP attention plus expert parallelism) at 128 and 256. max-num-seqs is ATOM's default except at # concurrency 1; decode graphs cover every size through 32 plus 48, 64 and 128 # (and 256 on the c256 point). @@ -90,17 +90,6 @@ override_tp2_c2: env: CONC: '2' -override_tp2_c8: - roles: - agg: - gpus: 2 - env: - ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '192' - benchmark: - env: - CONC: '8' - override_tp2_c16: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 6548662fdd..d63815c397 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -957,7 +957,7 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # parallelism) at 128 and 256. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202611051503 + image: rocm/atom-dev:nightly_202611071057 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -968,7 +968,7 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 16, 32], srt-recipe: agentic.yaml } - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8fe58ad0a3..b37412b637 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12230,9 +12230,9 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611051503 (digest sha256:c615bbe35972d6a808fc4829a4d987242a3c2c599fd9d6238280e236e3d4cbd0; ATOM 0.1.10.dev31+g64412f109, ROCm 7.2.4), in the master config and the AToMesh frontend image." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611071057 (digest sha256:e01f202b325049d3674c588273ff0000f14bd5f757cb15cfac769e8aa6332dec, ROCm 7.2.4). It carries ROCm/aiter#7602, which re-selects the FlyDSL MoE kernels of the DeepSeek-V4.1-Flash TP2 decode buckets, in the master config and the AToMesh frontend image." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - - "Plain TP2 at concurrency [1, 2, 8, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." + - "Plain TP2 at concurrency [1, 2, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4 and c8, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 30 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 point also captures 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 87da6cd016c69c9f553f50a2d80eedf5a75189a2 Mon Sep 17 00:00:00 2001 From: Yi Date: Mon, 9 Nov 2026 00:21:33 +0800 Subject: [PATCH 14/19] DSv4.1-Flash ATOM AgentX: nightly_202611081535, TP2 c32 delayer interval 40 Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 6 +++--- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index b7c1443b37..7cef8b3c6e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:e01f202b325049d3674c588273ff0000f14bd5f757cb15cfac769e8aa6332dec + container_image: rocm/atom-dev@sha256:90cd052b6a2d1f818f543cb2e275d6bff1a0e2d41cfc3b5431b6837a2fdba3e7 enable_multiple_frontends: false observability: enabled: false @@ -62,7 +62,7 @@ base: # running decode instead of interleaving with it, and max-num-seqs 2 (DSpark's # block-drafter warmup needs two sequences) with decode graphs at 1 and 2. # From concurrency 2 to 32 the delayer runs a decode interval that shrinks as -# the load grows (192 through c8, 96 at c16, 30 at c32), so a new prefill +# the load grows (192 through c8, 96 at c16, 40 at c32), so a new prefill # does not stall the running decodes, at a small TTFT cost. override_tp2_c1: @@ -107,7 +107,7 @@ override_tp2_c32: gpus: 2 env: ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '30' + ATOM_PREFILL_DECODE_INTERVAL: '40' benchmark: env: CONC: '32' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index c6d3ba537f..74734bdbd7 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -975,7 +975,7 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # parallelism) at 128 and 256. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202611071057 + image: rocm/atom-dev:nightly_202611081535 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b98b697c18..d2cc4633a7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12425,9 +12425,9 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611071057 (digest sha256:e01f202b325049d3674c588273ff0000f14bd5f757cb15cfac769e8aa6332dec, ROCm 7.2.4). It carries ROCm/aiter#7602, which re-selects the FlyDSL MoE kernels of the DeepSeek-V4.1-Flash TP2 decode buckets, in the master config and the AToMesh frontend image." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611081535 (digest sha256:90cd052b6a2d1f818f543cb2e275d6bff1a0e2d41cfc3b5431b6837a2fdba3e7, ROCm 7.2.4). It carries ROCm/aiter#7602, which re-selects the FlyDSL MoE kernels of the DeepSeek-V4.1-Flash TP2 decode buckets, in the master config and the AToMesh frontend image." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - "Plain TP2 at concurrency [1, 2, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4 and c8, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." - - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 30 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." + - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 40 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 point also captures 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 2fedaf8606d73225551b56fe322f78fa05c4318f Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 11 Nov 2026 00:34:24 +0800 Subject: [PATCH 15/19] DSv4.1-Flash ATOM AgentX: nightly_202611101510 Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 7cef8b3c6e..945493aaa7 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:90cd052b6a2d1f818f543cb2e275d6bff1a0e2d41cfc3b5431b6837a2fdba3e7 + container_image: rocm/atom-dev@sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 8a13d4afc1..4881a7c5c5 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -975,7 +975,7 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # parallelism) at 128 and 256. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202611081535 + image: rocm/atom-dev:nightly_202611101510 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3f3cede655..13003b4b57 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12477,7 +12477,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611081535 (digest sha256:90cd052b6a2d1f818f543cb2e275d6bff1a0e2d41cfc3b5431b6837a2fdba3e7, ROCm 7.2.4). It carries ROCm/aiter#7602, which re-selects the FlyDSL MoE kernels of the DeepSeek-V4.1-Flash TP2 decode buckets, in the master config and the AToMesh frontend image." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (digest sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, ROCm 7.2.4). It carries ROCm/aiter#7602, which re-selects the FlyDSL MoE kernels of the DeepSeek-V4.1-Flash TP2 decode buckets, in the master config and the AToMesh frontend image." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - "Plain TP2 at concurrency [1, 2, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4 and c8, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 40 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." From 733165f12f47641fb1b25bb5c33c7dcd00613637 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 11 Nov 2026 16:09:32 +0900 Subject: [PATCH 16/19] docs(dsv41flash-atom): sync the MI355X AgentX changelog block with the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (ATOM e5b74bc86, ROCm 7.2.4) in the master config, and the AToMesh frontend container_image from sha256:a60811a8... to sha256:160a4ec4..., the same image. - Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 at 64, DEP2 at [128, 256] and plain TP4 at [1, 2, 4, 8]. TP2 c4, c6, c8 and c64 and TP4 c32 are dropped; TP4 c2 and c4 are new. - Switch the KV cache from BF16 to MXFP8 at every point; the index cache stays FP4. - Drop max-num-seqs 128. Concurrency 1 runs max-num-seqs 2, decode graphs [1, 2] and ATOM_PREFILL_DECODE_INTERVAL=1000000, the C1 recipe of ATOM's recipes/Kimi-K3.md (ROCm/ATOM#2659). The c256 point also captures 256. - Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2-c8, 96 at TP2 c16, 40 at TP2 c32 and 10 at the DP points. - The DP points add dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512 and GPU_MAX_HW_QUEUES=5, with AIPerf session ids as X-Dynamo-Session-Id. - Every point adds timeout-keep-alive 900. - The new ATOM brings V4.1 DP attention (#2376), the V4.1 MXFP8 KV cache with DSpark (#2725), DP prefill balancing and graph replay (#2551, #2696, #2697) and the default HIP Mega-MHC kernel (#2568). - No change to the DSpark draft's weight precision; its KV cache follows the target from BF16 to MXFP8. Also replace the recipe and master-config header comments with line comments (AGENTS.md "Comments"); no setting changes. - 镜像从 rocm/atom-dev:nightly_202610121516 更新为 nightly_202611101510(ATOM e5b74bc86,ROCm 7.2.4),AToMesh 前端 container_image 同步更新为同一镜像。 - 纯 TP2 [1, 2, 4, 6, 8, 16, 32, 64] 和纯 TP4 [1, 8, 32] 改为纯 TP2 [1, 2, 16, 32]、DTP2 c64、DEP2 [128, 256] 和纯 TP4 [1, 2, 4, 8]。删除 TP2 c4、c6、c8、c64 和 TP4 c32;新增 TP4 c2 和 c4。 - 所有点的 KV cache 从 BF16 改为 MXFP8;index cache 仍为 FP4。 - 去掉 max-num-seqs 128。并发 1 使用 max-num-seqs 2、decode 图 [1, 2] 和 ATOM_PREFILL_DECODE_INTERVAL=1000000,即 ATOM recipes/Kimi-K3.md 的 C1 recipe(ROCm/ATOM#2659)。c256 额外捕获 256。 - 各点设置 ATOM_PREFILL_DECODE_INTERVAL:TP2 c2 和 TP4 c2-c8 为 192,TP2 c16 为 96,TP2 c32 为 40,DP 点为 10。 - DP 点增加 dp-load-balance least_tokens、ATOM_DP_SESSION_AFFINITY=1、ATOM_DP_LB_REQ_EQUIV=512 和 GPU_MAX_HW_QUEUES=5,AIPerf 以 X-Dynamo-Session-Id 发送 session id。 - 所有点增加 timeout-keep-alive 900。 - 新 ATOM 带来 V4.1 DP attention(#2376)、V4.1 MXFP8 KV cache 与 DSpark(#2725)、DP prefill 均衡与图回放(#2551、#2696、#2697)以及默认 HIP Mega-MHC kernel(#2568)。 - DSpark draft 权重精度不变;其 KV cache 随 target 从 BF16 变为 MXFP8。 另将 recipe 和 master config 的头部注释改为行注释(AGENTS.md "Comments"),不改变任何设置。 Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 22 +++---------------- inferencex-e2e/configs/amd-master.yaml | 3 --- inferencex-e2e/perf-changelog.yaml | 14 +++++++----- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 945493aaa7..772e20accf 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -51,29 +51,17 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point, on an MXFP8 KV cache: plain TP2 at concurrency 1, 2, 16 -# and 32, plain TP4 at 1, 2, 4 and 8, DTP2 (DP attention, MoE over TP) at 64 -# and DEP2 (DP attention plus expert parallelism) at 128 and 256. max-num-seqs is ATOM's default except at -# concurrency 1; decode graphs cover every size through 32 plus 48, 64 and 128 -# (and 256 on the c256 point). -# -# Concurrency 1 follows the C1 recipe of ROCm/ATOM#2659: the prefill delayer -# with a decode interval no step reaches, so a new turn's prefill waits for the -# running decode instead of interleaving with it, and max-num-seqs 2 (DSpark's -# block-drafter warmup needs two sequences) with decode graphs at 1 and 2. -# From concurrency 2 to 32 the delayer runs a decode interval that shrinks as -# the load grows (192 through c8, 96 at c16, 40 at c32), so a new prefill -# does not stall the running decodes, at a small TTFT cost. - override_tp2_c1: roles: agg: gpus: 2 args: &c1_args + # DSpark's block-drafter warmup needs two sequences. max-num-seqs: 2 cudagraph-capture-sizes: '[1,2]' env: &c1_env ATOM_ENABLE_PREFILL_DELAYER: '1' + # No step reaches this interval, so a new turn's prefill waits for the running decode. ATOM_PREFILL_DECODE_INTERVAL: '1000000' benchmark: env: @@ -155,16 +143,12 @@ override_tp4_c8: env: CONC: '8' -# DTP2 / DEP2: DP attention without TBO (which does not support SWA bounded -# replay); DEP2 adds expert parallelism, the DEP2 cell of ROCm/ATOM#2784. Each -# AIPerf session stays on one DP rank and at least 10 decode passes run between -# prefills. - override_tp2_c64_dtp: roles: agg: gpus: 2 args: + # TBO stays off: it does not support SWA bounded replay. enable-dp-attention: true # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. dp-load-balance: least_tokens diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 09dbc8359f..35ddf624d1 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -964,9 +964,6 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # High-Throughput: TP4 + HiCache, 180 GB of host KV per rank (720 of 735 GB). - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], srt-recipe: agentic.yaml } -# DeepSeek-V4.1-Flash on ATOM with an MXFP8 KV cache: plain TP2 / TP4 through -# concurrency 32, DTP2 (DP attention, no TBO) at 64 and DEP2 (plus expert -# parallelism) at 128 and 256. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: image: rocm/atom-dev:nightly_202611101510 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3c24f8da48..97e846962a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12716,9 +12716,13 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (digest sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, ROCm 7.2.4). It carries ROCm/aiter#7602, which re-selects the FlyDSL MoE kernels of the DeepSeek-V4.1-Flash TP2 decode buckets, in the master config and the AToMesh frontend image." - - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." - - "Plain TP2 at concurrency [1, 2, 16, 32] and plain TP4 at [1, 2, 4, 8]; DTP2 (DP attention without TBO) at 64; DEP2 (plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at [128, 256]. The DP points run dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5. TP2 c4 and c8, TP4 c16 and c32, plain TP at 64 and above, the DP points below 64 and DTP2 at 128 and 256 are dropped." - - "Plain TP points from concurrency 2 turn on the prefill delayer with ATOM_PREFILL_DECODE_INTERVAL 192 at c2-c8, 96 at c16 and 40 at c32. The server keeps idle connections for 900 s (timeout-keep-alive), past uvicorn's 5 s default that reset AIPerf's pooled connections." - - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 point also captures 256." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (ATOM e5b74bc86, ROCm 7.2.4) in the master config, and the AToMesh frontend container_image from sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c to sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, the same image." + - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8]. TP2 c4, c6, c8 and c64 and TP4 c32 are dropped; TP4 c2 and c4 are new." + - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8) at every point; the index cache stays FP4." + - "Drop max-num-seqs 128, so every point except concurrency 1 runs ATOM's default admission. Concurrency 1 (TP2 and TP4) runs max-num-seqs 2, decode graphs [1, 2] and ATOM_PREFILL_DECODE_INTERVAL=1000000, the C1 recipe of ATOM's recipes/Kimi-K3.md (ROCm/ATOM#2659). The c256 point also captures graph size 256." + - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32 and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." + - "The DP points add dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512 and GPU_MAX_HW_QUEUES=5, and AIPerf sends the correlation ID as the X-Dynamo-Session-Id header for session affinity." + - "Every point adds timeout-keep-alive 900 so the server keeps AIPerf's pooled connections open past uvicorn's 5 s default." + - "The new ATOM brings DeepSeek-V4.1 DP attention (ROCm/ATOM#2376), the V4.1 MXFP8 KV cache with DSpark on an FP8 cache (ROCm/ATOM#2725), DP-attention prefill balancing and graph replay (ROCm/ATOM#2551, #2696, #2697) and the default HIP Mega-MHC kernel on gfx950 (ROCm/ATOM#2568)." + - "No change to the DSpark draft's weight precision: ATOM loads the checkpoint's mtp.0-2 layers as shipped (FP8 E4M3 attention, shared experts and main_proj with 32x32 UE8M0 scales, MXFP4 routed experts, BF16 router, norms and heads) at both images. The draft shares the target's KV pool, so its KV cache follows the target from BF16 to MXFP8." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From cfc9ae25732ddda4fee1d5e460f75f7fa0301f59 Mon Sep 17 00:00:00 2001 From: honglie Date: Thu, 12 Nov 2026 15:14:10 +0800 Subject: [PATCH 17/19] Restore TP4 c32 on the 1110 nightly with the prefill delayer at interval 40 Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 11 +++++++++++ inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 772e20accf..b658dbb2eb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -143,6 +143,17 @@ override_tp4_c8: env: CONC: '8' +override_tp4_c32: + roles: + agg: + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '40' + benchmark: + env: + CONC: '32' + override_tp2_c64_dtp: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 4f31403afc..d954ce7f24 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1033,7 +1033,7 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 16, 32], srt-recipe: agentic.yaml } - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 32], srt-recipe: agentic.yaml } # The golden acceptance curve is the GLM-5.3 NVFP4 SGLang curve in # infx/golden_al_distribution/glm-5.3-nvfp4__mtp__native__default-default.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f3eeb22d2e..010da85a93 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12782,10 +12782,10 @@ - agentic-coding description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (ATOM e5b74bc86, ROCm 7.2.4) in the master config, and the AToMesh frontend container_image from sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c to sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, the same image." - - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8]. TP2 c4, c6, c8 and c64 and TP4 c32 are dropped; TP4 c2 and c4 are new." + - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8, 32]. TP2 c4, c6, c8 and c64 are dropped; TP4 c2 and c4 are new." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8) at every point; the index cache stays FP4." - "Drop max-num-seqs 128, so every point except concurrency 1 runs ATOM's default admission. Concurrency 1 (TP2 and TP4) runs max-num-seqs 2, decode graphs [1, 2] and ATOM_PREFILL_DECODE_INTERVAL=1000000, the C1 recipe of ATOM's recipes/Kimi-K3.md (ROCm/ATOM#2659). The c256 point also captures graph size 256." - - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32 and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." + - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32 and TP4 c32 and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." - "The DP points add dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512 and GPU_MAX_HW_QUEUES=5, and AIPerf sends the correlation ID as the X-Dynamo-Session-Id header for session affinity." - "Every point adds timeout-keep-alive 900 so the server keeps AIPerf's pooled connections open past uvicorn's 5 s default." - "The new ATOM brings DeepSeek-V4.1 DP attention (ROCm/ATOM#2376), the V4.1 MXFP8 KV cache with DSpark on an FP8 cache (ROCm/ATOM#2725), DP-attention prefill balancing and graph replay (ROCm/ATOM#2551, #2696, #2697) and the default HIP Mega-MHC kernel on gfx950 (ROCm/ATOM#2568)." From 76badcf5ae723e2c46acfa60521074d79d21e278 Mon Sep 17 00:00:00 2001 From: honglie Date: Thu, 12 Nov 2026 15:21:54 +0800 Subject: [PATCH 18/19] Revert "Restore TP4 c32 on the 1110 nightly with the prefill delayer at interval 40" This reverts commit cfc9ae25732ddda4fee1d5e460f75f7fa0301f59. Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 11 ----------- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index b658dbb2eb..772e20accf 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -143,17 +143,6 @@ override_tp4_c8: env: CONC: '8' -override_tp4_c32: - roles: - agg: - gpus: 4 - env: - ATOM_ENABLE_PREFILL_DELAYER: '1' - ATOM_PREFILL_DECODE_INTERVAL: '40' - benchmark: - env: - CONC: '32' - override_tp2_c64_dtp: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index d954ce7f24..4f31403afc 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1033,7 +1033,7 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 16, 32], srt-recipe: agentic.yaml } - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 32], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } # The golden acceptance curve is the GLM-5.3 NVFP4 SGLang curve in # infx/golden_al_distribution/glm-5.3-nvfp4__mtp__native__default-default.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 010da85a93..f3eeb22d2e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12782,10 +12782,10 @@ - agentic-coding description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (ATOM e5b74bc86, ROCm 7.2.4) in the master config, and the AToMesh frontend container_image from sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c to sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, the same image." - - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8, 32]. TP2 c4, c6, c8 and c64 are dropped; TP4 c2 and c4 are new." + - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8]. TP2 c4, c6, c8 and c64 and TP4 c32 are dropped; TP4 c2 and c4 are new." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8) at every point; the index cache stays FP4." - "Drop max-num-seqs 128, so every point except concurrency 1 runs ATOM's default admission. Concurrency 1 (TP2 and TP4) runs max-num-seqs 2, decode graphs [1, 2] and ATOM_PREFILL_DECODE_INTERVAL=1000000, the C1 recipe of ATOM's recipes/Kimi-K3.md (ROCm/ATOM#2659). The c256 point also captures graph size 256." - - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32 and TP4 c32 and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." + - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32 and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." - "The DP points add dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512 and GPU_MAX_HW_QUEUES=5, and AIPerf sends the correlation ID as the X-Dynamo-Session-Id header for session affinity." - "Every point adds timeout-keep-alive 900 so the server keeps AIPerf's pooled connections open past uvicorn's 5 s default." - "The new ATOM brings DeepSeek-V4.1 DP attention (ROCm/ATOM#2376), the V4.1 MXFP8 KV cache with DSpark on an FP8 cache (ROCm/ATOM#2725), DP-attention prefill balancing and graph replay (ROCm/ATOM#2551, #2696, #2697) and the default HIP Mega-MHC kernel on gfx950 (ROCm/ATOM#2568)." From 3906935f0af3121afba092f19def7efc8e60cf2c Mon Sep 17 00:00:00 2001 From: honglie Date: Thu, 12 Nov 2026 16:00:00 +0800 Subject: [PATCH 19/19] Restore TP4 c32 on the 1110 nightly with the prefill delayer at interval 16 Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 11 +++++++++++ inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 772e20accf..dcd6abd1ea 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -143,6 +143,17 @@ override_tp4_c8: env: CONC: '8' +override_tp4_c32: + roles: + agg: + gpus: 4 + env: + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '16' + benchmark: + env: + CONC: '32' + override_tp2_c64_dtp: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 4f31403afc..d954ce7f24 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1033,7 +1033,7 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 16, 32], srt-recipe: agentic.yaml } - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64], srt-recipe: agentic.yaml } - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [128, 256], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 32], srt-recipe: agentic.yaml } # The golden acceptance curve is the GLM-5.3 NVFP4 SGLang curve in # infx/golden_al_distribution/glm-5.3-nvfp4__mtp__native__default-default.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f3eeb22d2e..f847761c29 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12782,10 +12782,10 @@ - agentic-coding description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202611101510 (ATOM e5b74bc86, ROCm 7.2.4) in the master config, and the AToMesh frontend container_image from sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c to sha256:160a4ec49379e7f1980a4e783254366b1d5c643b012abca57a21c0bcbbfe83a7, the same image." - - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8]. TP2 c4, c6, c8 and c64 and TP4 c32 are dropped; TP4 c2 and c4 are new." + - "Replace plain TP2 at concurrency [1, 2, 4, 6, 8, 16, 32, 64] and plain TP4 at [1, 8, 32] with plain TP2 at [1, 2, 16, 32], DTP2 (DP attention) at 64, DEP2 (DP attention plus expert parallelism) at [128, 256] and plain TP4 at [1, 2, 4, 8, 32]. TP2 c4, c6, c8 and c64 are dropped; TP4 c2 and c4 are new." - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8) at every point; the index cache stays FP4." - "Drop max-num-seqs 128, so every point except concurrency 1 runs ATOM's default admission. Concurrency 1 (TP2 and TP4) runs max-num-seqs 2, decode graphs [1, 2] and ATOM_PREFILL_DECODE_INTERVAL=1000000, the C1 recipe of ATOM's recipes/Kimi-K3.md (ROCm/ATOM#2659). The c256 point also captures graph size 256." - - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32 and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." + - "Set ATOM_PREFILL_DECODE_INTERVAL per point: 192 at TP2 c2 and TP4 c2, c4 and c8, 96 at TP2 c16, 40 at TP2 c32, 16 at TP4 c32 (unchanged) and 10 at the DP points (previously unset except 4 at TP2 c64 and 16 at TP4 c32)." - "The DP points add dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512 and GPU_MAX_HW_QUEUES=5, and AIPerf sends the correlation ID as the X-Dynamo-Session-Id header for session affinity." - "Every point adds timeout-keep-alive 900 so the server keeps AIPerf's pooled connections open past uvicorn's 5 s default." - "The new ATOM brings DeepSeek-V4.1 DP attention (ROCm/ATOM#2376), the V4.1 MXFP8 KV cache with DSpark on an FP8 cache (ROCm/ATOM#2725), DP-attention prefill balancing and graph replay (ROCm/ATOM#2551, #2696, #2697) and the default HIP Mega-MHC kernel on gfx950 (ROCm/ATOM#2568)."