From e6a34cfabf97f72573cd93a1501adc0bf3c5b938 Mon Sep 17 00:00:00 2001 From: liyingli Date: Wed, 28 Oct 2026 07:53:31 +0000 Subject: [PATCH 01/12] feat(infx): register the infera-sglang framework and its installer infera-sglang runs the stock SGLang image with Infera installed at the master config's router version from Infera's GitHub release (configs/infera-sglang.sh, bound through INSTALLERS); synthetic acceptance follows SGLang. Signed-off-by: liyingli Co-authored-by: Cursor --- .../srt-slurm-recipes/configs/infera-sglang.sh | 18 ++++++++++++++++++ .../infx/srt_slurm/synthetic_acceptance.py | 1 + inferencex-e2e/infx/srt_slurm/workload.py | 1 + .../infx/tests/srt_slurm/test_setup_scripts.py | 2 ++ .../srt_slurm/test_synthetic_acceptance.py | 10 ++++++++++ 5 files changed, 32 insertions(+) create mode 100755 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh new file mode 100755 index 0000000000..b5b42fe82c --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Install Infera, at the master config's router version, into the stock SGLang image for the +# infera-sglang points. The release wheel carries the infera-router binary and the +# infera.engine.sglang worker wrapper, so the master's router version is the Infera version. +# Only nats-py is missing from the lmsysorg ROCm image; --no-deps keeps every package the +# image ships (torch, sglang, transformers, ...) at its pinned version. +set -eo pipefail +source /infmax-workspace/benchmarks/check_env.sh +check_env_vars ROUTER_VERSION +pip_install=(python3 -m pip install) +if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_install+=(--break-system-packages) +fi +"${pip_install[@]}" --quiet --no-cache-dir --no-deps \ + "nats-py==2.16.0" \ + "amd-infera==${ROUTER_VERSION}" \ + --find-links "https://github.com/AMD-AGI/Infera/releases/expanded_assets/v${ROUTER_VERSION}" +python3 -c "import shutil, sys, infera.engine.sglang; sys.exit(shutil.which('infera-router') is None)" diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index e7da9b72bb..55b960fe5b 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -28,6 +28,7 @@ "dynamo-vllm": "vllm", "llmd-vllm": "vllm", "dynamo-sglang": "sglang", + "infera-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", "atom": "atom", diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index e5eb375757..04d4c002c4 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -80,6 +80,7 @@ FABRIC_REFERENCE = "@fabric." INSTALLERS = { "glm5.3-tilert-rocm.sh": ("router", "tilert-pd-router"), + "infera-sglang.sh": ("router", "infera-router"), "kimik3-b300-mooncake.sh": ("kv-offload-backend", "mooncake"), "lmcache-mp-rocm.sh": ("kv-offload-backend", "lmcache"), "vllm-mooncake.sh": ("kv-offload-backend", "mooncake"), diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py index bc83f6fb9e..4704ce1026 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py @@ -62,6 +62,8 @@ def test_an_installer_fails_without_the_version_the_binder_writes(tmp_path, scri ("lmcache-mp-rocm.sh", {"KV_OFFLOAD_BACKEND_VERSION": "9.8.7"}, {"lmcache==9.8.7", "https://github.com/LMCache/LMCache/releases/expanded_assets/v9.8.7-rocm"}), ("glm5.3-tilert-rocm.sh", {"ROUTER_VERSION": "9.8.7", "TILERT_ROLE": "prefill"}, {"tilert==9.8.7"}), + ("infera-sglang.sh", {"ROUTER_VERSION": "9.8.7"}, + {"--no-deps", "amd-infera==9.8.7", "https://github.com/AMD-AGI/Infera/releases/expanded_assets/v9.8.7"}), ]) # fmt: skip def test_an_installer_installs_exactly_the_bound_version(tmp_path, script, env, expected): result, installs = run(tmp_path, script, env) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index e687fc922c..bb57db8a2b 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -294,6 +294,16 @@ def test_kimi_curve_requires_an_explicit_supported_sampler( "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", }, ), + ( + "infera-sglang", + {"speculative-algorithm": "EAGLE", "speculative-num-steps": 3}, + {}, + { + "SGLANG_SIMULATE_ACC_LEN": "2.4", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + }, + ), ( "trt", { From c956034812a37234d73dfd8cfdd35f485d7aa17d Mon Sep 17 00:00:00 2001 From: liyingli Date: Wed, 28 Oct 2026 09:09:32 +0000 Subject: [PATCH 02/12] feat(glm5.2): add the MI355X infera-sglang AgentX P/D curve glm5.2-fp4-mi355x-infera-sglang-agentic-disagg: stock SGLang nightly behind the Infera router (frontend.type: infera, carried as runners/srt-slurm/patches/630-infera-frontend.patch until the pin includes NVIDIA/srt-slurm#630). Prefill TP1 x PP4 with write-back HiCache at 250 GB per stage, decode TP + DP attention, EAGLE 5/1/6, Mooncake KV. Shapes: 1P1D with an 8-GPU decode (conc 32-88), 1P1D with a 4-GPU decode (64, 80), 2P1D (176). Signed-off-by: liyingli Co-authored-by: Cursor --- .../mi355x-fp4/agentx/disagg-variants.yaml | 166 +++++++++ inferencex-e2e/configs/amd-master.yaml | 75 ++++ inferencex-e2e/perf-changelog.yaml | 9 + .../patches/630-infera-frontend.patch | 329 ++++++++++++++++++ .../runners/srt-slurm/patches/README.md | 1 + 5 files changed, 580 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml create mode 100644 inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..71e701f64c --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,166 @@ +# GLM-5.2-MXFP4 AgentX on MI355X: stock SGLang P/D behind the Infera router. +# +# Every worker is `python3 -m infera.engine.sglang`, which starts SGLang's own server with +# the flags below and registers it in the job's etcd; `infera-router` discovers the +# workers there (srt-slurm `frontend.type: infera`) and routes kv-aware, with prefill +# session affinity and decode input demand. infera-sglang.sh installs Infera at the +# master config's router version. +# +# Prefill: TP1 x PP4 on four GPUs, 20/20/20/18 layers (the last stage also holds the LM +# head and the NextN layer), overlap scheduler off as PP requires, 8k chunks, +# shortest-prefill-first, write-back HiCache at 250 GB per stage. Prefill carries the +# decode's EAGLE flags so the KV transfer includes the draft layer. Decode: TP + DP +# attention over its GPUs, EAGLE MTP 5/1/6. KV moves over Mooncake on the eight rails. +schema: 2 +base: + name: mi355x-glm52-infera-sglang-agentx-disagg + slurm: + time_limit: "08:00:00" + resources: + gpu_type: mi355x + gpus_per_node: 8 + setup_script: infera-sglang.sh + frontend: + type: infera + enable_multiple_frontends: false + args: + router-policy: kv-aware + kv-prefill-overlap-weight: 20 + kv-decode-overlap-weight: 2 + session-affinity: prefill + pd-prefill-guard-release: completion + decode-input-demand: "on" + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + kv_events: true + env: + <<: &environment + PYTHONUNBUFFERED: "1" + PYTHONHASHSEED: "0" + # srtctl's pre-download sees only HF_HOME, so the node-local cache is mounted at $HF_HOME/hub. + HF_HOME: /mnt/hf_home + HF_HUB_CACHE: /mnt/hf_home/hub + GLOO_SOCKET_IFNAME: "@fabric.socket-ifname" + NCCL_SOCKET_IFNAME: "@fabric.socket-ifname" + NCCL_IB_DISABLE: "1" + NCCL_IGNORE_CPU_AFFINITY: "1" + HSA_NO_SCRATCH_RECLAIM: "1" + HIP_FORCE_DEV_KERNARG: "1" + AITER_USE_FLYDSL_MOE_SORTING: "1" + SAFETENSORS_FAST_GPU: "1" + SGLANG_DP_USE_GATHERV: "1" + SGLANG_OPT_USE_TOPK_V2: "true" + SGLANG_OPT_USE_JIT_KERNEL_GROUPED_TOPK: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + INFERA_ENGINE_READY_TIMEOUT: "3600" + # Same-host HIP IPC cannot reach a peer node; cross-node KV goes over RDMA. + MC_DISABLE_HIP: "1" + MC_GID_INDEX: "@fabric.rdma-gid-index" + # Rail-isolated fabric: each transfer uses the NIC of the same name on both ends. + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # ionic rejects a 2 GiB dma-buf registration that starts 1 GiB-aligned. + MC_MAX_MR_SIZE: "1073741824" + RDMAV_FORK_SAFE: "1" + SGLANG_HICACHE_HOST_REGISTER_CHUNK_GB: "512" + SGLANG_PP_LAYER_PARTITION: "20,20,20,18" + args: + <<: &server + served-model-name: amd/GLM-5.2-MXFP4 + trust-remote-code: true + disaggregation-transfer-backend: mooncake + disaggregation-ib-device: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + ep-size: 1 + # AITER custom all-reduce garbles decode output under DP attention with CUDA graphs. + disable-custom-all-reduce: true + kv-cache-dtype: fp8_e4m3 + dsa-prefill-backend: triton + dsa-decode-backend: triton + max-running-requests: 256 + json-model-override-args: '{"index_share_for_mtp_iteration":false}' + enable-fused-qk-norm-rope: true + enable-aiter-allreduce-fusion: true + reasoning-parser: glm45 + tool-call-parser: glm47 + enable-metrics: true + enable-cache-report: true + watchdog-timeout: 3600 + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + tp-size: 1 + pp-size: 4 + disable-overlap-schedule: true + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.88 + schedule-policy: shortest-prefill-first + enable-hierarchical-cache: true + hicache-size: 250 + hicache-ratio: 1.5 + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: *environment + args: + <<: *server + tp-size: 8 + dp-size: 8 + enable-dp-attention: true + chunked-prefill-size: 32768 + mem-fraction-static: 0.92 + cuda-graph-max-bs-decode: 256 + tokenizer-worker-num: 4 + sbatch_directives: + cpus-per-task: "128" + mem: "0" + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + extra_mount: + # The node-local hub cache the single-node GLM-5.2 points already fill. + - /var/lib/hf-hub-cache:/mnt/hf_home/hub + health_check: + # Cold start: weight load, the 1 TB pinned HiCache pool per prefill and graph capture. + max_attempts: 1080 + interval_seconds: 5 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # infera-router keys session affinity on X-Session-ID. + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: "1" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + +# One PP4 prefill and an 8-GPU decode (12 GPUs). Decode balances requests held per rank: +# under DP attention a step waits for the busiest rank. +override_1p1d: + frontend: + args: + decode-request-weight: 1000000 + +# One PP4 prefill and a 4-GPU decode (8 GPUs). KV-bound, so token-only decode demand. +override_1p1d_d4: + roles: + decode: + gpus: 4 + args: + tp-size: 4 + dp-size: 4 + +# Two PP4 prefills on one node and an 8-GPU decode (16 GPUs). +override_2p1d: + roles: + prefill: + workers: 2 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ed18bec2d6..39464565a6 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1459,6 +1459,81 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12, 14, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } +# GLM-5.2 MXFP4 AgentX on MI355X: stock SGLang prefill/decode disaggregation behind the +# Infera router (infera-sglang). Workers start through infera.engine.sglang and register in +# etcd; the router routes kv-aware with prefill session affinity and decode input demand. +# Prefill is TP1 x PP4 on four GPUs with write-back HiCache (250 GB per stage); decode is +# TP + DP attention; EAGLE MTP 5/1/6 on both roles; KV over Mooncake. Three shapes: +# (1) 1P1D, 8-GPU decode (12 GPUs), decode request weight 1,000,000, conc [32, 48, 64, 88]; +# (2) 1P1D, 4-GPU decode (8 GPUs), conc [64, 80]; +# (3) 2P1D, two prefills on one node and an 8-GPU decode (16 GPUs), conc [176]. +glm5.2-fp4-mi355x-infera-sglang-agentic-disagg: + image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018@sha256:8f7d8034546b17cc4ae57c63f3f65c584eb73e62378243d1416b8385aa7be286 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: infera-sglang + router: { name: infera-router, version: "0.3.2" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: glm5.2/infera-sglang/mi355x-fp4/agentx + scenarios: + agentic-coding: + # A PP4 prefill pins 4 x 250 GB of KV plus the DSA indexer's host pool, about 1.23 TB + # of the 4-GPU share. + - dram-utilization: 0.85 + search-space: + - spec-decoding: mtp + conc-list: [32, 48, 64, 88] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d + prefill: + num-worker: 1 + tp: 1 + pp: 4 + ep: 1 + dp-attn: false + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + - spec-decoding: mtp + conc-list: [64, 80] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_1p1d_d4 + prefill: + num-worker: 1 + tp: 1 + pp: 4 + ep: 1 + dp-attn: false + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + - spec-decoding: mtp + conc-list: [176] + kv-offloading: dram + kv-offload-backend: { name: hicache } + srt-recipe: disagg-variants.yaml:override_2p1d + prefill: + num-worker: 2 + tp: 1 + pp: 4 + ep: 1 + dp-attn: false + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + # GLM-5.2 FP8 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding: the FP8 precision sibling of glm5.2-fp4-mi355x-sglang-agentic-mtp, # serving the upstream zai-org/GLM-5.2-FP8 checkpoint (~756 GB block-FP8). diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 18e8e72c5c..dc3de08b1b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11556,3 +11556,12 @@ - "Disagg: 6 DEP2 prefill workers with a DEP8 decode at conc 280-360 or a DEP16 decode at conc 300; DeepGEMM MegaMoE, NIXL transfer plus a shared MooncakeStore DRAM pool (160 GB per rank), Dynamo KV router." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4468 scenario-type: [agentic-coding] + +- config-keys: + - glm5.2-fp4-mi355x-infera-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 MXFP4 AgentX on MI355X with stock SGLang (lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018) prefill/decode disaggregation behind the Infera router (framework infera-sglang, infera-router 0.3.2 installed by infera-sglang.sh). Workers start through infera.engine.sglang and register in etcd; the router routes kv-aware with prefill session affinity and decode input demand." + - "Prefill TP1 x PP4 (layers 20/20/20/18) with write-back HiCache at 250 GB per stage; decode TP + DP attention; EAGLE MTP 5/1/6 on both roles; Mooncake KV transfer. Shapes: 1P1D with an 8-GPU decode at concurrency 32, 48, 64, 88 (decode request weight 1,000,000); 1P1D with a 4-GPU decode at 64, 80; 2P1D at 176." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch new file mode 100644 index 0000000000..7dbb8ab8cb --- /dev/null +++ b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch @@ -0,0 +1,329 @@ +diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py +index a143f022..f6b92ed3 100644 +--- a/src/srtctl/backends/sglang.py ++++ b/src/srtctl/backends/sglang.py +@@ -355,6 +355,7 @@ class SGLangBackend(Backend): + # Direct frontends run the native server; Dynamo frontends run the registering worker. + use_sglang = frontend.worker_launch == "direct" + python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" ++ launcher = frontend.direct_worker_launcher(self, process, runtime) if use_sglang else None + + # Get served model name from config + served_model_name = self.get_served_model_name(runtime.model_path.name) +@@ -364,11 +365,9 @@ class SGLangBackend(Backend): + # Start with nsys prefix if provided + cmd: list[str] = list(nsys_prefix) if nsys_prefix else [] + ++ cmd.extend(launcher or ["python3", "-m", python_module]) + cmd.extend( + [ +- "python3", +- "-m", +- python_module, + "--model-path", + model_arg, + "--served-model-name", +diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py +index 611e15b4..cb0639d7 100644 +--- a/src/srtctl/frontends/__init__.py ++++ b/src/srtctl/frontends/__init__.py +@@ -10,6 +10,7 @@ importing it from here is what makes ``frontend.type: `` resolvable. + + Supported frontend types: + - dynamo: Dynamo frontend with NATS/etcd communication ++- infera: Infera router discovering self-registering SGLang workers through etcd + - llm-d: llm-d Endpoint Picker behind Envoy, with the P/D sidecar on decode workers + - sglang: Direct sglang.launch_server for a single aggregate worker (no router) + - sglang-router: SGLang Model Gateway router in front of static workers +@@ -30,6 +31,7 @@ from srtctl.frontends.base import ( + ) + from srtctl.frontends.dynamic_frontend import DynamicFrontend + from srtctl.frontends.dynamo import DynamoFrontend ++from srtctl.frontends.infera import InferaFrontend + from srtctl.frontends.llm_d import LLMDFrontend + from srtctl.frontends.sglang import SGLangRouterFrontend + from srtctl.frontends.sglang_direct import SGLangFrontend +@@ -45,6 +47,7 @@ __all__ = [ + "DynamicFrontend", + "DynamoFrontend", + "Frontend", ++ "InferaFrontend", + "LLMDFrontend", + "SGLangFrontend", + "SGLangRouterFrontend", +diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py +index c552fbe2..bb725a3f 100644 +--- a/src/srtctl/frontends/base.py ++++ b/src/srtctl/frontends/base.py +@@ -191,6 +191,16 @@ class Frontend(ABC): + """ + return frozenset() + ++ def direct_worker_launcher(self, backend: Any, process: "Process", runtime: "RuntimeContext") -> list[str] | None: ++ """Argv that starts ``backend``'s direct worker in place of its server module, or ``None``. ++ ++ A launcher runs the engine's own OpenAI server on the allocated port and ++ registers it with this frontend's discovery plane (Infera's ++ ``infera.engine.sglang``). The backend appends the engine flags it ++ would pass to its own server module, unchanged. ++ """ ++ return None ++ + def kv_events_subscriber( + self, process: "Process", runtime: "RuntimeContext", model_name: str + ) -> tuple[str, int, str] | None: +diff --git a/src/srtctl/frontends/infera.py b/src/srtctl/frontends/infera.py +new file mode 100644 +index 00000000..a04d789d +--- /dev/null ++++ b/src/srtctl/frontends/infera.py +@@ -0,0 +1,249 @@ ++# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. ++# SPDX-License-Identifier: Apache-2.0 ++ ++"""Infera's router in front of SGLang workers that register themselves in etcd. ++ ++Each worker is ``python3 -m infera.engine.sglang``: it starts SGLang's own ++server (``sglang.launch_server``) on the allocated HTTP port with the flags the ++SGLang backend builds, then registers the worker's URL, role, bootstrap address, ++DP size and KV-events endpoint under an etcd prefix. ``infera-router`` (the ++Rust data plane, started through ``python3 -m infera.server --router-backend ++rust``) watches that prefix and routes over HTTP, reading KV events straight ++from SGLang's ZMQ publishers. The worker is therefore a direct worker (the ++engine's own server); the frontend is dynamic because the router learns its ++workers from etcd. ++""" ++ ++from __future__ import annotations ++ ++import logging ++import shlex ++import threading ++from typing import TYPE_CHECKING, Any, ClassVar, Literal ++ ++from srtctl.core.health import WorkerHealthResult ++from srtctl.core.slurm import get_hostname_ip, start_srun_process ++from srtctl.frontends.base import Frontend, numactl_prefix, register_frontend ++from srtctl.frontends.dynamic_frontend import DynamicFrontend ++from srtctl.frontends.static_router import setup_script_preamble ++from srtctl.ports import ETCD_CLIENT_PORT ++from srtctl.services.config import ServiceConfig ++from srtctl.services.implicit import ETCD_SERVICE_NAME, EffectiveService, infra_placement ++ ++if TYPE_CHECKING: ++ from srtctl.backends.base import WorkerMode ++ from srtctl.core.processes import ManagedProcess ++ from srtctl.core.runtime import RuntimeContext ++ from srtctl.core.topology import Process ++ ++logger = logging.getLogger(__name__) ++ ++# Both sides speak HTTP and ZMQ only, so no NATS service is implied. ++ROUTER_MANAGED_ARGS = frozenset( ++ { ++ "router-backend", ++ "host", ++ "port", ++ "discovery-backend", ++ "etcd-endpoint", ++ "request-transport", ++ "kv-event-transport", ++ } ++) ++_ROLE_BY_DISAGG_MODE = {"prefill": "prefill", "decode": "decode", "mixed": "decode"} ++# Infera's registering launcher for each engine this frontend pairs with. ++WORKER_LAUNCHERS = {"sglang": "infera.engine.sglang"} ++ ++ ++def etcd_endpoint(runtime: RuntimeContext) -> str: ++ """The job's etcd on the infra node; workers and the router must name the same one.""" ++ return f"http://{runtime.infra_node_ip}:{ETCD_CLIENT_PORT}" ++ ++ ++@register_frontend("infera") ++class InferaFrontend(DynamicFrontend): ++ """``infera-router`` discovering ``infera.engine.sglang`` workers through etcd.""" ++ ++ type: ClassVar[str] = "infera" ++ required_backend: ClassVar[str | None] = "sglang" ++ # The worker is SGLang's own OpenAI server; the launcher only adds registration. ++ worker_launch: ClassVar[Literal["dynamo", "direct"]] = "direct" ++ ++ @property ++ def health_endpoint(self) -> str: ++ """The router lists every registered worker with its role and status here.""" ++ return "/v1/workers" ++ ++ def validate(self, config: Any) -> None: ++ clashes = sorted(ROUTER_MANAGED_ARGS & set(config.frontend.args or {})) ++ if clashes: ++ raise ValueError( ++ f"frontend.args {clashes} are set by srtctl for frontend.type: infera " ++ "(etcd discovery, HTTP requests, ZMQ KV events); remove them" ++ ) ++ for service in config.services or (): ++ if service.name == ETCD_SERVICE_NAME and service.enabled and service.external: ++ raise ValueError( ++ "frontend.type: infera registers workers in the job's own etcd; " ++ "an external etcd service is not supported" ++ ) ++ ++ def worker_metrics_path(self, backend: Any, mode: WorkerMode) -> str | None: ++ """Workers are SGLang's own servers, so metrics are the engine's, not a runtime's.""" ++ return Frontend.worker_metrics_path(self, backend, mode) ++ ++ def worker_metrics_port(self, process: Process, runtime: RuntimeContext) -> int | None: ++ """The leader rank's SGLang HTTP server carries /metrics; followers serve nothing.""" ++ if process.is_leader and process.http_port > 0: ++ return process.http_port ++ return None ++ ++ def worker_endpoint_port(self, process: Process, config: Any, runtime: RuntimeContext) -> int | None: ++ if process.is_leader and process.http_port > 0: ++ return process.http_port ++ return None ++ ++ def profiling_control_port(self, process: Process, config: Any, runtime: RuntimeContext) -> int | None: ++ return process.http_port if process.http_port > 0 else None ++ ++ def worker_ready_port(self, process: Process) -> int: ++ return process.http_port ++ ++ def implied_services(self, config: Any) -> list[EffectiveService]: ++ """etcd for worker registration; requests and KV events do not ride on NATS.""" ++ return [ ++ EffectiveService( ++ ServiceConfig(name=ETCD_SERVICE_NAME, type="etcd", placement=infra_placement(config)), ++ implicit=True, ++ reason="frontend.type infera", ++ ) ++ ] ++ ++ def direct_worker_launcher(self, backend: Any, process: Process, runtime: RuntimeContext) -> list[str] | None: ++ """Start the engine through Infera's launcher so the worker registers itself in etcd. ++ ++ The KV-events port is the allocator's: when the role publishes KV events ++ the backend passes ``--kv-events-config`` with it, and the launcher ++ advertises that endpoint. Infera's own KV probe plane, which the router ++ does not read, stays off so the worker opens no unallocated listener. ++ """ ++ command = [ ++ "python3", ++ "-m", ++ WORKER_LAUNCHERS[backend.type], ++ "--discovery-backend", ++ "etcd", ++ "--etcd-endpoint", ++ etcd_endpoint(runtime), ++ "--advertise-host", ++ get_hostname_ip(process.node, runtime.network_interface), ++ "--request-transport", ++ "http", ++ "--kv-event-transport", ++ "zmq", ++ "--kv-events", ++ "off", ++ ] ++ if process.kv_events_port is None: ++ command.append("--no-enable-kv-events") ++ return command ++ ++ def parse_health(self, response_json: dict, expected_prefill: int, expected_decode: int) -> WorkerHealthResult: ++ """Count active registrations by role; an aggregate (``mixed``) worker counts as decode.""" ++ counts = {"prefill": 0, "decode": 0} ++ for worker in response_json.get("workers") or []: ++ role = _ROLE_BY_DISAGG_MODE.get(str(worker.get("disagg_mode", "mixed"))) ++ if role is not None and worker.get("status", "active") == "active": ++ counts[role] += 1 ++ ready = counts["prefill"] >= expected_prefill and counts["decode"] >= expected_decode ++ return WorkerHealthResult( ++ ready=ready, ++ message=( ++ f"infera-router: {counts['prefill']}/{expected_prefill} prefill, " ++ f"{counts['decode']}/{expected_decode} decode registered" ++ ), ++ prefill_ready=counts["prefill"], ++ prefill_expected=expected_prefill, ++ decode_ready=counts["decode"], ++ decode_expected=expected_decode, ++ ) ++ ++ def build_router_command(self, port: int, runtime: RuntimeContext, config: Any) -> list[str]: ++ """Infera's launcher execs the Rust data plane after resolving a Hugging Face id to a local tokenizer. ++ ++ kv-aware routing hashes prompts with the served model's tokenizer, so it ++ defaults to the checkpoint the workers load. ++ """ ++ args = dict(config.frontend.args or {}) ++ args.setdefault("router-tokenizer-path", runtime.worker_model_arg) ++ return [ ++ "python3", ++ "-m", ++ "infera.server", ++ "--router-backend", ++ "rust", ++ "--host", ++ "0.0.0.0", ++ "--port", ++ str(port), ++ "--discovery-backend", ++ "etcd", ++ "--etcd-endpoint", ++ etcd_endpoint(runtime), ++ "--request-transport", ++ "http", ++ "--kv-event-transport", ++ "zmq", ++ *self.get_frontend_args_list(args), ++ ] ++ ++ def start_process(self, **kwargs: Any) -> Any: ++ """Launch one router process; split out for tests.""" ++ return start_srun_process(**kwargs) ++ ++ def start_frontends( ++ self, ++ topology: Any, ++ runtime: RuntimeContext, ++ config: Any, ++ backend: Any, ++ backend_processes: list[Process], ++ stop_event: threading.Event | None = None, # unused: returns immediately ++ ) -> list[ManagedProcess]: ++ from srtctl.core.processes import FRONTEND_TERMINATE_TIMEOUT_SECONDS, ManagedProcess ++ ++ processes: list[ManagedProcess] = [] ++ for idx, node in enumerate(topology.frontend_nodes): ++ log_file = runtime.log_dir / f"{node}_infera_router_{idx}.out" ++ cmd = numactl_prefix(config) + self.build_router_command(topology.frontend_port, runtime, config) ++ logger.info("Starting infera router %d on %s: %s", idx, node, shlex.join(cmd)) ++ ++ env = dict(runtime.environment) ++ env.update(config.frontend.env or {}) ++ step_name = f"infera_router_{idx}" ++ proc = self.start_process( ++ command=cmd, ++ nodelist=[node], ++ output=str(log_file), ++ container_image=config.frontend.container_image or str(runtime.container_image), ++ reuse_container=runtime.reuse_container, ++ container_prepare_timeout_seconds=runtime.container_prepare_timeout_seconds, ++ container_mounts=runtime.container_mounts, ++ env_to_set=env or None, ++ bash_preamble=setup_script_preamble(config), ++ het_group=runtime.nodes.het_group_for(node), ++ step_name=step_name, ++ srun_options=runtime.srun_options, ++ ) ++ processes.append( ++ ManagedProcess( ++ name=step_name, ++ popen=proc, ++ log_file=log_file, ++ node=node, ++ critical=True, ++ terminate_timeout=FRONTEND_TERMINATE_TIMEOUT_SECONDS, ++ step_name=step_name, ++ ) ++ ) ++ return processes diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index d9052d1aa8..a3e5478810 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,3 +9,4 @@ Each patch backports an upstream fix that the shared submodule pin does not incl | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exclusive node's exporter (DCGM or AMD) reports every GPU, which the collector otherwise flags as `unexpected_device`. The pin includes #573, but upstream merged a different design: it keeps every row and excuses idle worker-node GPUs only after a DCGM worker GPU UUID probe. [`infx/results/power/multinode.py`](../../../infx/results/power/multinode.py) does not apply that excusal yet, so keep this filter until it does. | +| `630-infera-frontend.patch` | [NVIDIA/srt-slurm#630](https://github.com/NVIDIA/srt-slurm/pull/630) (draft) | `frontend.type: infera`: SGLang workers start through Infera's `infera.engine.sglang`, register in the job's etcd and are discovered by `infera-router` (the `infera-sglang` framework). Adds `Frontend.direct_worker_launcher`; the patch carries only `src/srtctl` (frontend, registry, hook and the SGLang backend's use of it). | From 996ea04997242f4e163d62177c4eb8fb6e7d1fb7 Mon Sep 17 00:00:00 2001 From: liyingli Date: Thu, 29 Oct 2026 14:14:18 +0000 Subject: [PATCH 03/12] feat(infera-sglang): apply the SGLang fixes the pinned image lacks, with waiver infera-sglang.sh now runs infera-sglang-patches/apply.sh after installing Infera: B1 (sglang#33059), B2 (#46830), B6 (#46820), B7 (#47184), B8 (issue #46829), each anchored, idempotent and marker-checked; a fix that does not land fails the job. docs/waiver/4629.md covers them. The router check resolves the binary the launcher will exec (the wheel's bundled infera-router) instead of looking on PATH. The recipe registers VRAM the peer way (MOONCAKE_DISABLE_HIP_DMABUF=1, as ATOM P/D on this cluster) and mounts the host's Ionic verbs provider (as Kimi-K3 vLLM). Signed-off-by: liyingli --- .../configs/infera-sglang-patches/apply.sh | 59 +++ .../infera-sglang-patches/dsa_dp_sync.diff | 51 +++ .../patch_deepseek_v2_pp_embed_dim.py | 55 +++ .../patch_dsa_indexer_hip_dp_padded_rows.py | 351 ++++++++++++++++++ .../patch_kv_metrics_send_noblock.py | 70 ++++ .../patch_mla_absorb_bmm_default_config.py | 68 ++++ .../configs/infera-sglang.sh | 17 +- .../mi355x-fp4/agentx/disagg-variants.yaml | 6 +- inferencex-e2e/docs/waiver/4629.md | 57 +++ .../tests/srt_slurm/test_setup_scripts.py | 13 +- 10 files changed, 740 insertions(+), 7 deletions(-) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py create mode 100644 inferencex-e2e/docs/waiver/4629.md diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh new file mode 100644 index 0000000000..32ce4b651c --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# SGLang fixes the infera-sglang GLM-5.2 points need on the pinned +# lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018 image, applied in the container +# before the server starts. Waiver: docs/waiver/4629.md. Each fix is marker-gated and +# idempotent, and the run fails if any of them did not land. +# +# B1 patch_dsa_indexer_hip_dp_padded_rows.py sglang#33059 aiter indexer rows under DP padding +# B2 dsa_dp_sync.diff sglang#46830 DP-rank D2H sync deadlock +# B6 patch_kv_metrics_send_noblock.py sglang#46820 scheduler blocks on the KvMetrics PUSH +# B7 patch_deepseek_v2_pp_embed_dim.py sglang#47184 PP middle stage has no embedding +# B8 patch_mla_absorb_bmm_default_config.py sglang#46829 absorb-BMM tile config aborts Triton +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SGLANG_DIR="${SGLANG_DIR:-/sgl-workspace/sglang}" +SRT="$SGLANG_DIR/python/sglang/srt" +git_sgl() { git -c safe.directory='*' -C "$SGLANG_DIR" "$@"; } + +echo "[infera-sglang-patches] B1 dsa_indexer DP-padded rows" +python3 "$here/patch_dsa_indexer_hip_dp_padded_rows.py" + +echo "[infera-sglang-patches] B2 dsa_dp_sync.diff" +if git_sgl apply --check "$here/dsa_dp_sync.diff" 2>/dev/null; then + git_sgl apply "$here/dsa_dp_sync.diff" + echo " applied" +elif git_sgl apply --check -R "$here/dsa_dp_sync.diff" 2>/dev/null; then + echo " already applied" +else + git_sgl apply --check "$here/dsa_dp_sync.diff" + exit 1 +fi + +for patch in patch_kv_metrics_send_noblock.py patch_deepseek_v2_pp_embed_dim.py \ + patch_mla_absorb_bmm_default_config.py; do + echo "[infera-sglang-patches] $patch" + python3 "$here/$patch" +done + +# source file (under sglang/srt) : text only the patched source contains +markers=( + "layers/attention/dsa/dsa_indexer.py:_p1v2_trim" + "layers/attention/dsa_backend.py:must not be None for DRAFT_EXTEND_V2" + "managers/scheduler_components/kv_events_publisher.py:_infera_kv_metrics_noblock" + "models/deepseek_v2.py:_infera_pp_embed_dim" + "models/deepseek_common/attention_forward_methods/forward_mla_rocm.py:_infera_absorb_bmm_default" +) +fail=0 +for spec in "${markers[@]}"; do + file="$SRT/${spec%%:*}" marker="${spec#*:}" + rm -f "$(dirname "$file")/__pycache__/$(basename "$file" .py)".*.pyc + if grep -qF -- "$marker" "$file"; then + echo " ok ${spec%%:*}" + else + echo " MISSING ${spec%%:*} :: $marker" >&2 + fail=1 + fi +done +[[ "$fail" == 0 ]] || { echo "[infera-sglang-patches] verification failed" >&2; exit 1; } +echo "[infera-sglang-patches] all five applied" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff new file mode 100644 index 0000000000..6953005512 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff @@ -0,0 +1,51 @@ +diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py +index 37fcbe40421d..16f7e97d7d42 100644 +--- a/python/sglang/srt/layers/attention/dsa_backend.py ++++ b/python/sglang/srt/layers/attention/dsa_backend.py +@@ -791,7 +791,15 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): + # needs_cpu_seq_lens=False nulls the host mirror for spec-v2 relay + # batches; graph replay uses the static page-table width, so only this + # eager (e.g. over-capture-bs) fallback needs a length here. +- max_seqlen_k = int(forward_batch.seq_lens.max().item()) + draft_token_num ++ # ++ # Use the static width rather than seq_lens.max().item(): that is a ++ # device-to-host sync, and this branch is entered per rank (an idle ++ # DP peer keeps its host mirror and takes the arm above), so under ++ # DP attention the ranks stop issuing the same collective sequence ++ # and the group deadlocks. Over-allocating columns is safe -- the ++ # page table is only indexed through top-k, which masks per row by ++ # cache_seqlens, so the extra columns are never selected. ++ max_seqlen_k = self.req_to_token.shape[1] + # [b, max_seqlen_k] + page_table = self.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, :max_seqlen_k +@@ -851,19 +859,19 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): + page_table, repeats=self.speculative_num_draft_tokens, dim=0 + ) + elif forward_batch.forward_mode.is_draft_extend_v2(): +- if forward_batch.extend_prefix_lens_cpu is None: +- assert forward_batch.extend_prefix_lens is not None +- forward_batch.extend_prefix_lens_cpu = ( +- forward_batch.extend_prefix_lens.cpu().tolist() +- ) +- if forward_batch.seq_lens_cpu is None: +- forward_batch.seq_lens_cpu = forward_batch.seq_lens.cpu() +- forward_batch.seq_lens_sum = int(forward_batch.seq_lens_cpu.sum()) ++ # No host mirrors are materialized here. Both would be D2H syncs on ++ # this same per-rank branch, so with only the max_seqlen_k fix above ++ # the DP group still deadlocks. Neither is read on this path: ++ # extend_prefix_lens_cpu and seq_lens_sum are consumed only inside ++ # the is_extend() arm below, and DRAFT_EXTEND_V2 is not is_extend() ++ # (include_draft_extend_v2 defaults to False). What this branch does ++ # need -- extend_seq_lens_cpu -- is already supplied by ++ # eagle_worker_common.prepare_for_draft_extend, on the same ++ # gpu_only branch and for this same reason. + assert ( + forward_batch.extend_seq_lens_cpu is not None + and forward_batch.extend_seq_lens is not None +- and forward_batch.extend_prefix_lens_cpu is not None +- ), "All of them must not be None" ++ ), "extend_seq_lens{,_cpu} must not be None for DRAFT_EXTEND_V2" + + extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu + assert forward_batch.extend_seq_lens is not None diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py new file mode 100644 index 0000000000..934fc24c54 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Let a middle pipeline stage build DeepseekV2Model (models/deepseek_v2.py). + +WHAT: the gemm-output allocator sizing in `DeepseekV2Model.__init__` reads +`self.embed_tokens.embedding_dim` twice. Only the first pipeline stage (and the +last one when it carries an MTP draft layer) holds a real embedding; every other +stage has a `PPMissingLayer`, so `--pp-size` > 2 dies at load time with +AttributeError on PP rank 1. The embedding width is the hidden size, so both +reads become `config.hidden_size`; behaviour on PP1 and on every stage that has +an embedding is unchanged. + +Needed by the GLM-5.2 PD example's TP1 x PP4 prefill. Not upstream as of v0.5.22. + +Self-locating and idempotent; exits non-zero when the anchor drifted. +Marker: `_infera_pp_embed_dim`. +""" + +import importlib.util +import py_compile +import sys +from pathlib import Path + +TAG = "[deepseek-v2-pp-embed-dim]" +MARKER = "_infera_pp_embed_dim" +OLD = "self.embed_tokens.embedding_dim" +NEW = "config.hidden_size" +FIRST_OLD = f" and {OLD} == 7168\n" +FIRST_NEW = f" and {NEW} == 7168 # {MARKER}: PP middle stages have no embedding\n" + + +def main() -> int: + spec = importlib.util.find_spec("sglang") + if not spec or not spec.origin: + print(f"{TAG} cannot locate sglang") + return 1 + path = Path(spec.origin).parent / "srt" / "models" / "deepseek_v2.py" + source = path.read_text() + if MARKER in source: + print(f"{TAG} already present — skipping") + return 0 + if source.count(OLD) != 2 or source.count(FIRST_OLD) != 1: + print(f"{TAG} anchor not found as expected in {path} — re-cut the patch") + return 1 + path.write_text(source.replace(FIRST_OLD, FIRST_NEW, 1).replace(OLD, NEW)) + for cached in (path.parent / "__pycache__").glob(path.stem + ".*.pyc"): + cached.unlink() + py_compile.compile(str(path), doraise=True) + print(f"{TAG} patched {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py new file mode 100644 index 0000000000..df626f1859 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""PATCH 01 -- dsa_indexer HIP/aiter DP-padded rows. + +WHAT: the aiter (HIP) paged-MQA branch sizes its `logits` output from the +DP-PADDED row count while `lengths` is sized to the REAL count, so top-k dies +with "Expected lengths.size(0) == B to be true, but got false". Reconcile the +two counts to `min(real, padded)`: trim q/weights when q_fp8 is the longer side +(the contract every CUDA backend already honours) and restore the padding after +`topk_transform`, or clip the lengths when q_fp8 is the SHORTER side. See +"THE ROW COUNT DIVERGES BOTH WAYS" below. + +WHY: without it, DP-attention + EAGLE MTP for GLM-5.2 DSA crashes as soon as +more than one request is in flight -- on gfx950 at the very first batch, and on +gfx942 at conc > 1. CUDA was never affected: those backends take `q_offset=` and +slice internally. ROCm-specific. + +THE ROW COUNT DIVERGES BOTH WAYS (`GLM52_P1V3`) + An earlier revision of this fix guarded one direction only, `real < padded`, + i.e. it assumed DP padding always makes `q_fp8` LONGER. On a DP-attention IDLE + rank under MTP draft-extend the inequality inverts. Captured live with this + patch's own SGLANG_DEBUG_DSA_ROWS=1: + + mode=IDLE q_fp8=(1,32,128) q_offset=2 ntnp=0 agree=False lengths=(2,) + -> mqa_q=(1,32,128) + + `q_offset` (= sum of dsa_extend_len_cpu) is 2 while only 1 row is materialized + in q_fp8, so `2 < 1` is false, no trim runs, aiter sizes its logits from + q_fp8.shape[0] = 1, and fast_topk_v2 gets 1 score row against 2 lengths + entries -- killing the scheduler rank and dropping the router to + `active_workers: 1`. A trim cannot fix this direction: there are FEWER query + rows than lengths entries, so there is nothing to cut. The LENGTHS side is + clipped instead, through `topk_transform`'s existing `ke_offset` parameter. + + The two directions, and what each does: + + real < padded the #32762 case: q_fp8 carries DP padding, so TRIM q/weights + down to the real count, then re-pad after topk_transform. + Trim at _p1v2_real == 0 too -- a DP-IDLE rank has no requests, + so the real count is 0 while q_fp8 still carries padding. + q_fp8[:0] is a legal empty slice and is what an idle rank + should pass; the CUDA path likewise slices unconditionally. + DO NOT reintroduce a `0 < q_offset` lower bound here: an + earlier revision had one and it made fast_topk_v2 assert on + exactly those idle ranks. + real > padded the P1V3 case above: nothing to trim, so CLIP the lengths. + + BOUND BEFORE THE BRANCH. `_p1v2_clip` and `_p1v2_rows` are bound before the + `is_aiter()` test, not inside it. `_p1v2_clip` is read unconditionally at the + topk_transform call, so a branch-local binding would be a NameError on any + non-aiter backend; `_p1v2_rows` defaults to `_p1v2_padded`, making the restore + a no-op if it is ever reached without the branch having run. aiter is + unconditional on ROCm so neither has fired in practice, but a build-time patch + should not depend on that. + + This half is what makes the patch survive an AGENTIC workload. The bug needs + MTP AND DP-attention AND an idle rank at once: an 8-round fixed-shape sweep + ran MTP + DPA for 660 requests without hitting it, because + `--random-range-ratio 1.0` keeps batch shapes homogeneous and ranks rarely go + idle mid-flight. A breathing session population produces ragged batches + constantly, and the one-directional revision crashed the decode leg roughly + 13 minutes into an agentic benchmark. + + Reproduced twice under that workload (125 s and 766 s into two independent + runs). With the fix: 0 occurrences of `Expected lengths.size` across two full + ~4,000 s windows on two different clusters, 0 scheduler exceptions, + `active_workers: 2` throughout. Second-hand for the reader: it was validated + as a runtime script before being folded in here, so this is a re-shaping of an + already-measured fix rather than a fresh one. Evidence in the internal + reproduction kit -- ask the patch author. + +WHY A SCRIPT AND NOT A DIFF. Patches 02 and 04 are `--fuzz=0` context diffs +pinned to one base. This fix is needed on every engine base we ship, and +`dsa_indexer.py` drifts across all of them: v0.5.16/gfx942, v0.5.18/glm53 and +the v0.5.19 gfx950 nightly. Anchoring on source text instead of line numbers +covers all three from one source of truth. + + ANCHORS 1 AND 3 ARE DELIBERATELY LOOSE, so do not "tidy" them back onto the + adjacent code. Anchor 1 sits on the `contextlib`/`logging` import head rather + than the typing line, because v0.5.19 dropped `List` from that line. Anchor 3 + starts at `if self.paged_mqa_logits_backend.is_aiter():` rather than the + `weights.squeeze(2)` above it, because v0.5.19 inserted a nested helper + between the two. Both forms verified unique on v0.5.16, v0.5.18 and v0.5.19. + + NOTE on the P1V3 anchor. It is the bare `topk_transform(logits, + self.index_topk)` call. The complete patch was applied and bytecode-verified + against both release tags on 2026-09-02. If either source drifts, the + uniqueness check below writes NOTHING and exits 1, which is the intended + failure: a half-applied fix crashes in the same place the unpatched tree does. + +UPSTREAM STATUS (queried with `gh` on 2026-08-01; re-queried and each PR state +re-read 2026-08-03) + issue NONE, for either half. Searched sgl-project/sglang for + "Expected lengths.size", "DSA padding DP attention" and the + GLM-5.2 MTP crash text (P1V2); and for "DSA idle rank MTP draft + extend", "q_offset dsa_extend_len", "fast_topk_v2 lengths" + (P1V3): no match either time. Weak evidence -- `gh search` + matches titles and bodies, not diff content, so an upstream + change to this same function that never names the symptom would + not surface. The anchor-collision PRs below were found only by + searching for the FUNCTION name, not the symptom. + third-party PR #32762 "[NPU] Fix DSA eager padding mismatch in PD MTP warm-up" + (stellaxcpeng, OPEN). SAME BUG CLASS on a different platform, + and this patch is written in its shape: one boolean gates both + trim and restore, and the returned row count is asserted before + the padding is re-attached. It does not touch the aiter branch, + so it does not fix HIP. + #31683 "[ROCm][MI35X] Enable GLM-5.2-MXFP4 MTP" (long10024070, + OPEN). ADJACENT, NOT THE SAME FIX -- same function, but it + widens forward_cuda's guard from `seq_lens.numel() == 0` to + `is_idle() or ...`. Different symptom; verified against its diff. + #30378 / #30427 (both MERGED, in both bases) clamp padded-row + seq_lens VALUES in triton_ops/pad.py. This fixes the HIP-side + row COUNT -- a different defect in the same area. + anchor risk Two OPEN PRs rewrite the code this patch anchors on. Neither + fixes this defect; both would make the anchors drift, which + fails the build loudly rather than mis-applying. + #32738 "[Fix] DSA Indexer: pad heads for DeepGEMM paged MQA + logits on decode/target-verify" -- edits _get_topk_paged at the + SAME two aiter call sites, replacing `q_fp8` with a + `q_fp8_padded` from a new `_pad_heads_for_deep_gemm`. Head-dim + padding, orthogonal to our row-count fix, but a direct textual + collision with the P1V2 anchor. + #31480 "[DSA] Add an arch-independent torch paged-MQA-logits + backend" (updated 2026-08-03) -- extracts the backend into + paged_mqa_logits_backend.py, i.e. restructures the + `is_aiter()` dispatch this whole patch hangs off. + own PR #33059 "Fix DSA indexer aiter (HIP) padding mismatch under + DP-attention" (dorado269, OPEN, REVIEW_REQUIRED, against main, + 1 file +16/-4). CI is red only at `pr-gate`, which blocks on a + missing `run-ci` label, not on code. + +THIS PATCH vs OUR OWN PR #33059. Same defect, overlapping edits. They differ +deliberately: + base here: v0.5.16/gfx942 and v0.5.18/gfx950, with every edit verified + against both release tags. #33059: sglang `main`, whose + `_get_topk_paged` has drifted further (cutedsl / dg_native branches, + `_mask_init_and_local_tokens`). + scope #33059 carries the `real < padded` half ONLY. The P1V3 clip is NOT + in it and has not been filed upstream at all -- it should be, and + this repo already ships the instrumentation that proves the bug. + form here: explicit `_p1v2_rows` / `_p1v2_trim` / `_p1v2_clip` locals + with a post-kernel row-count ASSERT, following #32762's shape. + #33059: the minimal `q_fp8[:q_offset]` / `weights[:q_offset]` slice + and a plain `if q_offset < q_fp8.shape[0]` restore. + local-only Stripped for upstream: the `GLM52_P1V2` / `GLM52_P1V3` markers (the + build script greps `_p1v2_trim` and `_p1v2_rows` out of the + BYTECODE to prove the patch landed), the `_p1v2_*` naming, and the + `SGLANG_DEBUG_DSA_ROWS` logging block. All three are scaffolding + for this repo's verification loop and carry no upstream value. + trade-off The upstream form is smaller and easier to review. Ours fails + LOUDER: on a shape drift the assert crashes instead of silently + returning a short tensor -- and it handles a direction #33059 does + not. Do NOT delete this script when #33059 merges; the clip half + would go with it. + validation Ours: 4/4 probe, conc=64 1k/1k 256/256, 2540/2540 in the PD set, + plus two full ~4,000 s agentic windows on two gfx950 clusters for + the P1V3 half. #33059's port to `main` was NOT re-run on hardware. + +RELATION TO THE INDEXSHARE WORKAROUND. NOT substituted. +`--json-model-override-args '{"index_share_for_mtp_iteration":false}'` removes +the rank divergence that draft_cuda_graph_dp_vote and dsa_page_table_rows +address, but this is an independent bug -- present regardless of IndexShare, and +the arm that validated the override had THIS patch applied. + +Self-locating and idempotent. All edits or none: an anchor that is missing or no +longer unique writes NOTHING and fails (exit 1), because a half-applied fix +crashes in the same place the unpatched tree does. +""" + +import importlib.util +import sys +from pathlib import Path + +_TAG = "[dsa-indexer-rows]" + +_REL = "layers/attention/dsa/dsa_indexer.py" + +# (anchor, anchor + our edit). Each anchor must occur exactly once; the +# replacement doubles as the already-applied marker. +_EDITS: list[tuple[str, str]] = [ + # `os` for the debug-logging env switch below. + ( + """from __future__ import annotations + +import contextlib +import logging +""", + """from __future__ import annotations + +import contextlib +import logging +import os +""", + ), + # Anchored on the two platform flags rather than the whole block: v0.5.16 + # adds `_is_xpu` on the line after `_is_npu`. + ( + """_is_cuda = is_cuda() +_is_hip = is_hip() +""", + """_is_cuda = is_cuda() +_is_hip = is_hip() +# Set SGLANG_DEBUG_DSA_ROWS=1 to log the indexer row bookkeeping (padded vs real). +_DSA_DEBUG_ROWS = os.environ.get("SGLANG_DEBUG_DSA_ROWS", "0") == "1" + +""", + ), + # Trim q/weights to the real row count on the aiter branch only. + ( + """ if self.paged_mqa_logits_backend.is_aiter(): + logits = aiter_paged_mqa_logits( + q_fp8, + kv_cache_fp8, + weights, +""", + """ # GLM52_P1V2/P1V3: bound BEFORE the branch, not inside it -- _p1v2_clip is + # read unconditionally below, so a branch-local binding is a NameError on + # any non-aiter backend. See "BOUND BEFORE THE BRANCH" in the header. + _p1v2_trim = False + _p1v2_clip = False + _p1v2_real = q_offset + _p1v2_padded = q_fp8.shape[0] + _p1v2_rows = _p1v2_padded + if self.paged_mqa_logits_backend.is_aiter(): + # WHY aiter sizes logits from q_fp8.shape[0], which disagrees with + # `lengths` BOTH ways. HOW reconcile to min(): trim when longer, clip + # when shorter. Cases: "DIVERGES BOTH WAYS" in the header. + _p1v2_rows = min(_p1v2_real, _p1v2_padded) + _p1v2_trim = _p1v2_rows < _p1v2_padded + _p1v2_clip = _p1v2_rows < _p1v2_real + _q_mqa, _w_mqa = q_fp8, weights + if _p1v2_trim: + _q_mqa = q_fp8[:_p1v2_rows] + _w_mqa = weights[:_p1v2_rows] + if _DSA_DEBUG_ROWS: + # Cross-check against the padding bookkeeping #32762 keys off. + # NOT an assert: it does NOT agree with q_offset on the MTP + # draft-extend path -- that disagreement is the P1V3 case above, + # and this log is how it was caught. + _p1v2_ntnp = getattr(forward_batch, "_original_num_tokens", None) + if _p1v2_ntnp is None: + _p1v2_ntnp = forward_batch.num_token_non_padded_cpu + logger.info( + "[dsa-rows] mode=%s q_fp8=%s q_offset=%s ntnp=%s agree=%s " + "lengths=%s -> mqa_q=%s", + forward_batch.forward_mode, + tuple(q_fp8.shape), + _p1v2_real, + _p1v2_ntnp, + _p1v2_ntnp == _p1v2_real, + tuple(metadata.get_seqlens_expanded().shape), + tuple(_q_mqa.shape), + ) + logits = aiter_paged_mqa_logits( + _q_mqa, + kv_cache_fp8, + _w_mqa, +""", + ), + # GLM52_P1V3: clip the lengths when the rows were clipped rather than trimmed. + # Anchored on the bare call, which occurs once; the `if` wrapper is the + # already-applied marker. + ( + """ topk_result = metadata.topk_transform(logits, self.index_topk) +""", + """ # GLM52_P1V3: rows were CLIPPED (real > padded), so the lengths this + # indexes must be clipped to match. Via topk_transform's existing + # `ke_offset` -- an extension point that only overrides seq_lens_topk -- + # NOT by mutating shared, possibly graph-captured metadata. + if _p1v2_clip: + topk_result = metadata.topk_transform( + logits, + self.index_topk, + ke_offset=metadata.get_seqlens_expanded()[:_p1v2_rows], + ) + else: + topk_result = metadata.topk_transform(logits, self.index_topk) +""", + ), + # Restore the trimmed padding, gated on the SAME boolean. + ( + """ # Restore possible padding exist in the hidden states. + if not _is_hip and q_offset < q_fp8.shape[0]: + pad_len = q_fp8.shape[0] - q_offset + padding = torch.full( + (pad_len, topk_result.shape[1]), +""", + """ # GLM52_P1V2: restore the trimmed padding -- gate on the SAME boolean and + # assert the row count. BEFORE, a re-derived condition skipped the restore + # on a shape drift, returning a short tensor instead of crashing. + # + # Keyed off _p1v2_rows, not _p1v2_real: under P1V3's clip case the two + # differ, and it is _p1v2_rows that the kernel actually ran over. + if _p1v2_trim: + assert topk_result.shape[0] == _p1v2_rows, ( + "GLM52_P1V3: paged-MQA top-k returned " + f"{topk_result.shape[0]} rows for {_p1v2_rows} trimmed query " + f"rows (real={_p1v2_real}, padded={_p1v2_padded}); refusing to " + "pad a mis-shaped result." + ) + padding = torch.full( + (_p1v2_padded - _p1v2_rows, topk_result.shape[1]), +""", + ), +] + + +def _srt_dir(): + spec = importlib.util.find_spec("sglang") + if not spec or not spec.origin: + return None + d = Path(spec.origin).parent / "srt" + return d if d.is_dir() else None + + +def main(): + srt = _srt_dir() + if srt is None: + print(f"{_TAG} sglang not importable — skipping") + return 0 + + f = srt / _REL + if not f.is_file(): + print(f"{_TAG} {f} is missing — sglang layout changed, re-anchor the patch") + return 1 + + src = out = f.read_text() + for old, new in _EDITS: + if new in out: + continue # this edit is already in the tree + found = out.count(old) + if found != 1: + where = "absent" if found == 0 else f"{found}x ambiguous" + print(f"{_TAG} anchor {where} in {_REL}: {old.splitlines()[0]!r}") + print(f"{_TAG} sglang drifted — re-cut the patch, nothing written") + return 1 + out = out.replace(old, new, 1) + + if out == src: + print(f"{_TAG} already present — skipping") + return 0 + f.write_text(out) + print(f"{_TAG} patched {f}") + print(f"{_TAG} aiter paged-MQA now sees the real row count under DP-attention") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py new file mode 100644 index 0000000000..f7fd6f4c75 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Never block the scheduler on the auxiliary KV-metrics PUSH socket. + +WHAT: with KV events enabled, `KvEventsPublisher` also PUSHes a `KvMetrics` +record every scheduler step to an endpoint that nothing binds in an Infera +deployment (the router reads KV events and /metrics, not this socket). Once the +ZMQ send queue fills, `sock_send` blocks and the scheduler stops: a GLM-5.2 +prefill wedged this way (yihou agentx44_20261009, R001, py-spy 2026-10-09). + +FIX: send with zmq.NOBLOCK and drop the record on zmq.Again. KV events and the +Prometheus metrics are unaffected. The v0.5.21 deployments carried this as a +bind-mounted file; v0.5.22 still sends blocking. + +Self-locating and idempotent; exits non-zero when the anchor drifted. +Marker: `_infera_kv_metrics_noblock`. +""" + +import importlib.util +import py_compile +import sys +from pathlib import Path + +TAG = "[kv-metrics-noblock]" +MARKER = "_infera_kv_metrics_noblock" +OLD = ( + " if not self.send_metrics_from_scheduler.closed:\n" + " sock_send(self.send_metrics_from_scheduler, kv_metrics)\n" +) +NEW = ( + " if not self.send_metrics_from_scheduler.closed:\n" + " # Infera: nothing may bind this PUSH peer; a blocking send would\n" + " # stall the scheduler once the queue fills, so drop instead.\n" + " try:\n" + " sock_send(self.send_metrics_from_scheduler, kv_metrics, flags=zmq.NOBLOCK)\n" + f" except zmq.Again: # {MARKER}\n" + " pass\n" +) + + +def main() -> int: + spec = importlib.util.find_spec("sglang") + if not spec or not spec.origin: + print(f"{TAG} cannot locate sglang") + return 1 + path = ( + Path(spec.origin).parent + / "srt" + / "managers" + / "scheduler_components" + / "kv_events_publisher.py" + ) + source = path.read_text() + if MARKER in source: + print(f"{TAG} already present — skipping") + return 0 + if source.count(OLD) != 1 or "\nimport zmq\n" not in source: + print(f"{TAG} anchor not found exactly once in {path} — re-cut the patch") + return 1 + path.write_text(source.replace(OLD, NEW, 1)) + for cached in (path.parent / "__pycache__").glob(path.stem + ".*.pyc"): + cached.unlink() + py_compile.compile(str(path), doraise=True) + print(f"{TAG} patched {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py new file mode 100644 index 0000000000..633b2b1513 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Let aiter choose the MLA absorb-BMM config on ROCm (forward_mla_rocm.py). + +WHAT: v0.5.21 added `_absorb_bmm_config`, a hand-built Triton tile config that +`forward_mla_rocm.py` passes to aiter's batched FP8 GEMM for up to 256 rows. On a +GLM-5.2 prefill rank it aborted Triton's LLVM backend (Sequence.h:275 +`iota_range` assertion) and took the engine down (yihou agentx44_20261009, R003). +The v0.5.21 deployments ran with the function returning None, i.e. with aiter +picking its own config as on v0.5.19; this script bakes that in instead of the +bind-mounted file those runs used. + +v0.5.22 still carries the same function. The crash was not re-tested there; the +patch is kept so the image matches the v0.5.21 baseline it is compared against. + +Self-locating and idempotent; exits non-zero when the anchor drifted. +Marker: `_infera_absorb_bmm_default`. +""" + +import importlib.util +import py_compile +import sys +from pathlib import Path + +TAG = "[mla-absorb-bmm-default]" +MARKER = "_infera_absorb_bmm_default" +OLD = ( + "def _absorb_bmm_config(heads: int, m: int, n: int) -> Optional[dict]:\n" + ' """Return decode tile settings, or None outside the decode row limit."""\n' +) +NEW = OLD + ( + " # Infera: aiter picks its own config; the hand-built one below aborted\n" + " # Triton's LLVM backend on a GLM-5.2 prefill rank (v0.5.21).\n" + f" return None # {MARKER}\n" +) + + +def main() -> int: + spec = importlib.util.find_spec("sglang") + if not spec or not spec.origin: + print(f"{TAG} cannot locate sglang") + return 1 + path = ( + Path(spec.origin).parent + / "srt" + / "models" + / "deepseek_common" + / "attention_forward_methods" + / "forward_mla_rocm.py" + ) + source = path.read_text() + if MARKER in source: + print(f"{TAG} already present — skipping") + return 0 + if source.count(OLD) != 1: + print(f"{TAG} anchor not found exactly once in {path} — re-cut the patch") + return 1 + path.write_text(source.replace(OLD, NEW, 1)) + for cached in (path.parent / "__pycache__").glob(path.stem + ".*.pyc"): + cached.unlink() + py_compile.compile(str(path), doraise=True) + print(f"{TAG} patched {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh index b5b42fe82c..45cca1b70b 100755 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash # Install Infera, at the master config's router version, into the stock SGLang image for the -# infera-sglang points. The release wheel carries the infera-router binary and the -# infera.engine.sglang worker wrapper, so the master's router version is the Infera version. -# Only nats-py is missing from the lmsysorg ROCm image; --no-deps keeps every package the -# image ships (torch, sglang, transformers, ...) at its pinned version. +# infera-sglang points, then apply the SGLang fixes the pinned image still lacks +# (infera-sglang-patches/apply.sh, waiver docs/waiver/4629.md). The Linux x86_64 release +# wheel bundles the infera-router binary next to the infera.engine.sglang worker wrapper, so +# the master's router version is the Infera version. Only nats-py is missing from the +# lmsysorg ROCm image; --no-deps keeps every package the image ships (torch, sglang, +# transformers, ...) at its pinned version. set -eo pipefail source /infmax-workspace/benchmarks/check_env.sh check_env_vars ROUTER_VERSION @@ -15,4 +17,9 @@ fi "nats-py==2.16.0" \ "amd-infera==${ROUTER_VERSION}" \ --find-links "https://github.com/AMD-AGI/Infera/releases/expanded_assets/v${ROUTER_VERSION}" -python3 -c "import shutil, sys, infera.engine.sglang; sys.exit(shutil.which('infera-router') is None)" +# The router the launcher will exec: the wheel's bundled binary, not one on PATH. +python3 -c "import infera.engine.sglang; from infera.server.launch_rust import _find_binary; _find_binary()" +# INFERA_SGLANG_PATCHES=0 only for the installer unit test, which has no SGLang tree. +if [[ "${INFERA_SGLANG_PATCHES:-1}" != 0 ]]; then + bash "$(dirname "${BASH_SOURCE[0]}")/infera-sglang-patches/apply.sh" +fi diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml index 71e701f64c..1ebcc1518c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -64,7 +64,9 @@ base: MC_GID_INDEX: "@fabric.rdma-gid-index" # Rail-isolated fabric: each transfer uses the NIC of the same name on both ends. MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # ionic rejects a 2 GiB dma-buf registration that starts 1 GiB-aligned. + # Register VRAM the peer way, not through dma-buf, as ATOM P/D does on this cluster + # with the same Mooncake (78566d45); 1 GiB regions keep every registration small. + MOONCAKE_DISABLE_HIP_DMABUF: "1" MC_MAX_MR_SIZE: "1073741824" RDMAV_FORK_SAFE: "1" SGLANG_HICACHE_HOST_REGISTER_CHUNK_GB: "512" @@ -131,6 +133,8 @@ base: extra_mount: # The node-local hub cache the single-node GLM-5.2 points already fill. - /var/lib/hf-hub-cache:/mnt/hf_home/hub + # Match the host's Ionic provider without replacing the image's verbs core. + - /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro health_check: # Cold start: weight load, the 1 TB pinned HiCache pool per prefill and graph capture. max_attempts: 1080 diff --git a/inferencex-e2e/docs/waiver/4629.md b/inferencex-e2e/docs/waiver/4629.md new file mode 100644 index 0000000000..c0c7e96aa5 --- /dev/null +++ b/inferencex-e2e/docs/waiver/4629.md @@ -0,0 +1,57 @@ +# Inference-engine patch waiver — PR #4629 + +Filed per [`docs/PR_REVIEW_CHECKLIST.md`](../PR_REVIEW_CHECKLIST.md) (image-provenance / patch item): the +setup script of the `infera-sglang` points patches the pinned SGLang image before serving, which the +checklist prohibits unless covered by a filled-out waiver. + +## Config covered + +- **Master config entry:** `glm5.2-fp4-mi355x-infera-sglang-agentic-disagg` in + [`configs/amd-master.yaml`](../../configs/amd-master.yaml) (all three P/D shapes: 1P1D, 1P1D with a + 4-GPU decode, 2P1D) +- **Pinned image:** `lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018@sha256:8f7d8034546b17cc4ae57c63f3f65c584eb73e62378243d1416b8385aa7be286` + (SGLang source `e9b22a08`) +- **Patch entrypoint:** the recipe's `setup_script: infera-sglang.sh` + ([`benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh`](../../benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh)), + which srtctl runs in every worker and router container; after installing Infera it runs +- **Patch script:** [`benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh`](../../benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh) + +## What is patched + +`apply.sh` edits the image's SGLang source in place (`/sgl-workspace/sglang`, the tree the image's +`sglang` package is installed from). Every fix is anchored on the exact source it replaces, is +idempotent, and is checked by a marker afterwards; a fix that does not apply fails the job instead +of serving unpatched. + +| | Upstream PR/issue | What it changes | Why this config needs it | +|---|---|---|---| +| B1 | sglang [#33059](https://github.com/sgl-project/sglang/pull/33059) | DSA indexer on HIP: trim the DP-padded query rows before the aiter top-k kernel and restore them after (`layers/attention/dsa/dsa_indexer.py`) | DP-attention decode. On this source the aiter branch passes the padded `q_fp8` whole (`dsa_indexer.py:1366-1376`; the XPU branch slices to `[:q_offset]`) and the restore at `:1438` excludes HIP | +| B2 | sglang [#46830](https://github.com/sgl-project/sglang/pull/46830) | DSA backend: no device-to-host sync on per-rank branches (`layers/attention/dsa_backend.py`) | DP-attention decode with MTP. `seq_lens.max().item()` (`:1102`) and two `.cpu()` calls (`:1172-1179`) sit on branches that only some DP ranks take, so the ranks deadlock in a collective | +| B6 | sglang [#46820](https://github.com/sgl-project/sglang/pull/46820) | Send the per-step `KvMetrics` record with `zmq.NOBLOCK` and drop it when the queue is full (`managers/scheduler_components/kv_events_publisher.py`) | Prefill runs with KV events (the router routes on them). The record is PUSHed blocking (`:89-90`) to an endpoint nothing in SGLang binds; once the queue fills, the scheduler stops | +| B7 | sglang [#47184](https://github.com/sgl-project/sglang/pull/47184) (rebase of [#21194](https://github.com/sgl-project/sglang/pull/21194)) | Size the gemm-output allocator from `config.hidden_size` instead of `self.embed_tokens.embedding_dim` (`models/deepseek_v2.py`) | TP1 × PP4 prefill. A middle pipeline stage holds a `PPMissingLayer` (`:2923`), so the reads at `:2987, :3022` raise `AttributeError` at load | +| B8 | sglang issue [#46829](https://github.com/sgl-project/sglang/issues/46829) | Drop the hand-written Triton tile config of the MLA absorb BMM and use Triton's default (`models/deepseek_common/attention_forward_methods/forward_mla_rocm.py`) | GLM-5.2 prefill. `_absorb_bmm_config` (`:156`) trips an LLVM assertion in Triton 3.7.0 on gfx950, the Triton this image ships; reproduced on 10-27 | + +Not patched, by reading the same source: the DSA page-table rows (the recipe's +`index_share_for_mtp_iteration: false` keeps page-table and top-k rows equal), the non-final-chunk KV +wait event (only the overlap-scheduler path differs; PP prefill runs `--disable-overlap-schedule`), +and HiCache staged write-back on ROCm (the controller moves backup indices per pool on this source). + +Nothing else in the image changes: Infera is installed with `--no-deps` at the master's router +version, and `infera.engine.sglang` starts SGLang's own `launch_server` unmodified. + +## Why the unmodified upstream image cannot run this benchmark + +- B7: the TP1 × PP4 prefill does not load — PP rank 1 dies with + `AttributeError: 'PPMissingLayer' object has no attribute 'embedding_dim'`. +- B1, B2: the DP-attention decode either fails in the indexer (`Expected lengths.size(0) == B`) or + hangs with every DP rank stopped, as soon as ranks carry unequal batches. +- B6: under the AgentX load the prefill scheduler blocks on the metrics socket (py-spy capture of a + wedged GLM-5.2 prefill, 2026-10-09) and the point fails on stalled requests. +- B8: the first long prefill aborts in Triton's LLVM backend. + +## Removal plan + +Each fix goes when a pinned ROCm SGLang nightly contains it: drop its file and its line in +`apply.sh` in the PR that bumps the `image:` of `glm5.2-fp4-mi355x-infera-sglang-agentic-disagg`. +When the last one is gone, delete `infera-sglang-patches/`, the patch step in `infera-sglang.sh` and +this waiver in the same PR. diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py index 4704ce1026..c4178ee575 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py @@ -62,7 +62,7 @@ def test_an_installer_fails_without_the_version_the_binder_writes(tmp_path, scri ("lmcache-mp-rocm.sh", {"KV_OFFLOAD_BACKEND_VERSION": "9.8.7"}, {"lmcache==9.8.7", "https://github.com/LMCache/LMCache/releases/expanded_assets/v9.8.7-rocm"}), ("glm5.3-tilert-rocm.sh", {"ROUTER_VERSION": "9.8.7", "TILERT_ROLE": "prefill"}, {"tilert==9.8.7"}), - ("infera-sglang.sh", {"ROUTER_VERSION": "9.8.7"}, + ("infera-sglang.sh", {"ROUTER_VERSION": "9.8.7", "INFERA_SGLANG_PATCHES": "0"}, {"--no-deps", "amd-infera==9.8.7", "https://github.com/AMD-AGI/Infera/releases/expanded_assets/v9.8.7"}), ]) # fmt: skip def test_an_installer_installs_exactly_the_bound_version(tmp_path, script, env, expected): @@ -70,3 +70,14 @@ def test_an_installer_installs_exactly_the_bound_version(tmp_path, script, env, assert result.returncode == 0, result.stdout + result.stderr assert expected <= set(installs[0]) + + +def test_infera_sglang_fails_when_the_sglang_fixes_do_not_apply(tmp_path): + """The installer patches SGLang after installing Infera and never skips it silently: + without an SGLang tree the patch step, and so the setup script, fails.""" + result, installs = run(tmp_path, "infera-sglang.sh", + {"ROUTER_VERSION": "9.8.7", "SGLANG_DIR": str(tmp_path / "no-sglang")}) + + assert result.returncode != 0 + assert "[infera-sglang-patches] B1" in result.stdout + assert "amd-infera==9.8.7" in installs[0] From a89865143acda6176b31bdfd2052623aa92ce9f9 Mon Sep 17 00:00:00 2001 From: liyingli Date: Fri, 30 Oct 2026 05:58:07 +0000 Subject: [PATCH 04/12] fix(srt-slurm patch): backport the infera frontend to the pinned srtctl srt-slurm ae3d6877's start_srun_process takes neither reuse_container nor container_prepare_timeout_seconds (#630 is written against a newer main), so the router never started: AttributeError on RuntimeContext. Regenerated against ae3d6877, the router call matches the pin's static routers. Signed-off-by: liyingli --- .../srt-slurm/patches/630-infera-frontend.patch | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch index 7dbb8ab8cb..6f04bee13f 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch +++ b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch @@ -1,8 +1,8 @@ diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py -index a143f022..f6b92ed3 100644 +index a0c5b392..1c81284a 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py -@@ -355,6 +355,7 @@ class SGLangBackend(Backend): +@@ -343,6 +343,7 @@ class SGLangBackend(Backend): # Direct frontends run the native server; Dynamo frontends run the registering worker. use_sglang = frontend.worker_launch == "direct" python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" @@ -10,7 +10,7 @@ index a143f022..f6b92ed3 100644 # Get served model name from config served_model_name = self.get_served_model_name(runtime.model_path.name) -@@ -364,11 +365,9 @@ class SGLangBackend(Backend): +@@ -352,11 +353,9 @@ class SGLangBackend(Backend): # Start with nsys prefix if provided cmd: list[str] = list(nsys_prefix) if nsys_prefix else [] @@ -74,10 +74,10 @@ index c552fbe2..bb725a3f 100644 ) -> tuple[str, int, str] | None: diff --git a/src/srtctl/frontends/infera.py b/src/srtctl/frontends/infera.py new file mode 100644 -index 00000000..a04d789d +index 00000000..4d0fdd81 --- /dev/null +++ b/src/srtctl/frontends/infera.py -@@ -0,0 +1,249 @@ +@@ -0,0 +1,247 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + @@ -306,8 +306,6 @@ index 00000000..a04d789d + nodelist=[node], + output=str(log_file), + container_image=config.frontend.container_image or str(runtime.container_image), -+ reuse_container=runtime.reuse_container, -+ container_prepare_timeout_seconds=runtime.container_prepare_timeout_seconds, + container_mounts=runtime.container_mounts, + env_to_set=env or None, + bash_preamble=setup_script_preamble(config), From 4a400018e54e8b37d5d4ca81292c7d2752d22bcf Mon Sep 17 00:00:00 2001 From: liyingli Date: Fri, 30 Oct 2026 07:20:48 +0000 Subject: [PATCH 05/12] revert(srt-slurm patch): drop the ae3d6877 backport now that main pins 59920ea1 This reverts commit a8986514. main moved the srt-slurm submodule to v2.65.2 (59920ea1, #4601), the base NVIDIA/srt-slurm#630 is written against, where start_srun_process takes reuse_container and container_prepare_timeout_seconds. 630-infera-frontend.patch is again #630's source change; it applies to 59920ea1 after 573-participating-gpus.patch. Signed-off-by: liyingli --- .../srt-slurm/patches/630-infera-frontend.patch | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch index 6f04bee13f..7dbb8ab8cb 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch +++ b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch @@ -1,8 +1,8 @@ diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py -index a0c5b392..1c81284a 100644 +index a143f022..f6b92ed3 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py -@@ -343,6 +343,7 @@ class SGLangBackend(Backend): +@@ -355,6 +355,7 @@ class SGLangBackend(Backend): # Direct frontends run the native server; Dynamo frontends run the registering worker. use_sglang = frontend.worker_launch == "direct" python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" @@ -10,7 +10,7 @@ index a0c5b392..1c81284a 100644 # Get served model name from config served_model_name = self.get_served_model_name(runtime.model_path.name) -@@ -352,11 +353,9 @@ class SGLangBackend(Backend): +@@ -364,11 +365,9 @@ class SGLangBackend(Backend): # Start with nsys prefix if provided cmd: list[str] = list(nsys_prefix) if nsys_prefix else [] @@ -74,10 +74,10 @@ index c552fbe2..bb725a3f 100644 ) -> tuple[str, int, str] | None: diff --git a/src/srtctl/frontends/infera.py b/src/srtctl/frontends/infera.py new file mode 100644 -index 00000000..4d0fdd81 +index 00000000..a04d789d --- /dev/null +++ b/src/srtctl/frontends/infera.py -@@ -0,0 +1,247 @@ +@@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + @@ -306,6 +306,8 @@ index 00000000..4d0fdd81 + nodelist=[node], + output=str(log_file), + container_image=config.frontend.container_image or str(runtime.container_image), ++ reuse_container=runtime.reuse_container, ++ container_prepare_timeout_seconds=runtime.container_prepare_timeout_seconds, + container_mounts=runtime.container_mounts, + env_to_set=env or None, + bash_preamble=setup_script_preamble(config), From 2c1fcb7d5a9e119dbcd09aa6a67d88a18da46713 Mon Sep 17 00:00:00 2001 From: liyingli Date: Tue, 3 Nov 2026 03:01:31 +0000 Subject: [PATCH 06/12] feat(infera-sglang): install amd-infera from PyPI amd-infera 0.3.3 is on PyPI with the same Linux x86_64 wheel as the GitHub release (bundled infera-router). Install it by version from PyPI instead of --find-links on the release page; the bundled-router check stays, so a fallback to the pure-Python wheel still fails at setup. Checked in the pinned 20261018 image and the 20261101 nightly: pip picks the manylinux_2_28 wheel and the router resolves. Signed-off-by: liyingli --- .../srt-slurm-recipes/configs/infera-sglang.sh | 14 +++++++------- .../infx/tests/srt_slurm/test_setup_scripts.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh index 45cca1b70b..af5639c028 100755 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Install Infera, at the master config's router version, into the stock SGLang image for the # infera-sglang points, then apply the SGLang fixes the pinned image still lacks -# (infera-sglang-patches/apply.sh, waiver docs/waiver/4629.md). The Linux x86_64 release -# wheel bundles the infera-router binary next to the infera.engine.sglang worker wrapper, so -# the master's router version is the Infera version. Only nats-py is missing from the -# lmsysorg ROCm image; --no-deps keeps every package the image ships (torch, sglang, +# (infera-sglang-patches/apply.sh, waiver docs/waiver/4629.md). amd-infera's Linux x86_64 +# wheel on PyPI bundles the infera-router binary next to the infera.engine.sglang worker +# wrapper, so the master's router version is the Infera version. Only nats-py is missing from +# the lmsysorg ROCm image; --no-deps keeps every package the image ships (torch, sglang, # transformers, ...) at its pinned version. set -eo pipefail source /infmax-workspace/benchmarks/check_env.sh @@ -15,9 +15,9 @@ if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-package fi "${pip_install[@]}" --quiet --no-cache-dir --no-deps \ "nats-py==2.16.0" \ - "amd-infera==${ROUTER_VERSION}" \ - --find-links "https://github.com/AMD-AGI/Infera/releases/expanded_assets/v${ROUTER_VERSION}" -# The router the launcher will exec: the wheel's bundled binary, not one on PATH. + "amd-infera==${ROUTER_VERSION}" +# The router the launcher will exec: the wheel's bundled binary, not one on PATH. Fails if pip +# fell back to the pure-Python wheel (a platform outside manylinux_2_28 x86_64). python3 -c "import infera.engine.sglang; from infera.server.launch_rust import _find_binary; _find_binary()" # INFERA_SGLANG_PATCHES=0 only for the installer unit test, which has no SGLang tree. if [[ "${INFERA_SGLANG_PATCHES:-1}" != 0 ]]; then diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py index 3b8e618575..9cd7395e8a 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py @@ -61,7 +61,7 @@ def test_an_installer_fails_without_the_version_the_binder_writes(tmp_path, scri {"mooncake-transfer-engine-cuda13==9.8.7"}), ("glm5.3-tilert-rocm.sh", {"ROUTER_VERSION": "9.8.7", "TILERT_ROLE": "prefill"}, {"tilert==9.8.7"}), ("infera-sglang.sh", {"ROUTER_VERSION": "9.8.7", "INFERA_SGLANG_PATCHES": "0"}, - {"--no-deps", "amd-infera==9.8.7", "https://github.com/AMD-AGI/Infera/releases/expanded_assets/v9.8.7"}), + {"--no-deps", "amd-infera==9.8.7"}), ]) # fmt: skip def test_an_installer_installs_exactly_the_bound_version(tmp_path, script, env, expected): result, installs = run(tmp_path, script, env) From 2414dbea96202dbeca554fecd9bb385c055e6b05 Mon Sep 17 00:00:00 2001 From: liyingli Date: Fri, 13 Nov 2026 10:26:11 +0000 Subject: [PATCH 07/12] chore(srt-slurm patch): refresh the infera frontend to NVIDIA/srt-slurm#630 at 72c4d04 The pin is now 2d9079c. The patch is the PR's src/srtctl diff at 72c4d04 and applies in checkout order (573, 630, 900). #630 replaced direct_worker_launcher with worker_module / worker_launcher_args / worker_environment and decides KV events from the role's kv_events, which the recipe already sets on prefill. Co-authored-by: Cursor --- .../patches/630-infera-frontend.patch | 254 +++++++++++++++--- .../runners/srt-slurm/patches/README.md | 2 +- 2 files changed, 222 insertions(+), 34 deletions(-) diff --git a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch index 7dbb8ab8cb..a6a26cca2b 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch +++ b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch @@ -1,20 +1,42 @@ +diff --git a/src/srtctl/backends/base.py b/src/srtctl/backends/base.py +index 625c858..9d9d996 100644 +--- a/src/srtctl/backends/base.py ++++ b/src/srtctl/backends/base.py +@@ -294,6 +294,14 @@ class Backend(ABC): + """ + return {} + ++ def get_kv_events_config_for_mode(self, mode: WorkerMode) -> dict[str, Any] | None: ++ """``roles..kv_events`` for a worker mode over the engine's defaults; None when it publishes none. ++ ++ A worker gets an allocated KV-events port whether or not its role ++ publishes, so this, not ``Process.kv_events_port``, says whether it does. ++ """ ++ return None ++ + def get_mooncake_worker_env( + self, infra_node_ip: str, local_hostname: str, mode: WorkerMode | None = None + ) -> dict[str, str]: diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py -index a143f022..f6b92ed3 100644 +index 4b48b9f..766565b 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py -@@ -355,6 +355,7 @@ class SGLangBackend(Backend): +@@ -357,8 +357,9 @@ class SGLangBackend(Backend): + leader_ip = get_hostname_ip(endpoint_nodes[0], runtime.network_interface) + # Direct frontends run the native server; Dynamo frontends run the registering worker. ++ # A frontend may name a module that wraps the native server (Infera registers it in etcd). use_sglang = frontend.worker_launch == "direct" - python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" -+ launcher = frontend.direct_worker_launcher(self, process, runtime) if use_sglang else None +- python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" ++ python_module = frontend.worker_module(self.type) or ("sglang.launch_server" if use_sglang else "dynamo.sglang") # Get served model name from config served_model_name = self.get_served_model_name(runtime.model_path.name) -@@ -364,11 +365,9 @@ class SGLangBackend(Backend): +@@ -368,11 +369,9 @@ class SGLangBackend(Backend): # Start with nsys prefix if provided cmd: list[str] = list(nsys_prefix) if nsys_prefix else [] -+ cmd.extend(launcher or ["python3", "-m", python_module]) ++ cmd.extend(["python3", "-m", python_module, *frontend.worker_launcher_args(self, process, runtime)]) cmd.extend( [ - "python3", @@ -23,8 +45,56 @@ index a143f022..f6b92ed3 100644 "--model-path", model_arg, "--served-model-name", +diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py +index 80c8fb4..a24ab17 100644 +--- a/src/srtctl/cli/mixins/worker_stage.py ++++ b/src/srtctl/cli/mixins/worker_stage.py +@@ -221,8 +221,9 @@ class WorkerStageMixin: + return {self.runtime.visible_devices_env: process.cuda_visible_devices} + + def _worker_environment_defaults(self, process: "Process") -> dict[str, str]: +- env = {"HEAD_NODE_IP": self.runtime.head_node_ip} +- if get_frontend(self.config.frontend.type).worker_launch == "dynamo": ++ frontend = get_frontend(self.config.frontend.type) ++ env = {"HEAD_NODE_IP": self.runtime.head_node_ip, **frontend.worker_environment(process, self.runtime)} ++ if frontend.worker_launch == "dynamo": + env.update(discovery_env(self.config, self.runtime)) + env.update(lifecycle_otel_env(self.config.services, self.runtime.head_node_ip, process.endpoint_mode)) + env.update( +diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py +index 6a1b0bd..7660755 100755 +--- a/src/srtctl/core/schema.py ++++ b/src/srtctl/core/schema.py +@@ -2629,7 +2629,8 @@ class FrontendConfig: + """Frontend/router configuration. + + Attributes: +- type: Frontend type - "dynamo" (default); "sglang-router" (SGLang Model ++ type: Frontend type - "dynamo" (default) and "infera" (Infera router, ++ SGLang workers register in etcd); "sglang-router" (SGLang Model + Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "llm-d" + (llm-d Endpoint Picker behind Envoy, vLLM), "atomesh", and + "tilert-router" (static routers); "sglang", "vllm", and +diff --git a/src/srtctl/frontends/AGENTS.md b/src/srtctl/frontends/AGENTS.md +index 421d45c..dc4fb00 100644 +--- a/src/srtctl/frontends/AGENTS.md ++++ b/src/srtctl/frontends/AGENTS.md +@@ -4,11 +4,11 @@ Rules for `src/srtctl/frontends/`. The Design Rules in the root `CLAUDE.md` appl + + ## Frontends and readiness + +-The frontend is the process that owns the public OpenAI port (`FRONTEND_PUBLIC_PORT`) and decides when the job is ready. Implementations live under `src/srtctl/frontends/`, register with `@register_frontend("")`, and are imported from the package `__init__`; `frontend.type` resolves through that registry (`get_frontend`, `list_frontend_types`) and nowhere else. Registered today: `dynamo` (etcd/NATS discovery), `sglang-router`, `vllm-router` and `smg` (static URL routers built on `StaticRouterFrontend`; `smg` pairs with any backend), `llm-d` (the llm-d EPP behind Envoy, on `StaticRouterFrontend`: it writes the workers into an endpoints file instead of a command line, and reaches decode workers through the P/D sidecar service it implies), `trtllm_serve` (direct aggregate or the disaggregated orchestrator), `sglang` and `vllm` (one direct worker owns the port); `none` is the services-only job with no implementation and no gate. ++The frontend is the process that owns the public OpenAI port (`FRONTEND_PUBLIC_PORT`) and decides when the job is ready. Implementations live under `src/srtctl/frontends/`, register with `@register_frontend("")`, and are imported from the package `__init__`; `frontend.type` resolves through that registry (`get_frontend`, `list_frontend_types`) and nowhere else. Registered today: `dynamo` (etcd/NATS discovery), `infera` (etcd discovery of SGLang workers started by Infera's registering launcher), `sglang-router`, `vllm-router` and `smg` (static URL routers built on `StaticRouterFrontend`; `smg` pairs with any backend), `llm-d` (the llm-d EPP behind Envoy, on `StaticRouterFrontend`: it writes the workers into an endpoints file instead of a command line, and reaches decode workers through the P/D sidecar service it implies), `trtllm_serve` (direct aggregate or the disaggregated orchestrator), `sglang` and `vllm` (one direct worker owns the port); `none` is the services-only job with no implementation and no gate. + + `Frontend` hooks: `required_backend` and `validate(config)` (recipe rules, run by `SrtConfig._validate_frontend` at load so dry-run catches them; raise `ValueError` with the user-facing message), `health_endpoint` and `parse_health` (readiness), `get_backend_health_urls` (second gate: every advertised worker URL must answer 200 before traffic), `start_frontends` (launch on `topology.frontend_nodes`, one `ManagedProcess` per node with a `step_name`), `get_frontend_args_list` (`frontend.args` to CLI). The schema knows no individual frontend: pairing and per-type rules come from these two members. + +-Two base classes cover the two ways a frontend learns about its workers. `DynamicFrontend` (`frontends/dynamic_frontend.py`) is for frontends whose workers register themselves over a discovery plane: it fronts any engine, needs no per-worker URL gate, and no worker is the public endpoint; Dynamo is its only implementation. A router binary that can also take static URLs (vLLM Router's ZMQ discovery mode) is a mode of a static router, not a dynamic frontend. `StaticRouterFrontend` (`frontends/static_router.py`) is the base for routers that take worker URLs on the command line. Subclasses set `executable`, `pd_flag`, `process_name` and override only what differs: `worker_scheme` (the backend's `direct_worker_scheme`, normally HTTP/gRPC), `worker_bootstrap_port` (the P/D port advertised next to a prefill URL), `resolve_worker_host`, `get_managed_frontend_args` (arguments derived from the allocated topology and runtime paths; a conflicting user value raises instead of being overwritten), `build_bash_preamble`, `build_router_command`, `start_process` (test seam). `collect_workers` asks `Backend.router_worker_address` for the listener node and port. Its default uses a positive `Process.http_port` on the worker node, including vLLM node-local DP pools; a headless engine can instead advertise an allocated listener on the frontend node. Keep `Process.node` physical and use `runtime.shared_model_arg` for gateway model/tokenizer paths even when workers stage weights locally. ++Two base classes cover the two ways a frontend learns about its workers. `DynamicFrontend` (`frontends/dynamic_frontend.py`) is for frontends whose workers register themselves over a discovery plane: it fronts any engine, needs no per-worker URL gate, and no worker is the public endpoint; Dynamo and Infera implement it. Infera's workers are still `direct` (SGLang's own server): `worker_module` names the registering wrapper the backend starts as `python3 -m `, `worker_launcher_args` its flags, and the backend appends the same engine flags as for any direct worker; per-process variables such as the wrapper's readiness port go through `worker_environment`, which the worker stage merges for every frontend beneath the recipe's environment, never as an `env` prefix in the command. A router binary that can also take static URLs (vLLM Router's ZMQ discovery mode) is a mode of a static router, not a dynamic frontend. `StaticRouterFrontend` (`frontends/static_router.py`) is the base for routers that take worker URLs on the command line. Subclasses set `executable`, `pd_flag`, `process_name` and override only what differs: `worker_scheme` (the backend's `direct_worker_scheme`, normally HTTP/gRPC), `worker_bootstrap_port` (the P/D port advertised next to a prefill URL), `resolve_worker_host`, `get_managed_frontend_args` (arguments derived from the allocated topology and runtime paths; a conflicting user value raises instead of being overwritten), `build_bash_preamble`, `build_router_command`, `start_process` (test seam). `collect_workers` asks `Backend.router_worker_address` for the listener node and port. Its default uses a positive `Process.http_port` on the worker node, including vLLM node-local DP pools; a headless engine can instead advertise an allocated listener on the frontend node. Keep `Process.node` physical and use `runtime.shared_model_arg` for gateway model/tokenizer paths even when workers stage weights locally. + + Readiness runs in `BenchmarkStageMixin._wait_for_service_ready`: `wait_for_model` owns timing, abort, and progress logging, and calls the frontend's `probe_ready(host, port, n_prefill, n_decode, config)` once per poll with counts from the frontend's `health_expectations(config, processes)`; then `get_backend_health_urls` are polled with `wait_for_http_endpoints`. A probe raises `requests.RequestException` while the endpoint is down and otherwise returns a `WorkerHealthResult`. `core/health.py` has the three probe shapes to build on: `probe_json_health` (a JSON worker count through `parse_health`), `probe_http_ok` (a bare 200), `probe_direct_server` (`/health` then `/v1/models`). No frontend name appears in `wait_for_model` or `_get_health_expectations`. + diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py -index 611e15b4..cb0639d7 100644 +index 611e15b..cb0639d 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -10,6 +10,7 @@ importing it from here is what makes ``frontend.type: `` resolvable. @@ -52,32 +122,60 @@ index 611e15b4..cb0639d7 100644 "SGLangFrontend", "SGLangRouterFrontend", diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py -index c552fbe2..bb725a3f 100644 +index 124a38c..1b2c879 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py -@@ -191,6 +191,16 @@ class Frontend(ABC): +@@ -195,6 +195,25 @@ class Frontend(ABC): """ return frozenset() -+ def direct_worker_launcher(self, backend: Any, process: "Process", runtime: "RuntimeContext") -> list[str] | None: -+ """Argv that starts ``backend``'s direct worker in place of its server module, or ``None``. ++ def worker_module(self, backend_type: str) -> str | None: ++ """Python module that starts a worker in place of the engine's server module, or ``None``. + -+ A launcher runs the engine's own OpenAI server on the allocated port and ++ The module runs the engine's own OpenAI server on the allocated port and + registers it with this frontend's discovery plane (Infera's -+ ``infera.engine.sglang``). The backend appends the engine flags it -+ would pass to its own server module, unchanged. ++ ``infera.engine.sglang``). The backend still builds ``python3 -m ``, ++ then ``worker_launcher_args``, then the engine flags it would pass to its ++ own server module, unchanged. + """ + return None ++ ++ def worker_launcher_args(self, backend: Any, process: "Process", runtime: "RuntimeContext") -> list[str]: ++ """Arguments of ``worker_module`` for ``process``, ahead of the engine flags; empty without one.""" ++ return [] ++ ++ def worker_environment(self, process: "Process", runtime: "RuntimeContext") -> dict[str, str]: ++ """Per-process environment this frontend's workers need, beneath the recipe's own environment.""" ++ return {} + def kv_events_subscriber( self, process: "Process", runtime: "RuntimeContext", model_name: str ) -> tuple[str, int, str] | None: +diff --git a/src/srtctl/frontends/dynamic_frontend.py b/src/srtctl/frontends/dynamic_frontend.py +index 22e749f..d47697c 100644 +--- a/src/srtctl/frontends/dynamic_frontend.py ++++ b/src/srtctl/frontends/dynamic_frontend.py +@@ -15,10 +15,12 @@ itself the public endpoint. + A router binary that can also take static URLs (vLLM Router's ZMQ discovery + mode) is a mode of a static router, not a dynamic frontend. + +-Dynamo is the only implementation today. A subclass sets ``type`` and ++Dynamo and Infera implement it. A subclass sets ``type`` and + ``worker_launch``, registers with ``@register_frontend``, parses its own + registration count in ``parse_health``, decides which rank serves which port, +-and launches the process in ``start_frontends``. ++and launches the process in ``start_frontends``. Infera's workers are direct ++(the engine's own server, started by the wrapper module ``worker_module`` ++names, which registers it), so it overrides the Dynamo-shaped metrics path. + """ + + from __future__ import annotations diff --git a/src/srtctl/frontends/infera.py b/src/srtctl/frontends/infera.py new file mode 100644 -index 00000000..a04d789d +index 0000000..77d78d0 --- /dev/null +++ b/src/srtctl/frontends/infera.py -@@ -0,0 +1,249 @@ +@@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + @@ -131,8 +229,20 @@ index 00000000..a04d789d + } +) +_ROLE_BY_DISAGG_MODE = {"prefill": "prefill", "decode": "decode", "mixed": "decode"} -+# Infera's registering launcher for each engine this frontend pairs with. ++# Infera's registering launcher for each engine this frontend pairs with. Only SGLang: ++# Infera also wraps vLLM (infera.engine.vllm), but a row for it would also need the vLLM ++# backend to call the worker hooks and this frontend's port answers checked against it. +WORKER_LAUNCHERS = {"sglang": "infera.engine.sglang"} ++# The launcher's readiness listener port; unset, it binds a fixed default. ++READINESS_PORT_ENV = "INFERA_READINESS_PORT" ++# The release setup_script installs (configs/install-infera.sh); the router and the ++# workers must run the same one. ++VERSION_ENV = "INFERA_VERSION" ++# `python3 -m infera.server` routes kv-aware unless frontend.args says otherwise. ++DEFAULT_ROUTER_POLICY = "kv-aware" ++# kv-aware's decode balancing by in-flight input tokens, which needs no KV events; the ++# router reads this env when frontend.args leaves decode-input-demand unset. ++DECODE_INPUT_DEMAND_ENV = "INFERA_DECODE_INPUT_DEMAND" + + +def etcd_endpoint(runtime: RuntimeContext) -> str: @@ -140,6 +250,12 @@ index 00000000..a04d789d + return f"http://{runtime.infra_node_ip}:{ETCD_CLIENT_PORT}" + + ++def decode_input_demand(config: Any) -> str: ++ """The router's decode-input-demand mode: frontend.args, else the router's env, else off.""" ++ env = {**(config.environment or {}), **(config.frontend.env or {})} ++ return str((config.frontend.args or {}).get("decode-input-demand", env.get(DECODE_INPUT_DEMAND_ENV, "off"))) ++ ++ +@register_frontend("infera") +class InferaFrontend(DynamicFrontend): + """``infera-router`` discovering ``infera.engine.sglang`` workers through etcd.""" @@ -167,6 +283,40 @@ index 00000000..a04d789d + "frontend.type: infera registers workers in the job's own etcd; " + "an external etcd service is not supported" + ) ++ for role, backend in config.active_role_backends(): ++ if backend.is_grpc_mode(role): ++ raise ValueError(f"frontend.type: infera routes HTTP workers; roles.{role} serves gRPC") ++ policy = (config.frontend.args or {}).get("router-policy", DEFAULT_ROUTER_POLICY) ++ publishing = [ ++ role for role, backend in config.active_role_backends() if backend.get_kv_events_config_for_mode(role) ++ ] ++ if policy == "kv-aware" and not publishing: ++ if decode_input_demand(config) in ("on", "shadow"): ++ logger.warning( ++ "frontend.type: infera: no role publishes KV events, so kv-aware picks prefill workers " ++ "on load (and session affinity) alone; decode-input-demand still balances decode" ++ ) ++ else: ++ raise ValueError( ++ "frontend.type: infera routes kv-aware (infera.server's default router-policy) on KV events, " ++ "but no role publishes them, so the router would route on load alone; set kv_events: true " ++ "on the prefill (or agg) role, set frontend.args.router-policy, or turn on " ++ "decode-input-demand if kv-aware is there for decode balancing" ++ ) ++ versions = { ++ source: env[VERSION_ENV] ++ for source, env in ( ++ ("environment", config.environment), ++ ("frontend.env", config.frontend.env), ++ *((f"roles.{name}.env", role.env) for name, role in (config.roles or {}).items()), ++ ) ++ if env and VERSION_ENV in env ++ } ++ if len(set(versions.values())) > 1: ++ raise ValueError( ++ f"the Infera router and workers must run one release, but {VERSION_ENV} differs: " ++ + ", ".join(f"{source}={version}" for source, version in versions.items()) ++ ) + + def worker_metrics_path(self, backend: Any, mode: WorkerMode) -> str | None: + """Workers are SGLang's own servers, so metrics are the engine's, not a runtime's.""" @@ -199,18 +349,22 @@ index 00000000..a04d789d + ) + ] + -+ def direct_worker_launcher(self, backend: Any, process: Process, runtime: RuntimeContext) -> list[str] | None: -+ """Start the engine through Infera's launcher so the worker registers itself in etcd. ++ def worker_module(self, backend_type: str) -> str: ++ """Infera's launcher, which starts the engine's own server and registers it in etcd.""" ++ try: ++ return WORKER_LAUNCHERS[backend_type] ++ except KeyError: ++ raise ValueError(f"frontend.type: infera has no worker launcher for engine {backend_type!r}") from None ++ ++ def worker_launcher_args(self, backend: Any, process: Process, runtime: RuntimeContext) -> list[str]: ++ """Registration in the job's etcd over HTTP and ZMQ. + -+ The KV-events port is the allocator's: when the role publishes KV events -+ the backend passes ``--kv-events-config`` with it, and the launcher -+ advertises that endpoint. Infera's own KV probe plane, which the router -+ does not read, stays off so the worker opens no unallocated listener. ++ Every listener the launcher opens is the allocator's. A role that publishes ++ KV events gets ``--kv-events-config`` with its allocated port from the ++ backend, and the launcher advertises that endpoint. Infera's own KV probe ++ plane, which the router does not read, stays off. + """ -+ command = [ -+ "python3", -+ "-m", -+ WORKER_LAUNCHERS[backend.type], ++ args = [ + "--discovery-backend", + "etcd", + "--etcd-endpoint", @@ -224,16 +378,30 @@ index 00000000..a04d789d + "--kv-events", + "off", + ] -+ if process.kv_events_port is None: -+ command.append("--no-enable-kv-events") -+ return command ++ # Without --kv-events-config the launcher publishes anyway, on a port it picks itself. ++ if backend.get_kv_events_config_for_mode(process.endpoint_mode) is None: ++ args.append("--no-enable-kv-events") ++ return args ++ ++ def worker_environment(self, process: Process, runtime: RuntimeContext) -> dict[str, str]: ++ """The readiness listener the launcher opens once registered: the process's system port. ++ ++ srtctl reads readiness from the router, and the system port is otherwise ++ unused without Dynamo, so co-located workers do not contend for Infera's ++ default. ++ """ ++ return {READINESS_PORT_ENV: str(process.sys_port)} + + def parse_health(self, response_json: dict, expected_prefill: int, expected_decode: int) -> WorkerHealthResult: -+ """Count active registrations by role; an aggregate (``mixed``) worker counts as decode.""" ++ """Count active registrations by role; an aggregate (``mixed``) worker counts as decode. ++ ++ An entry without ``status`` or ``disagg_mode`` does not count, so a schema ++ change in the router fails readiness instead of passing it. ++ """ + counts = {"prefill": 0, "decode": 0} + for worker in response_json.get("workers") or []: -+ role = _ROLE_BY_DISAGG_MODE.get(str(worker.get("disagg_mode", "mixed"))) -+ if role is not None and worker.get("status", "active") == "active": ++ role = _ROLE_BY_DISAGG_MODE.get(str(worker.get("disagg_mode"))) ++ if role is not None and worker.get("status") == "active": + counts[role] += 1 + ready = counts["prefill"] >= expected_prefill and counts["decode"] >= expected_decode + return WorkerHealthResult( @@ -327,3 +495,23 @@ index 00000000..a04d789d + ) + ) + return processes +diff --git a/src/srtctl/mock.py b/src/srtctl/mock.py +index 1458b28..0b8e9eb 100644 +--- a/src/srtctl/mock.py ++++ b/src/srtctl/mock.py +@@ -295,6 +295,7 @@ def mock_infrastructure(*, options: MockOptions, output_dir: Path): + ("srtctl.cli.mixins.postprocess_stage.get_batch_host", _fake_batch_host), + ("srtctl.cli.mixins.postprocess_stage.shrink_job_allocation", _fake_shrink), + ("srtctl.frontends.dynamo.start_srun_process", _fake_srun), ++ ("srtctl.frontends.infera.start_srun_process", _fake_srun), + ("srtctl.frontends.sglang.start_srun_process", _fake_srun), + ("srtctl.frontends.static_router.start_srun_process", _fake_srun), + ("srtctl.frontends.trtllm_serve.start_srun_process", _fake_srun), +@@ -307,6 +308,7 @@ def mock_infrastructure(*, options: MockOptions, output_dir: Path): + ("srtctl.cli.mixins.frontend_stage.get_hostname_ip", _fake_hostname_ip), + ("srtctl.cli.mixins.benchmark_stage.get_hostname_ip", _fake_hostname_ip), + ("srtctl.cli.mixins.service_stage.get_hostname_ip", _fake_hostname_ip), ++ ("srtctl.frontends.infera.get_hostname_ip", _fake_hostname_ip), + ("srtctl.frontends.sglang.get_hostname_ip", _fake_hostname_ip), + ("srtctl.frontends.static_router.get_hostname_ip", _fake_hostname_ip), + ("srtctl.frontends.trtllm_serve.get_hostname_ip", _fake_hostname_ip), diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index 9ed5ee39d0..e0b89117ec 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,5 +9,5 @@ Each patch backports an upstream fix that the shared submodule pin does not incl | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exclusive node's exporter (DCGM or AMD) reports every GPU, which the collector otherwise flags as `unexpected_device`. The pin includes #573, but upstream merged a different design: it keeps every row and excuses idle worker-node GPUs only after a DCGM worker GPU UUID probe. [`infx/results/power/multinode.py`](../../../infx/results/power/multinode.py) does not apply that excusal yet, so keep this filter until it does. | -| `630-infera-frontend.patch` | [NVIDIA/srt-slurm#630](https://github.com/NVIDIA/srt-slurm/pull/630) (draft) | `frontend.type: infera`: SGLang workers start through Infera's `infera.engine.sglang`, register in the job's etcd and are discovered by `infera-router` (the `infera-sglang` framework). Adds `Frontend.direct_worker_launcher`; the patch carries only `src/srtctl` (frontend, registry, hook and the SGLang backend's use of it). | +| `630-infera-frontend.patch` | [NVIDIA/srt-slurm#630](https://github.com/NVIDIA/srt-slurm/pull/630) (open) | `frontend.type: infera`: SGLang workers start through Infera's `infera.engine.sglang`, register in the job's etcd and are discovered by `infera-router` (the `infera-sglang` framework). Adds the `Frontend.worker_module` / `worker_launcher_args` / `worker_environment` hooks and `Backend.get_kv_events_config_for_mode`; the patch is the PR's `src/srtctl` diff at `72c4d04`. | | `900-pool-expected-devices.patch` | pending | Plan every service pool node's `gpus_per_node` GPUs as expected power-telemetry devices under the service's name (`worker_role=`, `worker_index=`), so a services-only recipe (`frontend.type: none`, one `services[].nodes` pool, `benchmark.type: custom`) reaches collector readiness and publishes instead of failing with `exporter_startup_timeout` + `expected_device_missing` + `unexpected_device`. A services-only job records `worker_identity_proof.error = "no engine workers to probe"` instead of a probe traceback, the identity gate only scopes engine-role devices, and the offline validator accepts pool roles. Engine-only manifests are unchanged. With `573-participating-gpus.patch` an exclusive pool node's unplanned exporter rows are dropped at scrape time, so a recipe whose `gpus_per_node` is below the node's physical GPU count still validates; upstream alone reports them as `unexpected_device`. | From eb0bec2636b72cac453ad2987e9c82f6b1ccfe39 Mon Sep 17 00:00:00 2001 From: liyingli Date: Fri, 13 Nov 2026 10:28:11 +0000 Subject: [PATCH 08/12] feat(infera-sglang): drop the SGLang fixes that are now upstream sgl-project/sglang#33059 (B1), #46830 (B2), #46820 (B6) and #21194 (B7) are merged; the first ROCm 7.2 nightly with all four is the one after 11-11 10:24 UTC (20261111 was built from 34dfaccc, before #46830 and #46820). apply.sh keeps only B8 (issue #46829, fix #46918 still open), and the waiver shrinks to it. Needs the image bump to that nightly. Co-authored-by: Cursor --- .../configs/infera-sglang-patches/apply.sh | 39 +- .../infera-sglang-patches/dsa_dp_sync.diff | 51 --- .../patch_deepseek_v2_pp_embed_dim.py | 55 --- .../patch_dsa_indexer_hip_dp_padded_rows.py | 351 ------------------ .../patch_kv_metrics_send_noblock.py | 70 ---- .../configs/infera-sglang.sh | 2 +- inferencex-e2e/docs/waiver/4629.md | 33 +- .../tests/srt_slurm/test_setup_scripts.py | 2 +- 8 files changed, 23 insertions(+), 580 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh index 32ce4b651c..e39ad40772 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh @@ -1,47 +1,20 @@ #!/usr/bin/env bash -# SGLang fixes the infera-sglang GLM-5.2 points need on the pinned -# lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018 image, applied in the container -# before the server starts. Waiver: docs/waiver/4629.md. Each fix is marker-gated and -# idempotent, and the run fails if any of them did not land. +# The SGLang fix the infera-sglang GLM-5.2 points still need on the pinned image, applied in the +# container before the server starts. Waiver: docs/waiver/4629.md. The fix is marker-gated and +# idempotent, and the run fails if it did not land. # -# B1 patch_dsa_indexer_hip_dp_padded_rows.py sglang#33059 aiter indexer rows under DP padding -# B2 dsa_dp_sync.diff sglang#46830 DP-rank D2H sync deadlock -# B6 patch_kv_metrics_send_noblock.py sglang#46820 scheduler blocks on the KvMetrics PUSH -# B7 patch_deepseek_v2_pp_embed_dim.py sglang#47184 PP middle stage has no embedding # B8 patch_mla_absorb_bmm_default_config.py sglang#46829 absorb-BMM tile config aborts Triton set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SGLANG_DIR="${SGLANG_DIR:-/sgl-workspace/sglang}" SRT="$SGLANG_DIR/python/sglang/srt" -git_sgl() { git -c safe.directory='*' -C "$SGLANG_DIR" "$@"; } -echo "[infera-sglang-patches] B1 dsa_indexer DP-padded rows" -python3 "$here/patch_dsa_indexer_hip_dp_padded_rows.py" - -echo "[infera-sglang-patches] B2 dsa_dp_sync.diff" -if git_sgl apply --check "$here/dsa_dp_sync.diff" 2>/dev/null; then - git_sgl apply "$here/dsa_dp_sync.diff" - echo " applied" -elif git_sgl apply --check -R "$here/dsa_dp_sync.diff" 2>/dev/null; then - echo " already applied" -else - git_sgl apply --check "$here/dsa_dp_sync.diff" - exit 1 -fi - -for patch in patch_kv_metrics_send_noblock.py patch_deepseek_v2_pp_embed_dim.py \ - patch_mla_absorb_bmm_default_config.py; do - echo "[infera-sglang-patches] $patch" - python3 "$here/$patch" -done +echo "[infera-sglang-patches] B8 patch_mla_absorb_bmm_default_config.py" +python3 "$here/patch_mla_absorb_bmm_default_config.py" # source file (under sglang/srt) : text only the patched source contains markers=( - "layers/attention/dsa/dsa_indexer.py:_p1v2_trim" - "layers/attention/dsa_backend.py:must not be None for DRAFT_EXTEND_V2" - "managers/scheduler_components/kv_events_publisher.py:_infera_kv_metrics_noblock" - "models/deepseek_v2.py:_infera_pp_embed_dim" "models/deepseek_common/attention_forward_methods/forward_mla_rocm.py:_infera_absorb_bmm_default" ) fail=0 @@ -56,4 +29,4 @@ for spec in "${markers[@]}"; do fi done [[ "$fail" == 0 ]] || { echo "[infera-sglang-patches] verification failed" >&2; exit 1; } -echo "[infera-sglang-patches] all five applied" +echo "[infera-sglang-patches] B8 applied" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff deleted file mode 100644 index 6953005512..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/dsa_dp_sync.diff +++ /dev/null @@ -1,51 +0,0 @@ -diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py -index 37fcbe40421d..16f7e97d7d42 100644 ---- a/python/sglang/srt/layers/attention/dsa_backend.py -+++ b/python/sglang/srt/layers/attention/dsa_backend.py -@@ -791,7 +791,15 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): - # needs_cpu_seq_lens=False nulls the host mirror for spec-v2 relay - # batches; graph replay uses the static page-table width, so only this - # eager (e.g. over-capture-bs) fallback needs a length here. -- max_seqlen_k = int(forward_batch.seq_lens.max().item()) + draft_token_num -+ # -+ # Use the static width rather than seq_lens.max().item(): that is a -+ # device-to-host sync, and this branch is entered per rank (an idle -+ # DP peer keeps its host mirror and takes the arm above), so under -+ # DP attention the ranks stop issuing the same collective sequence -+ # and the group deadlocks. Over-allocating columns is safe -- the -+ # page table is only indexed through top-k, which masks per row by -+ # cache_seqlens, so the extra columns are never selected. -+ max_seqlen_k = self.req_to_token.shape[1] - # [b, max_seqlen_k] - page_table = self.req_to_token_pool.req_to_token[ - forward_batch.req_pool_indices, :max_seqlen_k -@@ -851,19 +859,19 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): - page_table, repeats=self.speculative_num_draft_tokens, dim=0 - ) - elif forward_batch.forward_mode.is_draft_extend_v2(): -- if forward_batch.extend_prefix_lens_cpu is None: -- assert forward_batch.extend_prefix_lens is not None -- forward_batch.extend_prefix_lens_cpu = ( -- forward_batch.extend_prefix_lens.cpu().tolist() -- ) -- if forward_batch.seq_lens_cpu is None: -- forward_batch.seq_lens_cpu = forward_batch.seq_lens.cpu() -- forward_batch.seq_lens_sum = int(forward_batch.seq_lens_cpu.sum()) -+ # No host mirrors are materialized here. Both would be D2H syncs on -+ # this same per-rank branch, so with only the max_seqlen_k fix above -+ # the DP group still deadlocks. Neither is read on this path: -+ # extend_prefix_lens_cpu and seq_lens_sum are consumed only inside -+ # the is_extend() arm below, and DRAFT_EXTEND_V2 is not is_extend() -+ # (include_draft_extend_v2 defaults to False). What this branch does -+ # need -- extend_seq_lens_cpu -- is already supplied by -+ # eagle_worker_common.prepare_for_draft_extend, on the same -+ # gpu_only branch and for this same reason. - assert ( - forward_batch.extend_seq_lens_cpu is not None - and forward_batch.extend_seq_lens is not None -- and forward_batch.extend_prefix_lens_cpu is not None -- ), "All of them must not be None" -+ ), "extend_seq_lens{,_cpu} must not be None for DRAFT_EXTEND_V2" - - extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu - assert forward_batch.extend_seq_lens is not None diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py deleted file mode 100644 index 934fc24c54..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_deepseek_v2_pp_embed_dim.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -"""Let a middle pipeline stage build DeepseekV2Model (models/deepseek_v2.py). - -WHAT: the gemm-output allocator sizing in `DeepseekV2Model.__init__` reads -`self.embed_tokens.embedding_dim` twice. Only the first pipeline stage (and the -last one when it carries an MTP draft layer) holds a real embedding; every other -stage has a `PPMissingLayer`, so `--pp-size` > 2 dies at load time with -AttributeError on PP rank 1. The embedding width is the hidden size, so both -reads become `config.hidden_size`; behaviour on PP1 and on every stage that has -an embedding is unchanged. - -Needed by the GLM-5.2 PD example's TP1 x PP4 prefill. Not upstream as of v0.5.22. - -Self-locating and idempotent; exits non-zero when the anchor drifted. -Marker: `_infera_pp_embed_dim`. -""" - -import importlib.util -import py_compile -import sys -from pathlib import Path - -TAG = "[deepseek-v2-pp-embed-dim]" -MARKER = "_infera_pp_embed_dim" -OLD = "self.embed_tokens.embedding_dim" -NEW = "config.hidden_size" -FIRST_OLD = f" and {OLD} == 7168\n" -FIRST_NEW = f" and {NEW} == 7168 # {MARKER}: PP middle stages have no embedding\n" - - -def main() -> int: - spec = importlib.util.find_spec("sglang") - if not spec or not spec.origin: - print(f"{TAG} cannot locate sglang") - return 1 - path = Path(spec.origin).parent / "srt" / "models" / "deepseek_v2.py" - source = path.read_text() - if MARKER in source: - print(f"{TAG} already present — skipping") - return 0 - if source.count(OLD) != 2 or source.count(FIRST_OLD) != 1: - print(f"{TAG} anchor not found as expected in {path} — re-cut the patch") - return 1 - path.write_text(source.replace(FIRST_OLD, FIRST_NEW, 1).replace(OLD, NEW)) - for cached in (path.parent / "__pycache__").glob(path.stem + ".*.pyc"): - cached.unlink() - py_compile.compile(str(path), doraise=True) - print(f"{TAG} patched {path}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py deleted file mode 100644 index df626f1859..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_dsa_indexer_hip_dp_padded_rows.py +++ /dev/null @@ -1,351 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -"""PATCH 01 -- dsa_indexer HIP/aiter DP-padded rows. - -WHAT: the aiter (HIP) paged-MQA branch sizes its `logits` output from the -DP-PADDED row count while `lengths` is sized to the REAL count, so top-k dies -with "Expected lengths.size(0) == B to be true, but got false". Reconcile the -two counts to `min(real, padded)`: trim q/weights when q_fp8 is the longer side -(the contract every CUDA backend already honours) and restore the padding after -`topk_transform`, or clip the lengths when q_fp8 is the SHORTER side. See -"THE ROW COUNT DIVERGES BOTH WAYS" below. - -WHY: without it, DP-attention + EAGLE MTP for GLM-5.2 DSA crashes as soon as -more than one request is in flight -- on gfx950 at the very first batch, and on -gfx942 at conc > 1. CUDA was never affected: those backends take `q_offset=` and -slice internally. ROCm-specific. - -THE ROW COUNT DIVERGES BOTH WAYS (`GLM52_P1V3`) - An earlier revision of this fix guarded one direction only, `real < padded`, - i.e. it assumed DP padding always makes `q_fp8` LONGER. On a DP-attention IDLE - rank under MTP draft-extend the inequality inverts. Captured live with this - patch's own SGLANG_DEBUG_DSA_ROWS=1: - - mode=IDLE q_fp8=(1,32,128) q_offset=2 ntnp=0 agree=False lengths=(2,) - -> mqa_q=(1,32,128) - - `q_offset` (= sum of dsa_extend_len_cpu) is 2 while only 1 row is materialized - in q_fp8, so `2 < 1` is false, no trim runs, aiter sizes its logits from - q_fp8.shape[0] = 1, and fast_topk_v2 gets 1 score row against 2 lengths - entries -- killing the scheduler rank and dropping the router to - `active_workers: 1`. A trim cannot fix this direction: there are FEWER query - rows than lengths entries, so there is nothing to cut. The LENGTHS side is - clipped instead, through `topk_transform`'s existing `ke_offset` parameter. - - The two directions, and what each does: - - real < padded the #32762 case: q_fp8 carries DP padding, so TRIM q/weights - down to the real count, then re-pad after topk_transform. - Trim at _p1v2_real == 0 too -- a DP-IDLE rank has no requests, - so the real count is 0 while q_fp8 still carries padding. - q_fp8[:0] is a legal empty slice and is what an idle rank - should pass; the CUDA path likewise slices unconditionally. - DO NOT reintroduce a `0 < q_offset` lower bound here: an - earlier revision had one and it made fast_topk_v2 assert on - exactly those idle ranks. - real > padded the P1V3 case above: nothing to trim, so CLIP the lengths. - - BOUND BEFORE THE BRANCH. `_p1v2_clip` and `_p1v2_rows` are bound before the - `is_aiter()` test, not inside it. `_p1v2_clip` is read unconditionally at the - topk_transform call, so a branch-local binding would be a NameError on any - non-aiter backend; `_p1v2_rows` defaults to `_p1v2_padded`, making the restore - a no-op if it is ever reached without the branch having run. aiter is - unconditional on ROCm so neither has fired in practice, but a build-time patch - should not depend on that. - - This half is what makes the patch survive an AGENTIC workload. The bug needs - MTP AND DP-attention AND an idle rank at once: an 8-round fixed-shape sweep - ran MTP + DPA for 660 requests without hitting it, because - `--random-range-ratio 1.0` keeps batch shapes homogeneous and ranks rarely go - idle mid-flight. A breathing session population produces ragged batches - constantly, and the one-directional revision crashed the decode leg roughly - 13 minutes into an agentic benchmark. - - Reproduced twice under that workload (125 s and 766 s into two independent - runs). With the fix: 0 occurrences of `Expected lengths.size` across two full - ~4,000 s windows on two different clusters, 0 scheduler exceptions, - `active_workers: 2` throughout. Second-hand for the reader: it was validated - as a runtime script before being folded in here, so this is a re-shaping of an - already-measured fix rather than a fresh one. Evidence in the internal - reproduction kit -- ask the patch author. - -WHY A SCRIPT AND NOT A DIFF. Patches 02 and 04 are `--fuzz=0` context diffs -pinned to one base. This fix is needed on every engine base we ship, and -`dsa_indexer.py` drifts across all of them: v0.5.16/gfx942, v0.5.18/glm53 and -the v0.5.19 gfx950 nightly. Anchoring on source text instead of line numbers -covers all three from one source of truth. - - ANCHORS 1 AND 3 ARE DELIBERATELY LOOSE, so do not "tidy" them back onto the - adjacent code. Anchor 1 sits on the `contextlib`/`logging` import head rather - than the typing line, because v0.5.19 dropped `List` from that line. Anchor 3 - starts at `if self.paged_mqa_logits_backend.is_aiter():` rather than the - `weights.squeeze(2)` above it, because v0.5.19 inserted a nested helper - between the two. Both forms verified unique on v0.5.16, v0.5.18 and v0.5.19. - - NOTE on the P1V3 anchor. It is the bare `topk_transform(logits, - self.index_topk)` call. The complete patch was applied and bytecode-verified - against both release tags on 2026-09-02. If either source drifts, the - uniqueness check below writes NOTHING and exits 1, which is the intended - failure: a half-applied fix crashes in the same place the unpatched tree does. - -UPSTREAM STATUS (queried with `gh` on 2026-08-01; re-queried and each PR state -re-read 2026-08-03) - issue NONE, for either half. Searched sgl-project/sglang for - "Expected lengths.size", "DSA padding DP attention" and the - GLM-5.2 MTP crash text (P1V2); and for "DSA idle rank MTP draft - extend", "q_offset dsa_extend_len", "fast_topk_v2 lengths" - (P1V3): no match either time. Weak evidence -- `gh search` - matches titles and bodies, not diff content, so an upstream - change to this same function that never names the symptom would - not surface. The anchor-collision PRs below were found only by - searching for the FUNCTION name, not the symptom. - third-party PR #32762 "[NPU] Fix DSA eager padding mismatch in PD MTP warm-up" - (stellaxcpeng, OPEN). SAME BUG CLASS on a different platform, - and this patch is written in its shape: one boolean gates both - trim and restore, and the returned row count is asserted before - the padding is re-attached. It does not touch the aiter branch, - so it does not fix HIP. - #31683 "[ROCm][MI35X] Enable GLM-5.2-MXFP4 MTP" (long10024070, - OPEN). ADJACENT, NOT THE SAME FIX -- same function, but it - widens forward_cuda's guard from `seq_lens.numel() == 0` to - `is_idle() or ...`. Different symptom; verified against its diff. - #30378 / #30427 (both MERGED, in both bases) clamp padded-row - seq_lens VALUES in triton_ops/pad.py. This fixes the HIP-side - row COUNT -- a different defect in the same area. - anchor risk Two OPEN PRs rewrite the code this patch anchors on. Neither - fixes this defect; both would make the anchors drift, which - fails the build loudly rather than mis-applying. - #32738 "[Fix] DSA Indexer: pad heads for DeepGEMM paged MQA - logits on decode/target-verify" -- edits _get_topk_paged at the - SAME two aiter call sites, replacing `q_fp8` with a - `q_fp8_padded` from a new `_pad_heads_for_deep_gemm`. Head-dim - padding, orthogonal to our row-count fix, but a direct textual - collision with the P1V2 anchor. - #31480 "[DSA] Add an arch-independent torch paged-MQA-logits - backend" (updated 2026-08-03) -- extracts the backend into - paged_mqa_logits_backend.py, i.e. restructures the - `is_aiter()` dispatch this whole patch hangs off. - own PR #33059 "Fix DSA indexer aiter (HIP) padding mismatch under - DP-attention" (dorado269, OPEN, REVIEW_REQUIRED, against main, - 1 file +16/-4). CI is red only at `pr-gate`, which blocks on a - missing `run-ci` label, not on code. - -THIS PATCH vs OUR OWN PR #33059. Same defect, overlapping edits. They differ -deliberately: - base here: v0.5.16/gfx942 and v0.5.18/gfx950, with every edit verified - against both release tags. #33059: sglang `main`, whose - `_get_topk_paged` has drifted further (cutedsl / dg_native branches, - `_mask_init_and_local_tokens`). - scope #33059 carries the `real < padded` half ONLY. The P1V3 clip is NOT - in it and has not been filed upstream at all -- it should be, and - this repo already ships the instrumentation that proves the bug. - form here: explicit `_p1v2_rows` / `_p1v2_trim` / `_p1v2_clip` locals - with a post-kernel row-count ASSERT, following #32762's shape. - #33059: the minimal `q_fp8[:q_offset]` / `weights[:q_offset]` slice - and a plain `if q_offset < q_fp8.shape[0]` restore. - local-only Stripped for upstream: the `GLM52_P1V2` / `GLM52_P1V3` markers (the - build script greps `_p1v2_trim` and `_p1v2_rows` out of the - BYTECODE to prove the patch landed), the `_p1v2_*` naming, and the - `SGLANG_DEBUG_DSA_ROWS` logging block. All three are scaffolding - for this repo's verification loop and carry no upstream value. - trade-off The upstream form is smaller and easier to review. Ours fails - LOUDER: on a shape drift the assert crashes instead of silently - returning a short tensor -- and it handles a direction #33059 does - not. Do NOT delete this script when #33059 merges; the clip half - would go with it. - validation Ours: 4/4 probe, conc=64 1k/1k 256/256, 2540/2540 in the PD set, - plus two full ~4,000 s agentic windows on two gfx950 clusters for - the P1V3 half. #33059's port to `main` was NOT re-run on hardware. - -RELATION TO THE INDEXSHARE WORKAROUND. NOT substituted. -`--json-model-override-args '{"index_share_for_mtp_iteration":false}'` removes -the rank divergence that draft_cuda_graph_dp_vote and dsa_page_table_rows -address, but this is an independent bug -- present regardless of IndexShare, and -the arm that validated the override had THIS patch applied. - -Self-locating and idempotent. All edits or none: an anchor that is missing or no -longer unique writes NOTHING and fails (exit 1), because a half-applied fix -crashes in the same place the unpatched tree does. -""" - -import importlib.util -import sys -from pathlib import Path - -_TAG = "[dsa-indexer-rows]" - -_REL = "layers/attention/dsa/dsa_indexer.py" - -# (anchor, anchor + our edit). Each anchor must occur exactly once; the -# replacement doubles as the already-applied marker. -_EDITS: list[tuple[str, str]] = [ - # `os` for the debug-logging env switch below. - ( - """from __future__ import annotations - -import contextlib -import logging -""", - """from __future__ import annotations - -import contextlib -import logging -import os -""", - ), - # Anchored on the two platform flags rather than the whole block: v0.5.16 - # adds `_is_xpu` on the line after `_is_npu`. - ( - """_is_cuda = is_cuda() -_is_hip = is_hip() -""", - """_is_cuda = is_cuda() -_is_hip = is_hip() -# Set SGLANG_DEBUG_DSA_ROWS=1 to log the indexer row bookkeeping (padded vs real). -_DSA_DEBUG_ROWS = os.environ.get("SGLANG_DEBUG_DSA_ROWS", "0") == "1" - -""", - ), - # Trim q/weights to the real row count on the aiter branch only. - ( - """ if self.paged_mqa_logits_backend.is_aiter(): - logits = aiter_paged_mqa_logits( - q_fp8, - kv_cache_fp8, - weights, -""", - """ # GLM52_P1V2/P1V3: bound BEFORE the branch, not inside it -- _p1v2_clip is - # read unconditionally below, so a branch-local binding is a NameError on - # any non-aiter backend. See "BOUND BEFORE THE BRANCH" in the header. - _p1v2_trim = False - _p1v2_clip = False - _p1v2_real = q_offset - _p1v2_padded = q_fp8.shape[0] - _p1v2_rows = _p1v2_padded - if self.paged_mqa_logits_backend.is_aiter(): - # WHY aiter sizes logits from q_fp8.shape[0], which disagrees with - # `lengths` BOTH ways. HOW reconcile to min(): trim when longer, clip - # when shorter. Cases: "DIVERGES BOTH WAYS" in the header. - _p1v2_rows = min(_p1v2_real, _p1v2_padded) - _p1v2_trim = _p1v2_rows < _p1v2_padded - _p1v2_clip = _p1v2_rows < _p1v2_real - _q_mqa, _w_mqa = q_fp8, weights - if _p1v2_trim: - _q_mqa = q_fp8[:_p1v2_rows] - _w_mqa = weights[:_p1v2_rows] - if _DSA_DEBUG_ROWS: - # Cross-check against the padding bookkeeping #32762 keys off. - # NOT an assert: it does NOT agree with q_offset on the MTP - # draft-extend path -- that disagreement is the P1V3 case above, - # and this log is how it was caught. - _p1v2_ntnp = getattr(forward_batch, "_original_num_tokens", None) - if _p1v2_ntnp is None: - _p1v2_ntnp = forward_batch.num_token_non_padded_cpu - logger.info( - "[dsa-rows] mode=%s q_fp8=%s q_offset=%s ntnp=%s agree=%s " - "lengths=%s -> mqa_q=%s", - forward_batch.forward_mode, - tuple(q_fp8.shape), - _p1v2_real, - _p1v2_ntnp, - _p1v2_ntnp == _p1v2_real, - tuple(metadata.get_seqlens_expanded().shape), - tuple(_q_mqa.shape), - ) - logits = aiter_paged_mqa_logits( - _q_mqa, - kv_cache_fp8, - _w_mqa, -""", - ), - # GLM52_P1V3: clip the lengths when the rows were clipped rather than trimmed. - # Anchored on the bare call, which occurs once; the `if` wrapper is the - # already-applied marker. - ( - """ topk_result = metadata.topk_transform(logits, self.index_topk) -""", - """ # GLM52_P1V3: rows were CLIPPED (real > padded), so the lengths this - # indexes must be clipped to match. Via topk_transform's existing - # `ke_offset` -- an extension point that only overrides seq_lens_topk -- - # NOT by mutating shared, possibly graph-captured metadata. - if _p1v2_clip: - topk_result = metadata.topk_transform( - logits, - self.index_topk, - ke_offset=metadata.get_seqlens_expanded()[:_p1v2_rows], - ) - else: - topk_result = metadata.topk_transform(logits, self.index_topk) -""", - ), - # Restore the trimmed padding, gated on the SAME boolean. - ( - """ # Restore possible padding exist in the hidden states. - if not _is_hip and q_offset < q_fp8.shape[0]: - pad_len = q_fp8.shape[0] - q_offset - padding = torch.full( - (pad_len, topk_result.shape[1]), -""", - """ # GLM52_P1V2: restore the trimmed padding -- gate on the SAME boolean and - # assert the row count. BEFORE, a re-derived condition skipped the restore - # on a shape drift, returning a short tensor instead of crashing. - # - # Keyed off _p1v2_rows, not _p1v2_real: under P1V3's clip case the two - # differ, and it is _p1v2_rows that the kernel actually ran over. - if _p1v2_trim: - assert topk_result.shape[0] == _p1v2_rows, ( - "GLM52_P1V3: paged-MQA top-k returned " - f"{topk_result.shape[0]} rows for {_p1v2_rows} trimmed query " - f"rows (real={_p1v2_real}, padded={_p1v2_padded}); refusing to " - "pad a mis-shaped result." - ) - padding = torch.full( - (_p1v2_padded - _p1v2_rows, topk_result.shape[1]), -""", - ), -] - - -def _srt_dir(): - spec = importlib.util.find_spec("sglang") - if not spec or not spec.origin: - return None - d = Path(spec.origin).parent / "srt" - return d if d.is_dir() else None - - -def main(): - srt = _srt_dir() - if srt is None: - print(f"{_TAG} sglang not importable — skipping") - return 0 - - f = srt / _REL - if not f.is_file(): - print(f"{_TAG} {f} is missing — sglang layout changed, re-anchor the patch") - return 1 - - src = out = f.read_text() - for old, new in _EDITS: - if new in out: - continue # this edit is already in the tree - found = out.count(old) - if found != 1: - where = "absent" if found == 0 else f"{found}x ambiguous" - print(f"{_TAG} anchor {where} in {_REL}: {old.splitlines()[0]!r}") - print(f"{_TAG} sglang drifted — re-cut the patch, nothing written") - return 1 - out = out.replace(old, new, 1) - - if out == src: - print(f"{_TAG} already present — skipping") - return 0 - f.write_text(out) - print(f"{_TAG} patched {f}") - print(f"{_TAG} aiter paged-MQA now sees the real row count under DP-attention") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py deleted file mode 100644 index f7fd6f4c75..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_kv_metrics_send_noblock.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -"""Never block the scheduler on the auxiliary KV-metrics PUSH socket. - -WHAT: with KV events enabled, `KvEventsPublisher` also PUSHes a `KvMetrics` -record every scheduler step to an endpoint that nothing binds in an Infera -deployment (the router reads KV events and /metrics, not this socket). Once the -ZMQ send queue fills, `sock_send` blocks and the scheduler stops: a GLM-5.2 -prefill wedged this way (yihou agentx44_20261009, R001, py-spy 2026-10-09). - -FIX: send with zmq.NOBLOCK and drop the record on zmq.Again. KV events and the -Prometheus metrics are unaffected. The v0.5.21 deployments carried this as a -bind-mounted file; v0.5.22 still sends blocking. - -Self-locating and idempotent; exits non-zero when the anchor drifted. -Marker: `_infera_kv_metrics_noblock`. -""" - -import importlib.util -import py_compile -import sys -from pathlib import Path - -TAG = "[kv-metrics-noblock]" -MARKER = "_infera_kv_metrics_noblock" -OLD = ( - " if not self.send_metrics_from_scheduler.closed:\n" - " sock_send(self.send_metrics_from_scheduler, kv_metrics)\n" -) -NEW = ( - " if not self.send_metrics_from_scheduler.closed:\n" - " # Infera: nothing may bind this PUSH peer; a blocking send would\n" - " # stall the scheduler once the queue fills, so drop instead.\n" - " try:\n" - " sock_send(self.send_metrics_from_scheduler, kv_metrics, flags=zmq.NOBLOCK)\n" - f" except zmq.Again: # {MARKER}\n" - " pass\n" -) - - -def main() -> int: - spec = importlib.util.find_spec("sglang") - if not spec or not spec.origin: - print(f"{TAG} cannot locate sglang") - return 1 - path = ( - Path(spec.origin).parent - / "srt" - / "managers" - / "scheduler_components" - / "kv_events_publisher.py" - ) - source = path.read_text() - if MARKER in source: - print(f"{TAG} already present — skipping") - return 0 - if source.count(OLD) != 1 or "\nimport zmq\n" not in source: - print(f"{TAG} anchor not found exactly once in {path} — re-cut the patch") - return 1 - path.write_text(source.replace(OLD, NEW, 1)) - for cached in (path.parent / "__pycache__").glob(path.stem + ".*.pyc"): - cached.unlink() - py_compile.compile(str(path), doraise=True) - print(f"{TAG} patched {path}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh index af5639c028..986c50e313 100755 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Install Infera, at the master config's router version, into the stock SGLang image for the -# infera-sglang points, then apply the SGLang fixes the pinned image still lacks +# infera-sglang points, then apply the SGLang fix the pinned image still lacks # (infera-sglang-patches/apply.sh, waiver docs/waiver/4629.md). amd-infera's Linux x86_64 # wheel on PyPI bundles the infera-router binary next to the infera.engine.sglang worker # wrapper, so the master's router version is the Infera version. Only nats-py is missing from diff --git a/inferencex-e2e/docs/waiver/4629.md b/inferencex-e2e/docs/waiver/4629.md index c0c7e96aa5..ec123e249a 100644 --- a/inferencex-e2e/docs/waiver/4629.md +++ b/inferencex-e2e/docs/waiver/4629.md @@ -19,17 +19,20 @@ checklist prohibits unless covered by a filled-out waiver. ## What is patched `apply.sh` edits the image's SGLang source in place (`/sgl-workspace/sglang`, the tree the image's -`sglang` package is installed from). Every fix is anchored on the exact source it replaces, is -idempotent, and is checked by a marker afterwards; a fix that does not apply fails the job instead +`sglang` package is installed from). The fix is anchored on the exact source it replaces, is +idempotent, and is checked by a marker afterwards; if it does not apply, the job fails instead of serving unpatched. | | Upstream PR/issue | What it changes | Why this config needs it | |---|---|---|---| -| B1 | sglang [#33059](https://github.com/sgl-project/sglang/pull/33059) | DSA indexer on HIP: trim the DP-padded query rows before the aiter top-k kernel and restore them after (`layers/attention/dsa/dsa_indexer.py`) | DP-attention decode. On this source the aiter branch passes the padded `q_fp8` whole (`dsa_indexer.py:1366-1376`; the XPU branch slices to `[:q_offset]`) and the restore at `:1438` excludes HIP | -| B2 | sglang [#46830](https://github.com/sgl-project/sglang/pull/46830) | DSA backend: no device-to-host sync on per-rank branches (`layers/attention/dsa_backend.py`) | DP-attention decode with MTP. `seq_lens.max().item()` (`:1102`) and two `.cpu()` calls (`:1172-1179`) sit on branches that only some DP ranks take, so the ranks deadlock in a collective | -| B6 | sglang [#46820](https://github.com/sgl-project/sglang/pull/46820) | Send the per-step `KvMetrics` record with `zmq.NOBLOCK` and drop it when the queue is full (`managers/scheduler_components/kv_events_publisher.py`) | Prefill runs with KV events (the router routes on them). The record is PUSHed blocking (`:89-90`) to an endpoint nothing in SGLang binds; once the queue fills, the scheduler stops | -| B7 | sglang [#47184](https://github.com/sgl-project/sglang/pull/47184) (rebase of [#21194](https://github.com/sgl-project/sglang/pull/21194)) | Size the gemm-output allocator from `config.hidden_size` instead of `self.embed_tokens.embedding_dim` (`models/deepseek_v2.py`) | TP1 × PP4 prefill. A middle pipeline stage holds a `PPMissingLayer` (`:2923`), so the reads at `:2987, :3022` raise `AttributeError` at load | -| B8 | sglang issue [#46829](https://github.com/sgl-project/sglang/issues/46829) | Drop the hand-written Triton tile config of the MLA absorb BMM and use Triton's default (`models/deepseek_common/attention_forward_methods/forward_mla_rocm.py`) | GLM-5.2 prefill. `_absorb_bmm_config` (`:156`) trips an LLVM assertion in Triton 3.7.0 on gfx950, the Triton this image ships; reproduced on 10-27 | +| B8 | sglang issue [#46829](https://github.com/sgl-project/sglang/issues/46829), fix [#46918](https://github.com/sgl-project/sglang/pull/46918) (open) | Drop the hand-written Triton tile config of the MLA absorb BMM and let aiter pick its own (`models/deepseek_common/attention_forward_methods/forward_mla_rocm.py`) | GLM-5.2 prefill. For 193-256 rows `_absorb_bmm_config` (`:174`) asks for 4 waves per EU, which trips an LLVM assertion in Triton 3.7.0 on gfx950, the Triton this image ships; reproduced on 10-27. #46918 caps the request at 2 | + +The four other fixes this config needed on the v0.5.22 nightly are upstream and in the pinned +nightly, so they are no longer patched: sglang +[#33059](https://github.com/sgl-project/sglang/pull/33059) (DSA indexer DP-padded rows on HIP), +[#46830](https://github.com/sgl-project/sglang/pull/46830) (no host syncs on rank-divergent DSA paths), +[#46820](https://github.com/sgl-project/sglang/pull/46820) (non-blocking KV metrics) and +[#21194](https://github.com/sgl-project/sglang/pull/21194) (PP middle stage without an embedding). Not patched, by reading the same source: the DSA page-table rows (the recipe's `index_share_for_mtp_iteration: false` keeps page-table and top-k rows equal), the non-final-chunk KV @@ -41,17 +44,11 @@ version, and `infera.engine.sglang` starts SGLang's own `launch_server` unmodifi ## Why the unmodified upstream image cannot run this benchmark -- B7: the TP1 × PP4 prefill does not load — PP rank 1 dies with - `AttributeError: 'PPMissingLayer' object has no attribute 'embedding_dim'`. -- B1, B2: the DP-attention decode either fails in the indexer (`Expected lengths.size(0) == B`) or - hangs with every DP rank stopped, as soon as ranks carry unequal batches. -- B6: under the AgentX load the prefill scheduler blocks on the metrics socket (py-spy capture of a - wedged GLM-5.2 prefill, 2026-10-09) and the point fails on stalled requests. -- B8: the first long prefill aborts in Triton's LLVM backend. +- B8: a prefill batch that gets the 4-wave config (193-256 new tokens, such as the last chunk of a + long prompt) aborts in Triton's LLVM backend; sglang#46829 reproduces the abort at 256 rows. ## Removal plan -Each fix goes when a pinned ROCm SGLang nightly contains it: drop its file and its line in -`apply.sh` in the PR that bumps the `image:` of `glm5.2-fp4-mi355x-infera-sglang-agentic-disagg`. -When the last one is gone, delete `infera-sglang-patches/`, the patch step in `infera-sglang.sh` and -this waiver in the same PR. +When a pinned ROCm SGLang nightly contains #46918 (or another fix for #46829), the PR that bumps the +`image:` of `glm5.2-fp4-mi355x-infera-sglang-agentic-disagg` deletes `infera-sglang-patches/`, the +patch step in `infera-sglang.sh` and this waiver. diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py index b6f5261bea..d2866722a0 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py @@ -77,7 +77,7 @@ def test_infera_sglang_fails_when_the_sglang_fixes_do_not_apply(tmp_path): {"ROUTER_VERSION": "9.8.7", "SGLANG_DIR": str(tmp_path / "no-sglang")}) assert result.returncode != 0 - assert "[infera-sglang-patches] B1" in result.stdout + assert "[infera-sglang-patches] B8" in result.stdout assert "amd-infera==9.8.7" in installs[0] From 20faef90724095394cc8615b5829bc8faa0bdb47 Mon Sep 17 00:00:00 2001 From: liyingli Date: Fri, 13 Nov 2026 10:28:57 +0000 Subject: [PATCH 09/12] feat(infera-sglang): serve the stock SGLang image unpatched Drops B8, the last SGLang patch, with its patch step and the waiver. Only valid on a nightly that carries sgl-project/sglang#46918 (issue #46829): without it a 193-256 row absorb BMM on gfx950 requests 4 waves per EU and aborts Triton 3.7.0's LLVM backend. Co-authored-by: Cursor --- .../configs/infera-sglang-patches/apply.sh | 32 --------- .../patch_mla_absorb_bmm_default_config.py | 68 ------------------- .../configs/infera-sglang.sh | 14 ++-- inferencex-e2e/docs/waiver/4629.md | 54 --------------- .../tests/srt_slurm/test_setup_scripts.py | 14 +--- 5 files changed, 5 insertions(+), 177 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py delete mode 100644 inferencex-e2e/docs/waiver/4629.md diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh deleted file mode 100644 index e39ad40772..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# The SGLang fix the infera-sglang GLM-5.2 points still need on the pinned image, applied in the -# container before the server starts. Waiver: docs/waiver/4629.md. The fix is marker-gated and -# idempotent, and the run fails if it did not land. -# -# B8 patch_mla_absorb_bmm_default_config.py sglang#46829 absorb-BMM tile config aborts Triton -set -euo pipefail - -here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SGLANG_DIR="${SGLANG_DIR:-/sgl-workspace/sglang}" -SRT="$SGLANG_DIR/python/sglang/srt" - -echo "[infera-sglang-patches] B8 patch_mla_absorb_bmm_default_config.py" -python3 "$here/patch_mla_absorb_bmm_default_config.py" - -# source file (under sglang/srt) : text only the patched source contains -markers=( - "models/deepseek_common/attention_forward_methods/forward_mla_rocm.py:_infera_absorb_bmm_default" -) -fail=0 -for spec in "${markers[@]}"; do - file="$SRT/${spec%%:*}" marker="${spec#*:}" - rm -f "$(dirname "$file")/__pycache__/$(basename "$file" .py)".*.pyc - if grep -qF -- "$marker" "$file"; then - echo " ok ${spec%%:*}" - else - echo " MISSING ${spec%%:*} :: $marker" >&2 - fail=1 - fi -done -[[ "$fail" == 0 ]] || { echo "[infera-sglang-patches] verification failed" >&2; exit 1; } -echo "[infera-sglang-patches] B8 applied" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py deleted file mode 100644 index 633b2b1513..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/patch_mla_absorb_bmm_default_config.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -"""Let aiter choose the MLA absorb-BMM config on ROCm (forward_mla_rocm.py). - -WHAT: v0.5.21 added `_absorb_bmm_config`, a hand-built Triton tile config that -`forward_mla_rocm.py` passes to aiter's batched FP8 GEMM for up to 256 rows. On a -GLM-5.2 prefill rank it aborted Triton's LLVM backend (Sequence.h:275 -`iota_range` assertion) and took the engine down (yihou agentx44_20261009, R003). -The v0.5.21 deployments ran with the function returning None, i.e. with aiter -picking its own config as on v0.5.19; this script bakes that in instead of the -bind-mounted file those runs used. - -v0.5.22 still carries the same function. The crash was not re-tested there; the -patch is kept so the image matches the v0.5.21 baseline it is compared against. - -Self-locating and idempotent; exits non-zero when the anchor drifted. -Marker: `_infera_absorb_bmm_default`. -""" - -import importlib.util -import py_compile -import sys -from pathlib import Path - -TAG = "[mla-absorb-bmm-default]" -MARKER = "_infera_absorb_bmm_default" -OLD = ( - "def _absorb_bmm_config(heads: int, m: int, n: int) -> Optional[dict]:\n" - ' """Return decode tile settings, or None outside the decode row limit."""\n' -) -NEW = OLD + ( - " # Infera: aiter picks its own config; the hand-built one below aborted\n" - " # Triton's LLVM backend on a GLM-5.2 prefill rank (v0.5.21).\n" - f" return None # {MARKER}\n" -) - - -def main() -> int: - spec = importlib.util.find_spec("sglang") - if not spec or not spec.origin: - print(f"{TAG} cannot locate sglang") - return 1 - path = ( - Path(spec.origin).parent - / "srt" - / "models" - / "deepseek_common" - / "attention_forward_methods" - / "forward_mla_rocm.py" - ) - source = path.read_text() - if MARKER in source: - print(f"{TAG} already present — skipping") - return 0 - if source.count(OLD) != 1: - print(f"{TAG} anchor not found exactly once in {path} — re-cut the patch") - return 1 - path.write_text(source.replace(OLD, NEW, 1)) - for cached in (path.parent / "__pycache__").glob(path.stem + ".*.pyc"): - cached.unlink() - py_compile.compile(str(path), doraise=True) - print(f"{TAG} patched {path}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh index 986c50e313..8c3aba690f 100755 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh @@ -1,11 +1,9 @@ #!/usr/bin/env bash # Install Infera, at the master config's router version, into the stock SGLang image for the -# infera-sglang points, then apply the SGLang fix the pinned image still lacks -# (infera-sglang-patches/apply.sh, waiver docs/waiver/4629.md). amd-infera's Linux x86_64 -# wheel on PyPI bundles the infera-router binary next to the infera.engine.sglang worker -# wrapper, so the master's router version is the Infera version. Only nats-py is missing from -# the lmsysorg ROCm image; --no-deps keeps every package the image ships (torch, sglang, -# transformers, ...) at its pinned version. +# infera-sglang points. amd-infera's Linux x86_64 wheel on PyPI bundles the infera-router +# binary next to the infera.engine.sglang worker wrapper, so the master's router version is the +# Infera version. Only nats-py is missing from the lmsysorg ROCm image; --no-deps keeps every +# package the image ships (torch, sglang, transformers, ...) at its pinned version. set -eo pipefail source /infmax-workspace/benchmarks/check_env.sh check_env_vars ROUTER_VERSION @@ -19,7 +17,3 @@ fi # The router the launcher will exec: the wheel's bundled binary, not one on PATH. Fails if pip # fell back to the pure-Python wheel (a platform outside manylinux_2_28 x86_64). python3 -c "import infera.engine.sglang; from infera.server.launch_rust import _find_binary; _find_binary()" -# INFERA_SGLANG_PATCHES=0 only for the installer unit test, which has no SGLang tree. -if [[ "${INFERA_SGLANG_PATCHES:-1}" != 0 ]]; then - bash "$(dirname "${BASH_SOURCE[0]}")/infera-sglang-patches/apply.sh" -fi diff --git a/inferencex-e2e/docs/waiver/4629.md b/inferencex-e2e/docs/waiver/4629.md deleted file mode 100644 index ec123e249a..0000000000 --- a/inferencex-e2e/docs/waiver/4629.md +++ /dev/null @@ -1,54 +0,0 @@ -# Inference-engine patch waiver — PR #4629 - -Filed per [`docs/PR_REVIEW_CHECKLIST.md`](../PR_REVIEW_CHECKLIST.md) (image-provenance / patch item): the -setup script of the `infera-sglang` points patches the pinned SGLang image before serving, which the -checklist prohibits unless covered by a filled-out waiver. - -## Config covered - -- **Master config entry:** `glm5.2-fp4-mi355x-infera-sglang-agentic-disagg` in - [`configs/amd-master.yaml`](../../configs/amd-master.yaml) (all three P/D shapes: 1P1D, 1P1D with a - 4-GPU decode, 2P1D) -- **Pinned image:** `lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018@sha256:8f7d8034546b17cc4ae57c63f3f65c584eb73e62378243d1416b8385aa7be286` - (SGLang source `e9b22a08`) -- **Patch entrypoint:** the recipe's `setup_script: infera-sglang.sh` - ([`benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh`](../../benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh)), - which srtctl runs in every worker and router container; after installing Infera it runs -- **Patch script:** [`benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh`](../../benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang-patches/apply.sh) - -## What is patched - -`apply.sh` edits the image's SGLang source in place (`/sgl-workspace/sglang`, the tree the image's -`sglang` package is installed from). The fix is anchored on the exact source it replaces, is -idempotent, and is checked by a marker afterwards; if it does not apply, the job fails instead -of serving unpatched. - -| | Upstream PR/issue | What it changes | Why this config needs it | -|---|---|---|---| -| B8 | sglang issue [#46829](https://github.com/sgl-project/sglang/issues/46829), fix [#46918](https://github.com/sgl-project/sglang/pull/46918) (open) | Drop the hand-written Triton tile config of the MLA absorb BMM and let aiter pick its own (`models/deepseek_common/attention_forward_methods/forward_mla_rocm.py`) | GLM-5.2 prefill. For 193-256 rows `_absorb_bmm_config` (`:174`) asks for 4 waves per EU, which trips an LLVM assertion in Triton 3.7.0 on gfx950, the Triton this image ships; reproduced on 10-27. #46918 caps the request at 2 | - -The four other fixes this config needed on the v0.5.22 nightly are upstream and in the pinned -nightly, so they are no longer patched: sglang -[#33059](https://github.com/sgl-project/sglang/pull/33059) (DSA indexer DP-padded rows on HIP), -[#46830](https://github.com/sgl-project/sglang/pull/46830) (no host syncs on rank-divergent DSA paths), -[#46820](https://github.com/sgl-project/sglang/pull/46820) (non-blocking KV metrics) and -[#21194](https://github.com/sgl-project/sglang/pull/21194) (PP middle stage without an embedding). - -Not patched, by reading the same source: the DSA page-table rows (the recipe's -`index_share_for_mtp_iteration: false` keeps page-table and top-k rows equal), the non-final-chunk KV -wait event (only the overlap-scheduler path differs; PP prefill runs `--disable-overlap-schedule`), -and HiCache staged write-back on ROCm (the controller moves backup indices per pool on this source). - -Nothing else in the image changes: Infera is installed with `--no-deps` at the master's router -version, and `infera.engine.sglang` starts SGLang's own `launch_server` unmodified. - -## Why the unmodified upstream image cannot run this benchmark - -- B8: a prefill batch that gets the 4-wave config (193-256 new tokens, such as the last chunk of a - long prompt) aborts in Triton's LLVM backend; sglang#46829 reproduces the abort at 256 rows. - -## Removal plan - -When a pinned ROCm SGLang nightly contains #46918 (or another fix for #46829), the PR that bumps the -`image:` of `glm5.2-fp4-mi355x-infera-sglang-agentic-disagg` deletes `infera-sglang-patches/`, the -patch step in `infera-sglang.sh` and this waiver. diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py index d2866722a0..0e59a6d27f 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_setup_scripts.py @@ -60,8 +60,7 @@ def test_an_installer_fails_without_the_version_the_binder_writes(tmp_path, scri ("vllm-mooncake.sh", {"KV_OFFLOAD_BACKEND_VERSION": "9.8.7"}, {"mooncake-transfer-engine-cuda13==9.8.7"}), ("glm5.3-tilert-rocm.sh", {"ROUTER_VERSION": "9.8.7", "TILERT_ROLE": "prefill"}, {"tilert==9.8.7"}), - ("infera-sglang.sh", {"ROUTER_VERSION": "9.8.7", "INFERA_SGLANG_PATCHES": "0"}, - {"--no-deps", "amd-infera==9.8.7"}), + ("infera-sglang.sh", {"ROUTER_VERSION": "9.8.7"}, {"--no-deps", "amd-infera==9.8.7"}), ]) # fmt: skip def test_an_installer_installs_exactly_the_bound_version(tmp_path, script, env, expected): result, installs = run(tmp_path, script, env) @@ -70,17 +69,6 @@ def test_an_installer_installs_exactly_the_bound_version(tmp_path, script, env, assert expected <= set(installs[0]) -def test_infera_sglang_fails_when_the_sglang_fixes_do_not_apply(tmp_path): - """The installer patches SGLang after installing Infera and never skips it silently: - without an SGLang tree the patch step, and so the setup script, fails.""" - result, installs = run(tmp_path, "infera-sglang.sh", - {"ROUTER_VERSION": "9.8.7", "SGLANG_DIR": str(tmp_path / "no-sglang")}) - - assert result.returncode != 0 - assert "[infera-sglang-patches] B8" in result.stdout - assert "amd-infera==9.8.7" in installs[0] - - def test_tokenspeed_installer_uses_direct_private_wheels_and_one_public_index(tmp_path): result, installs = run(tmp_path, "tokenspeed-nightly.sh", { "TOKENSPEED_WHEEL_URL": "https://example.test/tokenspeed.whl#sha256=abc", From 3bbcdd28f5b8a6780ca1a56d54b6c2be38f142a6 Mon Sep 17 00:00:00 2001 From: liyingli Date: Sat, 14 Nov 2026 18:06:41 +0000 Subject: [PATCH 10/12] feat(glm5.2): pin the MI355X infera-sglang curve to the v0.5.23 ROCm 10 nightly lmsysorg/sglang-rocm:v0.5.23-rocm10-mi35x-20261112 (SGLang cabb43af) carries sgl-project/sglang#33059, #46830, #46820 and #21194, and its Triton 3.8.0 compiles the absorb-BMM hand config that aborts Triton 3.7.0 on gfx950 (issue #46829), so the curve runs the image unpatched. perf-changelog gets the PR link. Co-authored-by: Cursor --- .../infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml | 4 ++-- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml index 1ebcc1518c..8d5c3dd7f2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/infera-sglang/mi355x-fp4/agentx/disagg-variants.yaml @@ -64,8 +64,8 @@ base: MC_GID_INDEX: "@fabric.rdma-gid-index" # Rail-isolated fabric: each transfer uses the NIC of the same name on both ends. MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # Register VRAM the peer way, not through dma-buf, as ATOM P/D does on this cluster - # with the same Mooncake (78566d45); 1 GiB regions keep every registration small. + # Register VRAM the peer way, not through dma-buf, as ATOM P/D does on this cluster; + # 1 GiB regions keep every registration small. MOONCAKE_DISABLE_HIP_DMABUF: "1" MC_MAX_MR_SIZE: "1073741824" RDMAV_FORK_SAFE: "1" diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 75043dd4f3..3ec8b63fac 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -786,7 +786,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: # (2) 1P1D, 4-GPU decode (8 GPUs), conc [64, 80]; # (3) 2P1D, two prefills on one node and an 8-GPU decode (16 GPUs), conc [176]. glm5.2-fp4-mi355x-infera-sglang-agentic-disagg: - image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018@sha256:8f7d8034546b17cc4ae57c63f3f65c584eb73e62378243d1416b8385aa7be286 + image: lmsysorg/sglang-rocm:v0.5.23-rocm10-mi35x-20261112@sha256:d62fd730ec3f962073bd0e303942f207c178a0d953ee7c9ef9760624d5d532f0 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a4a1fb1e65..f25fb2205b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12849,6 +12849,6 @@ scenario-type: - agentic-coding description: - - "Add GLM-5.2 MXFP4 AgentX on MI355X with stock SGLang (lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261018) prefill/decode disaggregation behind the Infera router (framework infera-sglang, infera-router 0.3.3 installed by infera-sglang.sh). Workers start through infera.engine.sglang and register in etcd; the router routes kv-aware with prefill session affinity and decode input demand." + - "Add GLM-5.2 MXFP4 AgentX on MI355X with stock, unpatched SGLang (lmsysorg/sglang-rocm:v0.5.23-rocm10-mi35x-20261112, pinned by digest) prefill/decode disaggregation behind the Infera router (framework infera-sglang, infera-router 0.3.3 installed by infera-sglang.sh). Workers start through infera.engine.sglang and register in etcd; the router routes kv-aware with prefill session affinity and decode input demand." - "Prefill TP1 x PP4 (layers 20/20/20/18) with write-back HiCache at 250 GB per stage; decode TP + DP attention; EAGLE MTP 5/1/6 on both roles; Mooncake KV transfer. Shapes: 1P1D with an 8-GPU decode at concurrency 32, 48, 64, 88 (decode request weight 1,000,000); 1P1D with a 4-GPU decode at 64, 80; 2P1D at 176." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4629 From fabd6a8ee59baac6e5f7fd2930ed90b137e48713 Mon Sep 17 00:00:00 2001 From: liyingli Date: Tue, 17 Nov 2026 05:05:21 +0000 Subject: [PATCH 11/12] fix(infera-sglang): lift RLIMIT_MEMLOCK in the worker's shell before it starts The ionic RDMA driver charges each KV registration against the soft RLIMIT_MEMLOCK before falling back to peer memory, and the remapped container root has no CAP_IPC_LOCK. With the 8 GiB soft limit Slurm propagates, Mooncake's concurrent 1 GiB registrations fail with ENOMEM and the decode exits before it is ready. The installer raises the soft limit to the hard limit in its parent shell, which starts the worker. AMD-AGI/Infera#239 moves this into Infera's wrapper; drop it once the router version carries that. Co-authored-by: Cursor --- .../srt-slurm-recipes/configs/infera-sglang.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh index 8c3aba690f..f974ca6991 100755 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/configs/infera-sglang.sh @@ -17,3 +17,17 @@ fi # The router the launcher will exec: the wheel's bundled binary, not one on PATH. Fails if pip # fell back to the pure-Python wheel (a platform outside manylinux_2_28 x86_64). python3 -c "import infera.engine.sglang; from infera.server.launch_rust import _find_binary; _find_binary()" +# Lift RLIMIT_MEMLOCK's soft limit to the hard limit in the shell that starts the worker +# ($PPID: srtctl runs this script, then the worker, from one shell). The RDMA driver charges +# every KV registration against the soft limit before falling back to peer memory, and the +# remapped container root has no CAP_IPC_LOCK, so the 8 GiB limit Slurm propagates fails +# Mooncake's concurrent 1 GiB registrations with ENOMEM. Infera does this itself from +# AMD-AGI/Infera#239; drop this once the router version carries it. +python3 - "$PPID" <<'PY' +import resource, sys +pid = int(sys.argv[1]) +soft, hard = resource.prlimit(pid, resource.RLIMIT_MEMLOCK) +if soft != hard: + resource.prlimit(pid, resource.RLIMIT_MEMLOCK, (hard, hard)) + print(f"RLIMIT_MEMLOCK of pid {pid}: soft {soft} -> {hard}") +PY From 89594d7f515e491af31634ba86dd790aee1606e1 Mon Sep 17 00:00:00 2001 From: liyingli Date: Tue, 17 Nov 2026 05:47:54 +0000 Subject: [PATCH 12/12] chore(srt-slurm patch): refresh the infera frontend to NVIDIA/srt-slurm#630 at 290bc0de 290bc0de adds b46f539e (the worker environment sets INFERA_DEFAULT_KV_FP8=0, so the KV-cache dtype is the recipe's) and a merge of srt-slurm main. On the pin 2d9079c the new patch gives the same src/srtctl as the previous one plus b46f539e, and applies in checkout order (573, 630, 900). This recipe sets kv-cache-dtype fp8_e4m3 explicitly, so its launch is unchanged. Co-authored-by: Cursor --- .../patches/630-infera-frontend.patch | 23 +++++++++++-------- .../runners/srt-slurm/patches/README.md | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch index a6a26cca2b..58a9ecba0b 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch +++ b/inferencex-e2e/runners/srt-slurm/patches/630-infera-frontend.patch @@ -62,10 +62,10 @@ index 80c8fb4..a24ab17 100644 env.update(lifecycle_otel_env(self.config.services, self.runtime.head_node_ip, process.endpoint_mode)) env.update( diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py -index 6a1b0bd..7660755 100755 +index 7b3b50a..45d1ff9 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py -@@ -2629,7 +2629,8 @@ class FrontendConfig: +@@ -2640,7 +2640,8 @@ class FrontendConfig: """Frontend/router configuration. Attributes: @@ -172,10 +172,10 @@ index 22e749f..d47697c 100644 from __future__ import annotations diff --git a/src/srtctl/frontends/infera.py b/src/srtctl/frontends/infera.py new file mode 100644 -index 0000000..77d78d0 +index 0000000..c75c1f9 --- /dev/null +++ b/src/srtctl/frontends/infera.py -@@ -0,0 +1,319 @@ +@@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + @@ -235,6 +235,9 @@ index 0000000..77d78d0 +WORKER_LAUNCHERS = {"sglang": "infera.engine.sglang"} +# The launcher's readiness listener port; unset, it binds a fixed default. +READINESS_PORT_ENV = "INFERA_READINESS_PORT" ++# Unless this is "0", the launcher sets SGLang's KV cache to fp8_e4m3 whenever the ++# command line has no --kv-cache-dtype. ++DEFAULT_KV_FP8_ENV = "INFERA_DEFAULT_KV_FP8" +# The release setup_script installs (configs/install-infera.sh); the router and the +# workers must run the same one. +VERSION_ENV = "INFERA_VERSION" @@ -384,13 +387,15 @@ index 0000000..77d78d0 + return args + + def worker_environment(self, process: Process, runtime: RuntimeContext) -> dict[str, str]: -+ """The readiness listener the launcher opens once registered: the process's system port. ++ """The launcher's readiness port, and its fp8 KV-cache default turned off. + -+ srtctl reads readiness from the router, and the system port is otherwise -+ unused without Dynamo, so co-located workers do not contend for Infera's -+ default. ++ The readiness listener the launcher opens once registered is the process's ++ system port: srtctl reads readiness from the router, and the system port is ++ otherwise unused without Dynamo, so co-located workers do not contend for ++ Infera's default. With the fp8 default off, the KV-cache dtype is the recipe's ++ (``kv-cache-dtype`` in the role's args), as under any other frontend. + """ -+ return {READINESS_PORT_ENV: str(process.sys_port)} ++ return {READINESS_PORT_ENV: str(process.sys_port), DEFAULT_KV_FP8_ENV: "0"} + + def parse_health(self, response_json: dict, expected_prefill: int, expected_decode: int) -> WorkerHealthResult: + """Count active registrations by role; an aggregate (``mixed``) worker counts as decode. diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index e0b89117ec..e8bc442914 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,5 +9,5 @@ Each patch backports an upstream fix that the shared submodule pin does not incl | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exclusive node's exporter (DCGM or AMD) reports every GPU, which the collector otherwise flags as `unexpected_device`. The pin includes #573, but upstream merged a different design: it keeps every row and excuses idle worker-node GPUs only after a DCGM worker GPU UUID probe. [`infx/results/power/multinode.py`](../../../infx/results/power/multinode.py) does not apply that excusal yet, so keep this filter until it does. | -| `630-infera-frontend.patch` | [NVIDIA/srt-slurm#630](https://github.com/NVIDIA/srt-slurm/pull/630) (open) | `frontend.type: infera`: SGLang workers start through Infera's `infera.engine.sglang`, register in the job's etcd and are discovered by `infera-router` (the `infera-sglang` framework). Adds the `Frontend.worker_module` / `worker_launcher_args` / `worker_environment` hooks and `Backend.get_kv_events_config_for_mode`; the patch is the PR's `src/srtctl` diff at `72c4d04`. | +| `630-infera-frontend.patch` | [NVIDIA/srt-slurm#630](https://github.com/NVIDIA/srt-slurm/pull/630) (open) | `frontend.type: infera`: SGLang workers start through Infera's `infera.engine.sglang`, register in the job's etcd and are discovered by `infera-router` (the `infera-sglang` framework). Adds the `Frontend.worker_module` / `worker_launcher_args` / `worker_environment` hooks and `Backend.get_kv_events_config_for_mode`; the patch is the PR's `src/srtctl` diff at `290bc0de`. | | `900-pool-expected-devices.patch` | pending | Plan every service pool node's `gpus_per_node` GPUs as expected power-telemetry devices under the service's name (`worker_role=`, `worker_index=`), so a services-only recipe (`frontend.type: none`, one `services[].nodes` pool, `benchmark.type: custom`) reaches collector readiness and publishes instead of failing with `exporter_startup_timeout` + `expected_device_missing` + `unexpected_device`. A services-only job records `worker_identity_proof.error = "no engine workers to probe"` instead of a probe traceback, the identity gate only scopes engine-role devices, and the offline validator accepts pool roles. Engine-only manifests are unchanged. With `573-participating-gpus.patch` an exclusive pool node's unplanned exporter rows are dropped at scrape time, so a recipe whose `gpus_per_node` is below the node's physical GPU count still validates; upstream alone reports them as `unexpected_device`. |