From 16c29fea5a69772ebeb3088c8ed984cc3963c4bc Mon Sep 17 00:00:00 2001 From: Managed user kimbo Date: Mon, 19 Oct 2026 18:34:01 +0000 Subject: [PATCH 1/5] =?UTF-8?q?feat(speedbench):=20add=20MiMo-V2.6-Flash?= =?UTF-8?q?=20SGLang=20MTP=20AL=20collector=20/=20feat(speedbench)?= =?UTF-8?q?=EF=BC=9A=E6=96=B0=E5=A2=9E=20MiMo-V2.6-Flash=20SGLang=20MTP=20?= =?UTF-8?q?AL=20=E9=87=87=E9=9B=86=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures SPEED-Bench coding acceptance length for MiMo-V2.6-Flash's bundled MTP layers (SGLang multi-layer EAGLE, topk 1, depths 1-3) on B300, thinking on, for the golden AL curve mimov2.6flash_mtp.yaml. 在 B300 上测量 MiMo-V2.6-Flash 自带 MTP 层（SGLang 多层 EAGLE， topk 1，深度 1-3）在 SPEED-Bench coding 类别上的接受长度（开启思考）， 用于生成 golden AL 曲线 mimov2.6flash_mtp.yaml。 Co-Authored-By: Claude Opus 5.5 --- .../mimov2.6flash_fp4_b300_sglang.sh | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100755 inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh new file mode 100755 index 0000000000..4747c7ea6d --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash + +# MiMo-V2.6-Flash B300 SGLang SPEED-Bench AL matrix collector for the checkpoint's native MTP. +# +# For each MTP depth, measure the REAL acceptance length (AL) on one SPEED-Bench category and +# emit a YAML matrix in the golden_al_distribution shape. The server matches the AgentX +# serving setup (TP4, fa4, text-only, mimo parsers). +# +# SGLang serves MiMoV2 MTP as multi-layer EAGLE: one bundled MTP layer per draft step, linear +# chain (topk 1), so MTP_LIST level N runs N steps with N + 1 draft tokens and stops at the +# checkpoint's num_nextn_predict_layers. +# SGLang's speed-bench loader applies the chat template client-side without kwargs, so every +# cell runs the template default (thinking on), which is the mode AgentX selects. +# +# AL is SGLang's accept_length (accepted tokens per verify step, bonus token included, +# cumulative since server start), the same quantity as vLLM's 1 + accepted/drafts. Each cell +# restarts the server, so it covers one cell only. +# +# Dispatch this collector through speedbench-al.yml with framework=sglang. + +set -o pipefail +repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" +source "$repo_root/benchmarks/check_env.sh" +check_env_vars \ + CATEGORY MODEL MODEL_PATH MTP_LIST OUT_YAML PORT SPEEDBENCH_OUTPUT_LEN THINKING_MODES + +if [[ "$THINKING_MODES" != "on" ]]; then + echo "CRITICAL: the SGLang speed-bench client cannot disable MiMo thinking; dispatch with thinking-modes=on (got '$THINKING_MODES')" + exit 1 +fi + +SERVE_MODEL="${MODEL_PATH}" +MODEL_KEY="$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')" +# AL is a per-draft accept/reject property independent of batch size, so batch the +# SPEED-Bench pass to cut wall-clock. +CONCURRENCY="32" +TEMPERATURE="1.0" +TOP_P="0.95" + +SPEEDBENCH_DIR="/workspace/speed_bench_data" +RESULTS_DIR="/workspace/speedbench_results" + +mkdir -p "$RESULTS_DIR" +nvidia-smi +GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" + +if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + if [[ ! -w "$(dirname "$MODEL_PATH")" ]]; then + echo "CRITICAL: $MODEL_PATH is empty and $(dirname "$MODEL_PATH") is not writable." + echo "The cluster's models.entries names this checkpoint but the weights were never staged." + exit 1 + fi + echo "=== $MODEL_PATH is empty; downloading $MODEL (first run only) ===" + hf download "$MODEL" --local-dir "$MODEL_PATH" +fi + +MTP_LAYERS=$(python3 -c ' +import json, sys +print(json.load(open(sys.argv[1]))["num_nextn_predict_layers"]) +' "$MODEL_PATH/config.json") +for mtp in $MTP_LIST; do + if [[ "$mtp" -lt 1 || "$mtp" -gt "$MTP_LAYERS" ]]; then + echo "CRITICAL: MTP level $mtp is outside 1..$MTP_LAYERS (num_nextn_predict_layers)" + exit 1 + fi +done + +echo "=== Downloading SPEED-Bench dataset ===" +pip install -q datasets tiktoken +curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ + | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" + +if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then + echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" + exit 1 +fi + +# SGLang's speed-bench loader only accepts the throughput-split categories on the CLI and +# reads prompts from "turns", while prepare.py writes "messages". Filter the qualitative +# split to $CATEGORY and restore "turns" from the user messages. +CATEGORY_JSONL="$SPEEDBENCH_DIR/qualitative_${CATEGORY}.jsonl" +python3 - "$SPEEDBENCH_DIR/qualitative.jsonl" "$CATEGORY_JSONL" "$CATEGORY" <<'EOF' +import json, sys +src, dst, category = sys.argv[1:] +with open(src) as f, open(dst, "w") as out: + for line in f: + row = json.loads(line) + if row.get("category") == category: + row["turns"] = [m["content"] for m in row["messages"] if m["role"] == "user"] + out.write(json.dumps(row) + "\n") +EOF +NUM_PROMPTS=$(wc -l < "$CATEGORY_JSONL") +if [[ "$NUM_PROMPTS" -eq 0 ]]; then + echo "CRITICAL: no SPEED-Bench qualitative rows for category '$CATEGORY'" + exit 1 +fi + +SERVER_PID="" +_descendants() { + local pid="$1" child + for child in $(pgrep -P "$pid" 2>/dev/null || true); do + echo "$child" + _descendants "$child" + done +} +cleanup_server() { + if [[ -n "$SERVER_PID" ]]; then + local descendants + descendants=$(_descendants "$SERVER_PID") + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + local pid + for pid in $descendants; do + kill -9 "$pid" 2>/dev/null || true + done + local waited=0 + while [[ $waited -lt 120 ]]; do + local used + used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) + if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi + sleep 3; waited=$((waited + 3)) + done + SERVER_PID="" + fi +} +trap 'cleanup_server' EXIT + +declare -A AL_RESULT + +run_cell() { + local mode="$1" mtp="$2" + + echo "" + echo "==========================================" + echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" + echo "==========================================" + + local serve_args=( + --model-path "$SERVE_MODEL" + --host 0.0.0.0 --port "$PORT" + --tp-size 4 + --trust-remote-code + # The default trtllm_mha rejects MiMoV2's asymmetric K/V head dims. + --attention-backend fa4 + --language-model-only + --mem-fraction-static 0.8 + --reasoning-parser mimo + --tool-call-parser mimo + # MiMoV2 is not in SGLang's bundled-draft list, so name the target checkpoint. + --speculative-algorithm EAGLE + --speculative-draft-model-path "$SERVE_MODEL" + --enable-multi-layer-eagle + --speculative-num-steps "$mtp" + --speculative-eagle-topk 1 + --speculative-num-draft-tokens "$((mtp + 1))" + ) + + local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" + python3 -m sglang.launch_server "${serve_args[@]}" > "$server_log" 2>&1 & + SERVER_PID=$! + + if ! PYTHONSAFEPATH=1 PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" python3 -m infx.bench wait \ + --url "http://0.0.0.0:${PORT}/health" --pid "$SERVER_PID" --log "$server_log"; then + echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" + AL_RESULT["${mode}_${mtp}"]="N/A" + cleanup_server + return + fi + + local result_file="$RESULTS_DIR/speedbench_${mode}_mtp${mtp}.jsonl" + python3 -m sglang.bench_serving \ + --backend sglang \ + --port "$PORT" \ + --model "$SERVE_MODEL" \ + --dataset-name speed-bench \ + --dataset-path "$CATEGORY_JSONL" \ + --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ + --num-prompts "$NUM_PROMPTS" \ + --max-concurrency "$CONCURRENCY" \ + --temperature "$TEMPERATURE" \ + --top-p "$TOP_P" \ + --disable-ignore-eos \ + --output-file "$result_file" \ + --output-details + + local al + al=$(python3 -c ' +import json, sys +with open(sys.argv[1]) as f: + al = json.loads(f.readlines()[-1]).get("accept_length") +print(f"{al:.2f}" if al else "N/A") +' "$result_file" 2>/dev/null || echo "N/A") + echo " -> thinking=$mode MTP=$mtp AL=$al" + AL_RESULT["${mode}_${mtp}"]="$al" + + cleanup_server +} + +for mtp in $MTP_LIST; do + run_cell on "$mtp" +done + +{ + echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." + echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" + echo "# thinking_on: chat template default" + echo "# draft: bundled MTP layers (multi-layer EAGLE, topk 1)" + echo "# Measured on $MODEL_KEY ($GPU_NAME, SGLang MTP), per num_speculative_tokens." + echo "# Auto-generated by benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh (speedbench-al.yml)." + echo "#" + echo "# key = num_speculative_tokens (MTP steps); value = golden AL" + echo "${MODEL_KEY}:" + echo " thinking_on:" + for mtp in $MTP_LIST; do + echo " $mtp: ${AL_RESULT[on_${mtp}]}" + done +} > "$OUT_YAML" + +echo "" +echo "==========================================" +echo " SPEED-Bench AL matrix written to: $OUT_YAML" +echo "==========================================" +cat "$OUT_YAML" From f2116366499b3bb6a2709b0892f23005c8d31ab9 Mon Sep 17 00:00:00 2001 From: Managed user kimbo Date: Mon, 19 Oct 2026 19:47:29 +0000 Subject: [PATCH 2/5] =?UTF-8?q?feat(speedbench):=20switch=20MiMo-V2.6-Flas?= =?UTF-8?q?h=20AL=20collector=20to=20vLLM=20DFlash=20/=20feat(speedbench)?= =?UTF-8?q?=EF=BC=9AMiMo-V2.6-Flash=20AL=20=E9=87=87=E9=9B=86=E6=94=B9?= =?UTF-8?q?=E7=94=A8=20vLLM=20DFlash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the SGLang MTP collector with a vLLM v0.31.0 collector for the checkpoint's BF16 dflash/ drafter: draft lengths 1-7, thinking off and on via enable_thinking, for the golden curve mimov2.6flash_dflash.yaml. 用 vLLM v0.31.0 采集脚本替换 SGLang MTP 采集脚本，测量检查点自带的 BF16 dflash/ 草稿模型：草稿长度 1-7，通过 enable_thinking 分别关闭和 开启思考，用于 golden 曲线 mimov2.6flash_dflash.yaml。 Co-Authored-By: Claude Opus 5.5 --- .../mimov2.6flash_fp4_b300_sglang.sh | 223 ---------------- .../speedbench/mimov2.6flash_fp4_b300_vllm.sh | 238 ++++++++++++++++++ 2 files changed, 238 insertions(+), 223 deletions(-) delete mode 100755 inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh create mode 100755 inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh deleted file mode 100755 index 4747c7ea6d..0000000000 --- a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env bash - -# MiMo-V2.6-Flash B300 SGLang SPEED-Bench AL matrix collector for the checkpoint's native MTP. -# -# For each MTP depth, measure the REAL acceptance length (AL) on one SPEED-Bench category and -# emit a YAML matrix in the golden_al_distribution shape. The server matches the AgentX -# serving setup (TP4, fa4, text-only, mimo parsers). -# -# SGLang serves MiMoV2 MTP as multi-layer EAGLE: one bundled MTP layer per draft step, linear -# chain (topk 1), so MTP_LIST level N runs N steps with N + 1 draft tokens and stops at the -# checkpoint's num_nextn_predict_layers. -# SGLang's speed-bench loader applies the chat template client-side without kwargs, so every -# cell runs the template default (thinking on), which is the mode AgentX selects. -# -# AL is SGLang's accept_length (accepted tokens per verify step, bonus token included, -# cumulative since server start), the same quantity as vLLM's 1 + accepted/drafts. Each cell -# restarts the server, so it covers one cell only. -# -# Dispatch this collector through speedbench-al.yml with framework=sglang. - -set -o pipefail -repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" -source "$repo_root/benchmarks/check_env.sh" -check_env_vars \ - CATEGORY MODEL MODEL_PATH MTP_LIST OUT_YAML PORT SPEEDBENCH_OUTPUT_LEN THINKING_MODES - -if [[ "$THINKING_MODES" != "on" ]]; then - echo "CRITICAL: the SGLang speed-bench client cannot disable MiMo thinking; dispatch with thinking-modes=on (got '$THINKING_MODES')" - exit 1 -fi - -SERVE_MODEL="${MODEL_PATH}" -MODEL_KEY="$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')" -# AL is a per-draft accept/reject property independent of batch size, so batch the -# SPEED-Bench pass to cut wall-clock. -CONCURRENCY="32" -TEMPERATURE="1.0" -TOP_P="0.95" - -SPEEDBENCH_DIR="/workspace/speed_bench_data" -RESULTS_DIR="/workspace/speedbench_results" - -mkdir -p "$RESULTS_DIR" -nvidia-smi -GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" - -if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - if [[ ! -w "$(dirname "$MODEL_PATH")" ]]; then - echo "CRITICAL: $MODEL_PATH is empty and $(dirname "$MODEL_PATH") is not writable." - echo "The cluster's models.entries names this checkpoint but the weights were never staged." - exit 1 - fi - echo "=== $MODEL_PATH is empty; downloading $MODEL (first run only) ===" - hf download "$MODEL" --local-dir "$MODEL_PATH" -fi - -MTP_LAYERS=$(python3 -c ' -import json, sys -print(json.load(open(sys.argv[1]))["num_nextn_predict_layers"]) -' "$MODEL_PATH/config.json") -for mtp in $MTP_LIST; do - if [[ "$mtp" -lt 1 || "$mtp" -gt "$MTP_LAYERS" ]]; then - echo "CRITICAL: MTP level $mtp is outside 1..$MTP_LAYERS (num_nextn_predict_layers)" - exit 1 - fi -done - -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# SGLang's speed-bench loader only accepts the throughput-split categories on the CLI and -# reads prompts from "turns", while prepare.py writes "messages". Filter the qualitative -# split to $CATEGORY and restore "turns" from the user messages. -CATEGORY_JSONL="$SPEEDBENCH_DIR/qualitative_${CATEGORY}.jsonl" -python3 - "$SPEEDBENCH_DIR/qualitative.jsonl" "$CATEGORY_JSONL" "$CATEGORY" <<'EOF' -import json, sys -src, dst, category = sys.argv[1:] -with open(src) as f, open(dst, "w") as out: - for line in f: - row = json.loads(line) - if row.get("category") == category: - row["turns"] = [m["content"] for m in row["messages"] if m["role"] == "user"] - out.write(json.dumps(row) + "\n") -EOF -NUM_PROMPTS=$(wc -l < "$CATEGORY_JSONL") -if [[ "$NUM_PROMPTS" -eq 0 ]]; then - echo "CRITICAL: no SPEED-Bench qualitative rows for category '$CATEGORY'" - exit 1 -fi - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --model-path "$SERVE_MODEL" - --host 0.0.0.0 --port "$PORT" - --tp-size 4 - --trust-remote-code - # The default trtllm_mha rejects MiMoV2's asymmetric K/V head dims. - --attention-backend fa4 - --language-model-only - --mem-fraction-static 0.8 - --reasoning-parser mimo - --tool-call-parser mimo - # MiMoV2 is not in SGLang's bundled-draft list, so name the target checkpoint. - --speculative-algorithm EAGLE - --speculative-draft-model-path "$SERVE_MODEL" - --enable-multi-layer-eagle - --speculative-num-steps "$mtp" - --speculative-eagle-topk 1 - --speculative-num-draft-tokens "$((mtp + 1))" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - python3 -m sglang.launch_server "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! PYTHONSAFEPATH=1 PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" python3 -m infx.bench wait \ - --url "http://0.0.0.0:${PORT}/health" --pid "$SERVER_PID" --log "$server_log"; then - echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local result_file="$RESULTS_DIR/speedbench_${mode}_mtp${mtp}.jsonl" - python3 -m sglang.bench_serving \ - --backend sglang \ - --port "$PORT" \ - --model "$SERVE_MODEL" \ - --dataset-name speed-bench \ - --dataset-path "$CATEGORY_JSONL" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts "$NUM_PROMPTS" \ - --max-concurrency "$CONCURRENCY" \ - --temperature "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --disable-ignore-eos \ - --output-file "$result_file" \ - --output-details - - local al - al=$(python3 -c ' -import json, sys -with open(sys.argv[1]) as f: - al = json.loads(f.readlines()[-1]).get("accept_length") -print(f"{al:.2f}" if al else "N/A") -' "$result_file" 2>/dev/null || echo "N/A") - echo " -> thinking=$mode MTP=$mtp AL=$al" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mtp in $MTP_LIST; do - run_cell on "$mtp" -done - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on: chat template default" - echo "# draft: bundled MTP layers (multi-layer EAGLE, topk 1)" - echo "# Measured on $MODEL_KEY ($GPU_NAME, SGLang MTP), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_sglang.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (MTP steps); value = golden AL" - echo "${MODEL_KEY}:" - echo " thinking_on:" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[on_${mtp}]}" - done -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh new file mode 100755 index 0000000000..09c881fa4a --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash + +# MiMo-V2.6-Flash B300 vLLM SPEED-Bench AL matrix collector for the checkpoint's DFlash drafter. +# +# For each thinking mode (on/off) and DFlash draft length (num_speculative_tokens), measure +# the REAL acceptance length (AL) on one SPEED-Bench category and emit a YAML matrix in the +# golden_al_distribution shape. +# +# The drafter is the BF16 dflash/ subfolder of the target checkpoint, served as shipped. Its +# block size of 8 bounds the draft length at 7. Thinking is ON by default in MiMo's chat +# template, so the OFF cell MUST pass enable_thinking:false and the ON cell enable_thinking:true. +# +# AL is 1 + delta(accepted draft tokens) / delta(verification drafts) from vLLM's spec-decode +# counters around each SPEED-Bench pass. +# +# Dispatch this collector through speedbench-al.yml with framework=vllm. + +set -o pipefail +repo_root="$(cd "$(dirname "$0")/../../.." && pwd)" +source "$repo_root/benchmarks/check_env.sh" +check_env_vars \ + CATEGORY CHAT_TEMPLATE_KWARGS_ON MODEL MODEL_PATH MTP_LIST OUT_YAML PORT \ + SPEEDBENCH_OUTPUT_LEN THINKING_MODES + +if ! python3 -c 'import json, sys; sys.exit(json.loads(sys.argv[1]).get("enable_thinking") is not True)' \ + "$CHAT_TEMPLATE_KWARGS_ON"; then + echo "CRITICAL: dispatch with thinking-kwargs='{\"enable_thinking\": true}' (got '$CHAT_TEMPLATE_KWARGS_ON')" + exit 1 +fi + +SERVE_MODEL="${MODEL_PATH}" +DRAFT_MODEL_PATH="$MODEL_PATH/dflash" +MODEL_KEY="$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')" +# AL is a per-draft accept/reject property independent of batch size, so batch the +# SPEED-Bench pass to cut wall-clock. +CONCURRENCY="32" +TEMPERATURE="1.0" +TOP_P="0.95" +CHAT_TEMPLATE_KWARGS_OFF='{"enable_thinking": false}' + +SPEEDBENCH_DIR="/workspace/speed_bench_data" +RESULTS_DIR="/workspace/speedbench_results" + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 + +mkdir -p "$RESULTS_DIR" +nvidia-smi +GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" + +if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + if [[ ! -w "$(dirname "$MODEL_PATH")" ]]; then + echo "CRITICAL: $MODEL_PATH is empty and $(dirname "$MODEL_PATH") is not writable." + echo "The cluster's models.entries names this checkpoint but the weights were never staged." + exit 1 + fi + echo "=== $MODEL_PATH is empty; downloading $MODEL (first run only) ===" + hf download "$MODEL" --local-dir "$MODEL_PATH" +fi +if [[ ! -f "$DRAFT_MODEL_PATH/config.json" ]]; then + echo "CRITICAL: no DFlash drafter at $DRAFT_MODEL_PATH" + exit 1 +fi + +DRAFT_BLOCK=$(python3 -c ' +import json, sys +print(json.load(open(sys.argv[1]))["block_size"]) +' "$DRAFT_MODEL_PATH/config.json") +for mtp in $MTP_LIST; do + if [[ "$mtp" -lt 1 || "$mtp" -ge "$DRAFT_BLOCK" ]]; then + echo "CRITICAL: DFlash draft length $mtp is outside 1..$((DRAFT_BLOCK - 1)) (block_size $DRAFT_BLOCK)" + exit 1 + fi +done + +echo "=== Downloading SPEED-Bench dataset ===" +pip install -q datasets tiktoken +curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ + | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" + +if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then + echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" + exit 1 +fi + +fetch_metric() { + local port="$1" name="$2" + curl -s "http://localhost:${port}/metrics" \ + | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" +} + +SERVER_PID="" +_descendants() { + local pid="$1" child + for child in $(pgrep -P "$pid" 2>/dev/null || true); do + echo "$child" + _descendants "$child" + done +} +cleanup_server() { + if [[ -n "$SERVER_PID" ]]; then + local descendants + descendants=$(_descendants "$SERVER_PID") + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + local pid + for pid in $descendants; do + kill -9 "$pid" 2>/dev/null || true + done + local waited=0 + while [[ $waited -lt 120 ]]; do + local used + used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) + if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi + sleep 3; waited=$((waited + 3)) + done + SERVER_PID="" + fi +} +trap 'cleanup_server' EXIT + +declare -A AL_RESULT + +run_cell() { + local mode="$1" mtp="$2" + local think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_OFF") + if [[ "$mode" == "on" ]]; then + think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") + fi + + echo "" + echo "==========================================" + echo " Cell: thinking=$mode DFLASH=$mtp category=$CATEGORY" + echo "==========================================" + + local serve_args=( + --host 0.0.0.0 --port "$PORT" + --tensor-parallel-size 4 + --trust-remote-code + --no-enable-prefix-caching + --gpu-memory-utilization 0.9 + --max-model-len 32768 + --reasoning-parser mimo + --tool-call-parser mimo + --enable-auto-tool-choice + --speculative-config "{\"method\": \"dflash\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $mtp}" + --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' + ) + + local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" + vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & + SERVER_PID=$! + + if ! PYTHONSAFEPATH=1 PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" python3 -m infx.bench wait \ + --url "http://0.0.0.0:${PORT}/health" --pid "$SERVER_PID" --log "$server_log"; then + echo " -> server failed to start (thinking=$mode dflash=$mtp), recording N/A" + AL_RESULT["${mode}_${mtp}"]="N/A" + cleanup_server + return + fi + + local acc_before drf_before acc_after drf_after + acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") + drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") + + vllm bench serve \ + --model "$SERVE_MODEL" \ + --port "$PORT" \ + --dataset-name speed_bench \ + --dataset-path "$SPEEDBENCH_DIR" \ + --speed-bench-category "$CATEGORY" \ + --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ + --num-prompts -1 \ + --max-concurrency "$CONCURRENCY" \ + --save-result \ + --save-detailed \ + --result-dir "$RESULTS_DIR" \ + --result-filename "speedbench_${mode}_mtp${mtp}" \ + --trust-remote-code \ + --temperature "$TEMPERATURE" \ + --top-p "$TOP_P" \ + "${think_args[@]}" + + acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") + drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") + + local delta_acc delta_drf al + delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") + delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") + if [[ "$delta_drf" -gt 0 ]]; then + al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") + else + al="N/A" + fi + echo " -> thinking=$mode DFLASH=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" + AL_RESULT["${mode}_${mtp}"]="$al" + + cleanup_server +} + +for mode in $THINKING_MODES; do + for mtp in $MTP_LIST; do + run_cell "$mode" "$mtp" + done +done + +emit_mode_block() { + local mode="$1" + for mtp in $MTP_LIST; do + echo " $mtp: ${AL_RESULT[${mode}_${mtp}]}" + done +} + +{ + echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." + echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" + echo "# thinking_on chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" + echo "# thinking_off chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_OFF" + echo "# draft: $MODEL/dflash (BF16, greedy drafting)" + echo "# Measured on $MODEL_KEY ($GPU_NAME, vLLM DFlash), per num_speculative_tokens." + echo "# Auto-generated by benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh (speedbench-al.yml)." + echo "#" + echo "# key = num_speculative_tokens (DFlash draft tokens); value = golden AL" + echo "${MODEL_KEY}:" + if [[ " $THINKING_MODES " == *" on "* ]]; then + echo " thinking_on:" + emit_mode_block on + fi + if [[ " $THINKING_MODES " == *" off "* ]]; then + echo " thinking_off:" + emit_mode_block off + fi +} > "$OUT_YAML" + +echo "" +echo "==========================================" +echo " SPEED-Bench AL matrix written to: $OUT_YAML" +echo "==========================================" +cat "$OUT_YAML" From 094f27d465e53b86cba5df1cd37156986a0f249c Mon Sep 17 00:00:00 2001 From: Managed user kimbo Date: Sun, 1 Nov 2026 20:09:40 +0000 Subject: [PATCH 3/5] =?UTF-8?q?fix(speedbench):=20harden=20the=20MiMo-V2.6?= =?UTF-8?q?-Flash=20vLLM=20AL=20collector=20/=20fix(speedbench)=EF=BC=9A?= =?UTF-8?q?=E5=8A=A0=E5=9B=BA=20MiMo-V2.6-Flash=20vLLM=20AL=20=E9=87=87?= =?UTF-8?q?=E9=9B=86=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the benchmark client through vLLM's Python entrypoint so the Rust client delegation cannot reject --speed-bench-output-len, --save-detailed or --chat-template-kwargs, preflight those flags before any server start, log a failing client exit per cell, and fail the job when no cell yields an AL value. Mirrors the MiniMax-M3.1-Flash collector. 中文：基准客户端改走 vLLM 的 Python 入口，避免 Rust 客户端拒绝 --speed-bench-output-len、--save-detailed 与 --chat-template-kwargs； 在启动任何服务前预检这些参数，逐单元记录客户端异常退出，并在没有任何 单元得到 AL 值时让作业失败。做法与 MiniMax-M3.1-Flash 采集脚本一致。 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BGggnGqhy9dML7NM9ow3Fm --- .../speedbench/mimov2.6flash_fp4_b300_vllm.sh | 64 ++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh index 09c881fa4a..d47fcc5326 100755 --- a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh +++ b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh @@ -44,6 +44,35 @@ RESULTS_DIR="/workspace/speedbench_results" export VLLM_ENGINE_READY_TIMEOUT_S=3600 mkdir -p "$RESULTS_DIR" + +# `vllm bench serve` os.execv's the CLIENT into the Rust vllm-rs binary whenever the +# dataset (speed_bench qualifies) and backend are supported, with no opt-out. The +# Rust flag surface lacks --speed-bench-output-len, --save-detailed and +# --chat-template-kwargs, so every cell would die at argument parsing. Call the +# Python entrypoint directly (vllm.benchmarks.serve.main), as the CLI's own Python +# fallback does. The SERVER keeps the Rust frontend; AL is read from /metrics. +BENCH_DRIVER="$RESULTS_DIR/bench_serve_python.py" +cat > "$BENCH_DRIVER" <<'PYEOF' +# Python-only `vllm bench serve`: bypasses the Rust os.execv delegation in +# vllm/entrypoints/cli/benchmark/serve.py by importing the benchmark directly. +from vllm.benchmarks.serve import add_cli_args, main + +try: # import path moved in newer vLLM + from vllm.utils.argparse_utils import FlexibleArgumentParser +except ImportError: + from vllm.utils import FlexibleArgumentParser + +parser = FlexibleArgumentParser( + description="vllm bench serve (Python entrypoint, no Rust delegation)" +) +add_cli_args(parser) +main(parser.parse_args()) +PYEOF +if [[ ! -s "$BENCH_DRIVER" ]]; then + echo "CRITICAL: could not write the benchmark driver to $BENCH_DRIVER — aborting." + exit 1 +fi + nvidia-smi GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" @@ -82,6 +111,19 @@ if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then exit 1 fi +# Preflight the client flags every cell uses; otherwise a CLI mismatch only shows up +# as an all-N/A matrix after every server start. --help=all prints every flag group. +BENCH_HELP="$(python3 "$BENCH_DRIVER" --help=all 2>&1)" +for flag in --speed-bench-category --speed-bench-output-len --chat-template-kwargs --save-detailed; do + if [[ "$BENCH_HELP" != *"$flag"* ]]; then + echo "CRITICAL: the Python benchmark entrypoint does not support $flag — aborting." + echo "--- python3 $BENCH_DRIVER --help=all ---" + echo "$BENCH_HELP" + exit 1 + fi +done +echo "=== Benchmark client flag preflight OK ===" + fetch_metric() { local port="$1" name="$2" curl -s "http://localhost:${port}/metrics" \ @@ -162,7 +204,7 @@ run_cell() { acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - vllm bench serve \ + python3 "$BENCH_DRIVER" \ --model "$SERVE_MODEL" \ --port "$PORT" \ --dataset-name speed_bench \ @@ -179,6 +221,10 @@ run_cell() { --temperature "$TEMPERATURE" \ --top-p "$TOP_P" \ "${think_args[@]}" + local bench_rc=$? + if [[ $bench_rc -ne 0 ]]; then + echo " -> benchmark client exited rc=$bench_rc (thinking=$mode dflash=$mtp)" + fi acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") @@ -206,7 +252,7 @@ done emit_mode_block() { local mode="$1" for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]}" + echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" done } @@ -236,3 +282,17 @@ echo "==========================================" echo " SPEED-Bench AL matrix written to: $OUT_YAML" echo "==========================================" cat "$OUT_YAML" + +# A matrix where every cell is N/A is a failed collection, not a result: fail the +# job so it is not mistaken for a curve worth reviewing. +MEASURED=0 +for mode in $THINKING_MODES; do + for mtp in $MTP_LIST; do + [[ "${AL_RESULT[${mode}_${mtp}]:-N/A}" != "N/A" ]] && MEASURED=$((MEASURED + 1)) + done +done +if [[ "$MEASURED" -eq 0 ]]; then + echo "CRITICAL: no cell produced an AL value — see the server logs and the" + echo "benchmark client output above." + exit 1 +fi From dcbe590558c1015868251f9f01dcebf00ee2f6c4 Mon Sep 17 00:00:00 2001 From: Managed user kimbo Date: Sun, 1 Nov 2026 20:24:47 +0000 Subject: [PATCH 4/5] =?UTF-8?q?fix(speedbench):=20serve=20the=20MiMo=20AL?= =?UTF-8?q?=20collector=20on=20a=20free=20port=20when=208888=20is=20taken?= =?UTF-8?q?=20/=20fix(speedbench)=EF=BC=9A8888=20=E8=A2=AB=E5=8D=A0?= =?UTF-8?q?=E7=94=A8=E6=97=B6=20MiMo=20AL=20=E9=87=87=E9=9B=86=E6=94=B9?= =?UTF-8?q?=E7=94=A8=E7=A9=BA=E9=97=B2=E7=AB=AF=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 41345416194 lost every cell to "Address already in use" on dsxe-sa-b300-prd0-gpu-00: a process outside the Slurm job held port 8888 in the shared network namespace. Report the holder and fall back to a free port before the first server start. 中文：运行 41345416194 在 dsxe-sa-b300-prd0-gpu-00 上所有单元都因 "Address already in use" 失败：Slurm 作业之外的进程在共享网络命名空间中 占用了 8888 端口。现在在首次启动服务前报告占用者，并改用空闲端口。 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BGggnGqhy9dML7NM9ow3Fm --- .../speedbench/mimov2.6flash_fp4_b300_vllm.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh index d47fcc5326..228682a4e1 100755 --- a/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh +++ b/inferencex-e2e/benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh @@ -124,6 +124,27 @@ for flag in --speed-bench-category --speed-bench-output-len --chat-template-kwar done echo "=== Benchmark client flag preflight OK ===" +# Containers share the node's network namespace, so a process left on the node outside +# Slurm can hold $PORT and every cell then dies with "Address already in use". Report +# the holder and serve on a free port instead. +port_in_use() { + python3 -c ' +import socket, sys +s = socket.socket() +try: + s.bind(("0.0.0.0", int(sys.argv[1]))) +except OSError: + sys.exit(0) +sys.exit(1) +' "$1" +} +if port_in_use "$PORT"; then + echo "WARNING: port $PORT is already in use on $(hostname); holder:" + (ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null) | grep -E ":$PORT\b" || echo " (ss/netstat unavailable)" + PORT=$(python3 -c 'import socket; s = socket.socket(); s.bind(("0.0.0.0", 0)); print(s.getsockname()[1])') + echo "Serving on free port $PORT instead." +fi + fetch_metric() { local port="$1" name="$2" curl -s "http://localhost:${port}/metrics" \ From 814921bc52517847b69a487ded447a04cfe013d5 Mon Sep 17 00:00:00 2001 From: Managed user kimbo Date: Sun, 1 Nov 2026 22:47:34 +0000 Subject: [PATCH 5/5] =?UTF-8?q?feat(golden-al):=20add=20the=20MiMo-V2.6-Fl?= =?UTF-8?q?ash=20DFlash=20golden=20AL=20curve=20/=20feat(golden-al)?= =?UTF-8?q?=EF=BC=9A=E6=96=B0=E5=A2=9E=20MiMo-V2.6-Flash=20DFlash=20?= =?UTF-8?q?=E9=BB=84=E9=87=91=20AL=20=E6=9B=B2=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit infx/golden_al_distribution/mimov2.6flash_dflash.yaml from SPEED-Bench coding run 41346734342 (B300, vLLM v0.31.0, BF16 dflash/ drafter, TP4, temperature 1.0, top_p 0.95, output_len 4096). Every one of the 14 cells completed 80/80 requests with no failures; thinking-on outputs all carry reasoning tags and thinking-off outputs none. Register the curve in both READMEs and the curve-name test. 中文：根据 SPEED-Bench coding 运行 41346734342（B300、vLLM v0.31.0、 BF16 dflash/ 草稿模型、TP4、temperature 1.0、top_p 0.95、output_len 4096） 提交 infx/golden_al_distribution/mimov2.6flash_dflash.yaml。14 个单元均 完成 80/80 请求且无失败；thinking-on 输出全部带推理标签，thinking-off 输出均不带。同时在中英文 README 与曲线名测试中登记该曲线。 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BGggnGqhy9dML7NM9ow3Fm --- .../infx/golden_al_distribution/README.md | 1 + .../infx/golden_al_distribution/README_zh.md | 1 + .../mimov2.6flash_dflash.yaml | 27 +++++++++++++++++++ .../test_golden_al_distribution.py | 1 + 4 files changed, 30 insertions(+) create mode 100644 inferencex-e2e/infx/golden_al_distribution/mimov2.6flash_dflash.yaml diff --git a/inferencex-e2e/infx/golden_al_distribution/README.md b/inferencex-e2e/infx/golden_al_distribution/README.md index dfdc080cf5..fc86626cf5 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README.md +++ b/inferencex-e2e/infx/golden_al_distribution/README.md @@ -146,6 +146,7 @@ Before accepting an updated curve, reviewers should verify: | GLM-5.3 | MTP (provisional, K=3 copied from GLM-5.2) | [`glm5.3_mtp.yaml`](glm5.3_mtp.yaml) | [28058352479](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28058352479) (GLM-5.2) | | Qwen3.8-Flash-Next | MTP (native) | [`qwen3.8next_mtp.yaml`](qwen3.8next_mtp.yaml) | [33034290269](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/33034290269) | | Muse Spark Open | DFlash2 | [`musespark_dflash.yaml`](musespark_dflash.yaml) | [38808046215](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38808046215) | +| MiMo-V2.6-Flash | DFlash | [`mimov2.6flash_dflash.yaml`](mimov2.6flash_dflash.yaml) | [41346734342](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41346734342) | ## Using golden curves in srt-slurm runs diff --git a/inferencex-e2e/infx/golden_al_distribution/README_zh.md b/inferencex-e2e/infx/golden_al_distribution/README_zh.md index e25bdd372c..cafc810b68 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README_zh.md +++ b/inferencex-e2e/infx/golden_al_distribution/README_zh.md @@ -139,6 +139,7 @@ gh workflow run speedbench-al.yml \ | GLM-5.3 | MTP（临时，K=3 取自 GLM-5.2） | [`glm5.3_mtp.yaml`](glm5.3_mtp.yaml) | [28058352479](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/28058352479)（GLM-5.2） | | Qwen3.8-Flash-Next | MTP (native) | [`qwen3.8next_mtp.yaml`](qwen3.8next_mtp.yaml) | [33034290269](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/33034290269) | | Muse Spark Open | DFlash2 | [`musespark_dflash.yaml`](musespark_dflash.yaml) | [38808046215](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38808046215) | +| MiMo-V2.6-Flash | DFlash | [`mimov2.6flash_dflash.yaml`](mimov2.6flash_dflash.yaml) | [41346734342](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41346734342) | ## 在 srt-slurm 运行中使用黄金曲线 diff --git a/inferencex-e2e/infx/golden_al_distribution/mimov2.6flash_dflash.yaml b/inferencex-e2e/infx/golden_al_distribution/mimov2.6flash_dflash.yaml new file mode 100644 index 0000000000..7d75194ab2 --- /dev/null +++ b/inferencex-e2e/infx/golden_al_distribution/mimov2.6flash_dflash.yaml @@ -0,0 +1,27 @@ +# Source GitHub Actions run: https://github.com/SemiAnalysisAI/InferenceX/actions/runs/41346734342 +# Acceptance Length (AL) reference values measured with SPEED-Bench. +# dataset: coding | temperature: 1.0 | top_p: 0.95 | output_len: 4096 +# thinking_on chat_template_kwargs: {"enable_thinking": true} +# thinking_off chat_template_kwargs: {"enable_thinking": false} +# draft: XiaomiMiMo/MiMo-V2.6-Flash-RL/dflash (BF16, greedy drafting) +# Measured on mimo-v2.6-flash-rl (NVIDIA B300 SXM6 AC, vLLM DFlash), per num_speculative_tokens. +# Auto-generated by benchmarks/single_node/speedbench/mimov2.6flash_fp4_b300_vllm.sh (speedbench-al.yml). +# +# key = num_speculative_tokens (DFlash draft tokens); value = golden AL +mimo-v2.6-flash-rl: + thinking_on: + 1: 1.84 + 2: 2.51 + 3: 2.97 + 4: 3.35 + 5: 3.76 + 6: 3.86 + 7: 4.08 + thinking_off: + 1: 1.83 + 2: 2.52 + 3: 2.95 + 4: 3.30 + 5: 3.74 + 6: 3.89 + 7: 4.00 diff --git a/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py b/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py index dbab825262..ffd57cb9bd 100644 --- a/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py +++ b/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py @@ -39,6 +39,7 @@ def test_every_committed_curve_is_valid() -> None: ("dsv4dsparkprob", {"method": "dspark"}, "dsv4-pro-0813-dspark"), ("dsv41flash", {"method": "dspark"}, "dsv41flash_dspark"), ("musespark", {"method": "dflash"}, "musespark_dflash"), + ("mimov2.6flash", {"method": "dflash"}, "mimov2.6flash_dflash"), ("minimaxm31flash", {"method": "dspark"}, "minimaxm31flash_dspark"), ("kimik3", {"method": "dspark", "draft_sample_method": "greedy"}, "kimik3_dspark"), (