diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0ef456826a..cfe8867fe3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -50,9 +50,9 @@ usage: python -m infx.matrix.generate full-sweep If neither `--single-node` nor `--multi-node` is specified, both types are generated. -By default, throughput runs for every generated config and eval-only jobs run for the selected 8k1k subset and the AgentX GSM8K subset. `--no-evals` disables eval jobs, `--evals-only` emits only that selected subset, and adding `--all-evals` expands it to every fixed-sequence config. `--all-evals` alone is an equivalent eval-only shorthand, but it cannot be combined with `--no-evals`. +By default, throughput runs for every generated config and eval-only jobs run for the selected 8k1k subset and the AgentX GSM8K subset. `--no-evals` disables eval jobs, `--evals-only` emits only that selected subset, and adding `--all-evals` expands it to every eligible 8k1k fixed-sequence and agentic config. `--all-evals` always emits only those eval-only jobs, with or without `--evals-only`, but it cannot be combined with `--no-evals`. -`--step-size` must be greater than 1 and applies to concurrency ranges. Explicit `conc-list` values are emitted directly and are filtered by `--min-conc` / `--max-conc` when provided. When both bounds are set, `--min-conc` must not exceed `--max-conc`. +`--step-size` must be greater than 1 and applies to concurrency ranges. `--min-conc` / `--max-conc` drop concurrencies outside the bounds, including explicit `conc-list` values. A single-node fixed-sequence `conc-start`/`conc-end` range that crosses a bound is clamped to it, and a fixed-sequence config with no concurrency left under `--max-conc` runs at the cap. When both bounds are set, `--min-conc` must not exceed `--max-conc`. `--trim-conc` (the `trim-conc` input of `e2e-tests.yml`) keeps only the minimum concurrency of every generated single- and multi-node deployment shape, after eval selection, for a lowest-concurrency smoke run; in changelog-ref mode only throughput rows are trimmed, and evals keep their selected concurrency. No PR label enables trimming. @@ -65,7 +65,7 @@ full-sweep --config-files configs/nvidia-master.yaml **Test all single-node dsr1 configurations on B200 with 8k1k sequence lengths:** ``` -full-sweep --single-node --model-prefix dsr1 --runner-type b200 --seq-lens 8k1k --config-files configs/nvidia-master.yaml +full-sweep --single-node --model-prefix dsr1 --runner-type cluster:b200-nscale --seq-lens 8k1k --config-files configs/nvidia-master.yaml ``` **Test all single-node fp8 precision configs for 8k1k workloads:** @@ -75,12 +75,12 @@ full-sweep --single-node --precision fp8 --seq-lens 8k1k --config-files configs/ **Test all single-node TRT configs on H200 runners:** ``` -full-sweep --single-node --framework trt --runner-type h200 b200-trt --config-files configs/nvidia-master.yaml +full-sweep --single-node --framework trt --runner-type h200 cluster:h200-dgxc --config-files configs/nvidia-master.yaml ``` **Test specific single-node model on specific hardware with specific sequence lengths:** ``` -full-sweep --single-node --model-prefix dsr1 --runner-type b200 --precision fp4 --framework sglang --seq-lens 8k1k --config-files configs/nvidia-master.yaml +full-sweep --single-node --model-prefix dsr1 --runner-type cluster:b200-nscale --precision fp4 --framework sglang --seq-lens 8k1k --config-files configs/nvidia-master.yaml ``` **Limit concurrency and parallelism for faster testing:** diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 6fbadb13c9..9f61b9ecd7 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -73,7 +73,6 @@ on: type: string required: false default: '' - # Agentic-coding inputs. Fixed-seq-len jobs leave these empty. scenario-type: description: "Scenario type (fixed-seq-len or agentic-coding)" type: string @@ -114,17 +113,10 @@ env: INFERENCEX_E2E_ROOT: ${{ github.workspace }} PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' SALLOC_TIME_LIMIT: '480' - KEEP_LOGS: '0' IS_MULTINODE: 'true' RANDOM_RANGE_RATIO: 0.8 - # Day-zero models resolved via hf: ids download from the Hub inside the - # slurm job (srtctl pre-download + dynamo hub fetch). Anonymous requests - # get 429-rate-limited when several workers pull a 444 GB snapshot at - # once; sbatch/srun inherit this env so the token reaches the workers. HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} EXP_NAME: ${{ fromJSON(inputs.config).exp-name }} RECIPE_FINGERPRINT: ${{ fromJSON(inputs.config)['recipe-fingerprint'] || '' }} @@ -153,9 +145,7 @@ env: POWER: ${{ fromJSON(inputs.config).power && '1' || '0' }} REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }} POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }} - # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} - SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} CONC: ${{ inputs.conc }} DURATION: ${{ inputs.duration }} @@ -218,9 +208,6 @@ jobs: ) || format('[{0}]', toJSON(inputs.runner)) ) }} - # Full-context AgentX warmup can legitimately exceed the fixed-sequence - # eight-hour envelope at high concurrency. Keep one hour beyond the - # recipe's 12-hour Slurm limit for checkout, staging, and artifact upload. timeout-minutes: ${{ inputs.scenario-type == 'agentic-coding' && 780 || 480 }} name: >- ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }} @@ -245,8 +232,6 @@ jobs: done fi - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -271,7 +256,6 @@ jobs: with: ref: ${{ github.workflow_sha }} path: .result-tooling - # Only inferencex-e2e/ is read from this checkout. sparse-checkout: inferencex-e2e persist-credentials: false @@ -320,7 +304,6 @@ jobs: set -x RESULT_FILENAME=$(python3 -m infx.results.result_filename) export RESULT_FILENAME - # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" echo "EVAL_ARTIFACT_RECIPE=${RECIPE_FINGERPRINT:0:16}" >> "$GITHUB_ENV" eval_artifact_conc="$(python3 -c 'import hashlib,os; print(hashlib.sha256(os.environ["CONC_LIST"].encode()).hexdigest()[:12])')" @@ -328,7 +311,6 @@ jobs: source benchmarks/runtime_settings.sh source benchmarks/multi_node/runtime_settings.sh - # Assign data literally; recipe values must never become shell code. settings_json=$(jq -cen \ --argjson prefill "$PREFILL_ADDITIONAL_SETTINGS" \ --argjson decode "$DECODE_ADDITIONAL_SETTINGS" \ @@ -338,7 +320,6 @@ jobs: while IFS= read -r -d '' setting; do export "$setting" done < <(jq -j '.[] + "\u0000"' <<< "$settings_json") - # Resolve the workflow's documented automatic eval-concurrency selection. if [[ -z "$EVAL_CONC" ]]; then EVAL_CONC=$(python3 -c 'import os; print(max(map(int, os.environ["CONC_LIST"].split())))') fi @@ -347,7 +328,6 @@ jobs: "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run if [ "${EVAL_ONLY}" = "true" ]; then echo "Eval-only mode: skipping benchmark result file check" - # Verify eval produced results if ! ls results*.json 1>/dev/null 2>&1; then echo "Eval-only run failed: no results*.json files found." >&2 exit 1 @@ -361,9 +341,6 @@ jobs: echo "Run failed: expected $expected_count agentic results, found ${#agentic_results[@]}." >&2 exit 1 fi - # Existence is not enough: the agentic aggregation step writes the - # aggregate even when aiperf recorded zero valid requests. Require - # successful requests in every concurrency result. for result_file in "${agentic_results[@]}"; do echo "Found agentic result file: $result_file" ok=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(int(bool(d.get('num_requests_successful'))))" "$result_file" 2>/dev/null || echo 0) @@ -390,8 +367,6 @@ jobs: if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi source "$PYTHONPATH/benchmarks/check_env.sh" check_env_vars INFERENCEX_RESULTS_PYTHON - # Launcher stamp supplies the producer SHA unless the - # power-producer-sha input is set as a manual override if [ -z "$POWER_PRODUCER_SHA" ] && [ -f power-producer-sha.txt ]; then export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" @@ -455,7 +430,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: multinode_server_logs_${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }} - # multinode launchers package server logs into this tarball. path: | ${{ env.INFERENCEX_E2E_ROOT }}/multinode_server_logs.tar.gz ${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index abe72ae7f2..adb39e1962 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -54,7 +54,6 @@ on: description: "Git ref (branch/sha) to checkout" required: false type: string - # Agentic-coding inputs. Fixed-seq-len jobs leave these empty. scenario-type: description: "Scenario type (fixed-seq-len or agentic-coding)" required: false @@ -95,11 +94,8 @@ env: INFERENCEX_E2E_ROOT: ${{ github.workspace }} PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' SALLOC_TIME_LIMIT: '480' - KEEP_LOGS: '0' IS_MULTINODE: 'false' REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }} RANDOM_RANGE_RATIO: 0.8 @@ -130,9 +126,7 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} EVAL_SUITE: ${{ inputs.eval-suite }} - # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} - SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} KV_OFFLOADING: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offloading) || '' }} KV_OFFLOAD_BACKEND: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offload-backend.name) || '' }} @@ -206,8 +200,6 @@ jobs: done fi - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -232,7 +224,6 @@ jobs: with: ref: ${{ github.workflow_sha }} path: .result-tooling - # Only inferencex-e2e/ is read from this checkout. sparse-checkout: inferencex-e2e persist-credentials: false @@ -272,8 +263,6 @@ jobs: RUNNER_TYPE: ${{ inputs.runner }} JOB_CHECK_RUN_ID: ${{ job.check_run_id }} RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_a${{ github.run_attempt }}_${{ runner.name }} - # Suppress per-job eval markdown from being appended to the step summary. - # We'll publish a single combined eval table in the collection job instead. GITHUB_STEP_SUMMARY: '' run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi @@ -284,7 +273,6 @@ jobs: export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE)) echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV" - # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" source benchmarks/runtime_settings.sh @@ -292,7 +280,6 @@ jobs: if [ "${EVAL_ONLY}" = "true" ]; then echo "Eval-only mode: skipping benchmark result file check" - # Verify eval produced results if ! ls results*.json 1>/dev/null 2>&1; then echo "Eval-only run failed: no results*.json files found." >&2 exit 1 @@ -339,7 +326,6 @@ jobs: echo "no raw result to process: $RESULT_FILENAME.json" >&2 exit 1 fi - # A native power job's launcher stamp names the telemetry producer if [ -f power-producer-sha.txt ]; then export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" fi @@ -467,7 +453,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.eval-only && 'eval_server_logs_' || 'server_logs_' }}${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }} - # fixed-seq writes server.log at root; agentic writes logs under results/. path: | ${{ env.INFERENCEX_E2E_ROOT }}/server.log ${{ env.INFERENCEX_E2E_ROOT }}/results/*.log @@ -584,7 +569,6 @@ jobs: run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi rm -f meta_env.json || true - # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true rm -f -- ./*_report.json || true rm -f -- ./*_results.jsonl || true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6771b527e1..f37b4d869a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,5 +70,4 @@ jobs: if: matrix.check == 'Tests' run: | python -c "import torch; assert torch.version.cuda is None and torch.version.hip is None" - # SRT is initialized for our connector tests; its upstream suite runs in its own CI. python -m pytest -c inferencex-e2e/pyproject.toml inferencex-e2e/infx/tests/ inferencex-e2e/utils/ inferencex-e2e/runners/ collectivex/tests/ operatorx/tests/ --ignore=inferencex-e2e/utils/srt-slurm -n 4 diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 8eed6e55bb..7fd5fb8d7c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -14,7 +14,6 @@ jobs: concurrency: group: pr-review-${{ github.event.pull_request.number }} cancel-in-progress: false - # Only review PRs opened by someone with write access. if: contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) permissions: contents: read @@ -42,7 +41,6 @@ jobs: uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234 with: path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} - # Repository review credential; the job gates eligible review requests. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] track_progress: true allowed_bots: '' diff --git a/.github/workflows/codeowner-signoff-verify.yml b/.github/workflows/codeowner-signoff-verify.yml index c91e33bea4..956deb92cb 100644 --- a/.github/workflows/codeowner-signoff-verify.yml +++ b/.github/workflows/codeowner-signoff-verify.yml @@ -39,8 +39,6 @@ jobs: concurrency: group: codeowner-signoff-pr-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr-number }} cancel-in-progress: false - # The verifier agent reads untrusted PR content, so this job's token is read-only. - # The separate publish job holds the write permissions. permissions: contents: read pull-requests: read # Read PR metadata, diff, and review comments. @@ -103,7 +101,6 @@ jobs: SIGNOFF_KIND: ${{ steps.resolve.outputs.signoff-kind }} SIGNOFF_FETCH_CMD: ${{ steps.resolve.outputs.signoff-fetch-cmd }} run: | - # These placeholders must remain literal for envsubst. # shellcheck disable=SC2016 envsubst '${REPO} ${PR_NUMBER} ${HEAD_SHA} ${SIGNOFF_AUTHOR} ${SIGNOFF_KIND} ${SIGNOFF_FETCH_CMD}' \ < .github/codeowner-signoff-verify-prompt.md \ @@ -124,13 +121,11 @@ jobs: if: steps.resolve.outputs.proceed == 'true' uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234 env: - # Read-only job token: the agent reads untrusted PR content and never publishes. GH_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ github.token }} BASH_DEFAULT_TIMEOUT_MS: "1800000" BASH_MAX_TIMEOUT_MS: "3600000" with: - # Repository verifier credential; trusted code checks writer authorization before this step. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] github_token: ${{ github.token }} path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} @@ -141,8 +136,6 @@ jobs: settings: | {"fastMode": true} - # Bash is limited to the read-only commands the verifier prompt uses. - # /proc is denied so file tools cannot read credentials from process environments. claude_args: | --model 'claude-opus-5-5' --allowedTools "Read,Write,Glob,Grep,WebFetch,mcp__github_ci__*,Bash(gh pr view:*),Bash(gh pr diff:*),Bash(gh api:*),Bash(gh run view:*),Bash(gh run list:*),Bash(gh run download:*),Bash(git log:*),Bash(git show:*),Bash(uv run --project inferencex-e2e --locked python -m infx.workflows.pareto_coverage:*),Bash(uv run --project inferencex-e2e --locked python -m infx.workflows.signoff_validate:*)" @@ -153,8 +146,6 @@ jobs: already rendered with this run's PR number, head SHA, and sign-off metadata. - # The publisher reads the verdict only when verification succeeded, so success - # here guarantees an uploaded verdict artifact for the publish job. - name: Record verification outcome id: outcome if: always() && steps.resolve.outputs.proceed == 'true' @@ -214,7 +205,6 @@ jobs: inferencex-e2e/pyproject.toml inferencex-e2e/uv.lock - # Any download failure fails this job rather than posting a false rejection. - name: Download verdict if: needs.verify.outputs.verification-succeeded == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/collect-job-events.yml b/.github/workflows/collect-job-events.yml index cbf4d7c5a2..3d1380e81e 100644 --- a/.github/workflows/collect-job-events.yml +++ b/.github/workflows/collect-job-events.yml @@ -25,7 +25,6 @@ jobs: ref: ${{ inputs.tooling-ref }} persist-credentials: false - # Job records are diagnostics: they never fail the run. - name: Download job events continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index d45c1b9962..d0aa6d2936 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -4,7 +4,6 @@ run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || gith permissions: contents: read -# Independent GPU dispatches queue through the priority scheduler; newer runs must not cancel them. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: @@ -77,7 +76,7 @@ on: # zizmor: ignore[concurrency-limits] type: boolean default: false all-evals: - description: "Expand eval selection to every fixed-sequence config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" + description: "Expand eval selection to every eligible 8k1k fixed-sequence and agentic config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" required: false type: boolean default: false @@ -165,7 +164,7 @@ on: # zizmor: ignore[concurrency-limits] type: boolean default: false all-evals: - description: "Expand eval selection to every fixed-sequence config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" + description: "Expand eval selection to every eligible 8k1k fixed-sequence and agentic config; eval-only with generate-cli-command, changelog mode keeps throughput unless evals-only" required: false type: boolean default: false diff --git a/.github/workflows/klaud-candidate.yml b/.github/workflows/klaud-candidate.yml index 5a13b7d9d7..96bb728068 100644 --- a/.github/workflows/klaud-candidate.yml +++ b/.github/workflows/klaud-candidate.yml @@ -16,7 +16,6 @@ on: candidate: type: string required: true - # A resume run continues the session of this planner run, as session number `session`. parent-run: type: string required: false @@ -36,7 +35,6 @@ jobs: name: Klaud Cold runs-on: ubuntu-latest env: - # Repair attempts after the initial update; the baseline is already published. MAX_REPAIRS: '5' KLAUD_BRANCH: klaud/auto-${{ inputs.candidate }} KLAUD_PARENT_RUN: ${{ inputs.parent-run || github.run_id }} @@ -94,9 +92,6 @@ jobs: rm -r "$RUNNER_TEMP/plan" outcome_schema=$(uv run --project "$INFERENCEX_TOOLING_PROJECT" --locked python -P -m infx.klaud outcome-schema) echo "outcome-schema=$outcome_schema" >> "$GITHUB_OUTPUT" - # A planner run's own session runs in its first attempt. A resume run that recovery - # dispatched runs only the session it granted; anything else skips the steps below. - # lifecycle.GATE_STEP and AGENT_STEP name these steps. - name: Check whether this session should run id: gate env: @@ -119,7 +114,6 @@ jobs: id: claude if: steps.gate.outputs.run == 'true' continue-on-error: true - # Stops a session that runs out before the 6-hour job limit, so the steps below still run. timeout-minutes: 330 uses: anthropics/claude-code-action@9171db3e57d6a3140a37ddc2ba92788584e0ead6 # v1.0.234 env: @@ -173,7 +167,6 @@ jobs: if: always() && steps.gate.outputs.run == 'true' with: name: klaud-candidate-${{ inputs.candidate }} - # Never upload the execution transcript or private API/scratch data. path: ${{ runner.temp }}/candidate-diagnostics.json retention-days: 14 - name: Require a verified candidate outcome diff --git a/.github/workflows/klaud-plan.yml b/.github/workflows/klaud-plan.yml index 3547706571..0c2003a351 100644 --- a/.github/workflows/klaud-plan.yml +++ b/.github/workflows/klaud-plan.yml @@ -1,15 +1,11 @@ name: Klaud Cold auto-sweep -# lifecycle.resume_title matches the resume runs' names. run-name: >- ${{ inputs.resume-parent && format('Klaud Cold resume {0} {1} session {2}', inputs.resume-parent, inputs.resume-candidate, inputs.resume-session) || 'Klaud Cold auto-sweep' }} -# Overlapping waves are intentional; candidate ownership claims prevent duplicate work. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: - # Recovery dispatches a resume run with these to continue an unfinished session; a manual - # dispatch leaves them empty and runs a normal wave. inputs: resume-parent: description: Planner run that owns the session to resume @@ -32,13 +28,9 @@ permissions: pull-requests: read # Read PR metadata. env: - # Total candidates per invocation, resumed sessions included; they run in parallel. MAX_CANDIDATES_PER_RUN: '5' jobs: - # The overlap review reads untrusted open-PR content (including fork PRs). It runs in its own - # job with no write-capable credential; the credentialed jobs never share its workspace, runner - # temp directory or tool caches and only consume its validated structured output as data. recover: name: Reconcile interrupted candidates if: github.ref == 'refs/heads/main' && github.run_attempt == 1 && inputs.resume-parent == '' @@ -50,7 +42,6 @@ jobs: outputs: tooling-ref: ${{ steps.checkout.outputs.commit }} blocked-families: ${{ steps.recover.outputs.blocked-families }} - # Unfinished sessions recovery resumed in new runs; they share this wave's cap. resumed: ${{ steps.recover.outputs.resumed }} steps: - id: checkout @@ -87,8 +78,6 @@ jobs: - name: Reconcile interrupted owned candidates id: recover env: - # Recovery mutates only sessions recorded by this workflow after their parent finishes. - # Repository automation credential; the planner runs only on main. GH_TOKEN: ${{ secrets.AGENT_PAT }} # zizmor: ignore[secrets-outside-env] run: | uv run --project "$INFERENCEX_PYTHON_PROJECT" --locked python -m infx.klaud recover --max-resumes "$MAX_CANDIDATES_PER_RUN" @@ -127,7 +116,6 @@ jobs: echo "INFERENCEX_PYTHON_PROJECT=$python_project" >> "$GITHUB_ENV" - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: - # Never save a cache the agent's job could have poisoned for credentialed jobs to restore. enable-cache: false python-version: '3.12' - name: Restore recovery-blocked families @@ -138,7 +126,6 @@ jobs: id: prepare env: GH_TOKEN: ${{ github.token }} - # Read-only dashboard key, used before the review agent starts and absent from its step. KLAUD_DASHBOARD_API_KEY: ${{ secrets.DASH_API_KEY }} # zizmor: ignore[secrets-outside-env] run: >- uv run --project "$INFERENCEX_PYTHON_PROJECT" --locked python -m infx.klaud plan @@ -146,7 +133,6 @@ jobs: --review-batch-size "$REVIEW_BATCH_SIZE" --cooldown-hours "$CANDIDATE_COOLDOWN_HOURS" --stale-pr-days "$STALE_PR_DAYS" - # Uploaded before the agent runs, so selection reads the planner's files, not agent-touched ones. - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: klaud-review-input @@ -178,7 +164,6 @@ jobs: KLAUD_EVIDENCE: ${{ runner.temp }}/klaud with: path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} - # Repository automation credential; the planner runs only on main. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] github_token: ${{ github.token }} track_progress: false @@ -233,7 +218,6 @@ jobs: if: always() && steps.review.outputs.execution_file != '' env: EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} - # Keep only the final result fields selection diagnostics read; never transcript content. run: | execution=$(jq -c ' [.[] | objects | select(.type == "result")] | last @@ -295,9 +279,7 @@ jobs: - name: Select reviewed candidates within the total cap id: select env: - # Repository automation credential; the planner runs only on main. GH_TOKEN: ${{ secrets.AGENT_PAT }} # zizmor: ignore[secrets-outside-env] - # Repository automation credential; the planner runs only on main. KLAUD_DASHBOARD_API_KEY: ${{ secrets.DASH_API_KEY }} # zizmor: ignore[secrets-outside-env] KLAUD_PR_REVIEW: ${{ needs.review.outputs.review }} KLAUD_REVIEW_OUTCOME: ${{ needs.review.outputs.outcome }} @@ -313,7 +295,6 @@ jobs: if: always() with: name: klaud-plan - # Deliberately exclude arbitrary agent scratch files. path: | ${{ runner.temp }}/klaud/candidates.json ${{ runner.temp }}/klaud/open-prs.json @@ -340,8 +321,6 @@ jobs: AGENT_PAT: ${{ secrets.AGENT_PAT }} DASH_API_KEY: ${{ secrets.DASH_API_KEY }} - # A resume run continues one unfinished session. Only recovery, as Klaud-Cold, starts one, and - # klaud-candidate.yml's gate starts its agent only with recovery's grant for that session. resume: name: Run candidate ${{ inputs.resume-candidate }} if: >- diff --git a/.github/workflows/merge-ingest.yml b/.github/workflows/merge-ingest.yml index 04a21de17c..25b47294e7 100644 --- a/.github/workflows/merge-ingest.yml +++ b/.github/workflows/merge-ingest.yml @@ -1,9 +1,6 @@ name: "Merge Ingest" run-name: Merge Ingest - ${{ github.event.head_commit.message }} -# Publishes a merged PR's reused run-sweep.yml artifacts. Pushes to main never -# run benchmarks: without a valid reuse authorization this run fails. - concurrency: group: merge-ingest-${{ github.sha }} cancel-in-progress: false @@ -101,7 +98,6 @@ jobs: SOURCE_RUN_ID: ${{ steps.reuse.outputs.reuse-source-run-id }} MERGE_RUN_ID: ${{ github.run_id }} with: - # Repository integration credential for the scoped ingest job. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | const agentic = process.env.AGENTIC === 'true'; diff --git a/.github/workflows/operatorx-sweep.yml b/.github/workflows/operatorx-sweep.yml index 7f3a5b8527..bf3b768696 100644 --- a/.github/workflows/operatorx-sweep.yml +++ b/.github/workflows/operatorx-sweep.yml @@ -90,7 +90,6 @@ jobs: set -eo pipefail source inferencex-e2e/benchmarks/check_env.sh check_env_vars OPERATORX_RUNNER OPERATORX_BACKENDS OPERATORX_TESTLISTS OPERATORX_WORLD_SIZES OPERATORX_MODE OPERATORX_CHUNK_SIZE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA GITHUB_OUTPUT - # Cases take InferenceX's srt-slurm recipes, expanded by InferenceX's own code (srtctl). git submodule update --init inferencex-e2e/utils/srt-slurm uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml --with ruamel.yaml \ --with marshmallow --with marshmallow-dataclass --with pydantic --with jinja2 --with requests --with rich \ @@ -123,8 +122,6 @@ jobs: timeout-minutes: 70 strategy: fail-fast: false - # Keep every shard eligible: a second matrix cap can strand priority labels - # on GitHub-held jobs. Runner/node admission is owned by the scheduler. matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} runs-on: >- # zizmor: ignore[self-hosted-runner] GPU Slurm runners by design; this job runs only on workflow_dispatch ${{ fromJSON(vars.PRIORITY_SCHEDULER_ENABLED == 'true' && @@ -141,8 +138,6 @@ jobs: PYTHONDONTWRITEBYTECODE: '1' OPERATORX_CLEANUP_SECONDS: '180' steps: - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -259,7 +254,6 @@ jobs: trigger-ingest: name: Trigger OperatorX database ingest - # A run with failed shards is still stored: its unrun cases read as missing. if: >- always() && github.event_name == 'workflow_dispatch' && inputs.ingest && (needs.summarize.result == 'success' || needs.summarize.result == 'failure') @@ -271,7 +265,6 @@ jobs: env: RUN_ID: ${{ github.run_id }} with: - # Cross-repository dispatch credential, as run-sweep.yml's database ingest uses. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | await github.rest.repos.createDispatchEvent({ diff --git a/.github/workflows/pr-infx-diff-comment.yml b/.github/workflows/pr-infx-diff-comment.yml index b68239cfab..7c1dcfe166 100644 --- a/.github/workflows/pr-infx-diff-comment.yml +++ b/.github/workflows/pr-infx-diff-comment.yml @@ -1,15 +1,11 @@ name: PR infx diff -# synchronize covers pushes to PR branches, including branches in forks. -# Use API data only: never check out or execute code from the PR. on: - # Control plane only: reads PR metadata through the API, never executes PR code. pull_request_target: # zizmor: ignore[dangerous-triggers] types: [opened, reopened, ready_for_review, synchronize, edited] permissions: {} -# Serialize comment writes; each run reads the current PR rather than old event data. concurrency: group: pr-infx-diff-${{ github.event.pull_request.number }} cancel-in-progress: false diff --git a/.github/workflows/recover-reused-ingest.yml b/.github/workflows/recover-reused-ingest.yml index a6225d44f6..fd71c202fb 100644 --- a/.github/workflows/recover-reused-ingest.yml +++ b/.github/workflows/recover-reused-ingest.yml @@ -29,7 +29,6 @@ jobs: SOURCE_RUN_ID: ${{ inputs.source-run-id }} MERGE_RUN_ID: ${{ inputs.merge-run-id }} with: - # Cross-repository dispatch credential for maintainer-triggered ingest recovery. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | await github.rest.repos.createDispatchEvent({ diff --git a/.github/workflows/reuse-sweep-comment.yml b/.github/workflows/reuse-sweep-comment.yml index 2aaad24cfe..9b5ff9b87e 100644 --- a/.github/workflows/reuse-sweep-comment.yml +++ b/.github/workflows/reuse-sweep-comment.yml @@ -31,7 +31,7 @@ jobs: - name: Checkout trusted workflow code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.sha }} # issue_comment uses the default-branch commit + ref: ${{ github.sha }} persist-credentials: false - name: Validate request and update reaction diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 96ae8973a3..5df90087ef 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -39,7 +39,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - # Labels opt into sweeps regardless of draft status; readiness only starts reviews. if: >- github.event.pull_request.head.repo.full_name == github.repository && ( @@ -235,7 +234,6 @@ jobs: with: path_to_claude_code_executable: ${{ steps.claude_cli.outputs.path }} github_token: ${{ github.token }} - # Repository integration credential for the scoped sweep/ingest job. anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] track_progress: false settings: | @@ -245,7 +243,7 @@ jobs: --effort low --max-turns 8 --allowedTools "Read,Glob,Grep,Bash(git diff:*)" - --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","musespark","dsv41flash","kimik3","glm5.2","dsv4","minimaxm3","qwen3.5","qwen3.8next","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' + --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","draft_model","sglang","vllm","dynamo-sglang","dynamo-vllm","musespark","dsv41flash","kimik3","glm5.2","dsv4","minimaxm3","qwen3.5","qwen3.8next","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' prompt: | Inspect this pull request's diff against its base branch. @@ -255,7 +253,7 @@ jobs: - agentic: agentic-coding or AgentX workload - eval-only: evaluation without throughput measurement - fp4: FP4 precision - - mtp, eagle, eagle3: speculative decoding method + - mtp, draft_model: speculative decoding method - sglang, vllm, dynamo-vllm: runtime framework - model criterion: matching configured model family - checklist-complete: PR checklist is satisfied @@ -342,9 +340,6 @@ jobs: | if $idx == null then . else del(.[$idx]) end end; - # Canary is a benchmark-only smoke test — exclude entries - # whose primary purpose is eval (run-eval == true) so the - # picked canary never runs an eval pass. (((.single_node["1k1k"] // []) + (.single_node["8k1k"] // []) + (.single_node.agentic // [])) | map(select(.["run-eval"] != true))) as $single_candidates | ((.multi_node.agentic // []) | map(select(.["run-eval"] != true))) as $multi_candidates @@ -385,7 +380,6 @@ jobs: with: config: ${{ toJSON(matrix.config) }} pr-number: &pr-number ${{ github.event.pull_request.number }} - # Only same-repository PRs owned by Klaud receive background priority. klaud-run: &klaud-run ${{ github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login == 'Klaud-Cold' && (startsWith(github.head_ref, 'klaud/auto-') || startsWith(github.head_ref, 'klaude/auto-')) }} runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} @@ -620,9 +614,6 @@ jobs: run-eval: true eval-only: true - # Agentic GSM8K eval rows carry the agentic input shape, so they are - # dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len - # inputs (isl/osl/max-model-len, which agentic rows don't have). sweep-agentic-evals: needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] if: >- @@ -691,12 +682,6 @@ jobs: eval-only: true eval-conc: ${{ (matrix.config['eval-framework'] || 'lm-eval') == 'lm-eval' && matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} - # Multi-node agentic GSM8K eval rows carry the agentic input shape, so - # they are dispatched with sweep-multi-node-agentic's inputs rather than - # sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, - # which agentic rows don't have). Agentic selection uses one highest - # eval-conc per topology; fixed-sequence --all-evals rows may instead pass - # a joined concurrency list to lm-eval. sweep-multi-node-agentic-evals: needs: [setup, canary-select, canary-sweep, canary-multi-node-sweep] if: >- @@ -802,7 +787,6 @@ jobs: upload-changelog-metadata: name: upload-changelog-metadata - # collect-evals makes an eval-only rerun upload a manifest for the current run attempt. needs: [setup, collect-results, collect-evals] if: ${{ always() && needs.setup.result == 'success' }} runs-on: ubuntu-latest @@ -827,7 +811,6 @@ jobs: Path('sweep_manifest.json').write_text(json.dumps({ 'head': os.environ['SWEEP_HEAD'], 'run-id': int(os.environ['GITHUB_RUN_ID']), 'run-attempt': int(os.environ['GITHUB_RUN_ATTEMPT']), - # Klaud validates only full sweeps that ran fail-fast. 'full-sweep': sweep_labels == {'full-sweep-fail-fast'}, 'matrix': matrix, })) @@ -915,7 +898,6 @@ jobs: runs-on: ubuntu-latest env: - # Repository integration credential for the scoped sweep/ingest job. DATABASE_URL: ${{ secrets.DB_RO_URL }} # zizmor: ignore[secrets-outside-env] steps: diff --git a/.github/workflows/runner-offline-digest.yml b/.github/workflows/runner-offline-digest.yml index 13563c6a68..5af58f8a31 100644 --- a/.github/workflows/runner-offline-digest.yml +++ b/.github/workflows/runner-offline-digest.yml @@ -1,18 +1,8 @@ name: Runner Offline Digest -# Daily check that lists self-hosted runners reporting `offline` and posts -# a summary + threaded details to Slack. No post when nothing is offline. -# -# Required secrets: -# RUNNERS_PAT — PAT (or GitHub App token) with repo "Administration: Read" -# so it can list self-hosted runners. The default GITHUB_TOKEN -# cannot read this endpoint. -# SLACK_BOT_TOKEN — Slack bot token with `chat:write`. The bot must be a member -# of channel C09PULGMVNG. - on: schedule: - - cron: '13 14 * * *' # daily 14:13 UTC ≈ 8:13 AM CST (9:13 AM CDT during DST) + - cron: '13 14 * * *' workflow_dispatch: permissions: @@ -30,7 +20,6 @@ jobs: - name: List offline runners id: runners env: - # Repository runner/Slack integration credential for the scheduled digest. GH_TOKEN: ${{ secrets.RUNNERS_PAT }} # zizmor: ignore[secrets-outside-env] REPO: ${{ github.repository }} run: | @@ -65,7 +54,6 @@ jobs: - name: Post Slack digest if: steps.runners.outputs.offline_count != '0' env: - # Repository runner/Slack integration credential for the scheduled digest. SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} # zizmor: ignore[secrets-outside-env] CHANNEL: C09PULGMVNG OFFLINE_COUNT: ${{ steps.runners.outputs.offline_count }} diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index fa61b3e23a..e036c0c98c 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -1,12 +1,5 @@ name: SpeedBench AL Collection -# Push-button (workflow_dispatch) collection of a SPEED-Bench acceptance-length -# (AL) matrix: thinking_on/off x MTP levels, for the given model (defaults to -# DeepSeek-V4-Pro). Produces the golden reference consumed by the -# synthetic-acceptance framework and (optionally) opens a PR updating -# benchmarks/speedbench-reference-al.yaml. - -# Independent collection requests use priority-scheduled runner slots. on: # zizmor: ignore[concurrency-limits] workflow_dispatch: inputs: @@ -26,13 +19,17 @@ on: # zizmor: ignore[concurrency-limits] type: string default: 'dsv4' framework: - description: "Serving framework; selects the _fp4_b300_.sh collector" + description: "Serving framework; selects the _fp4_b300_[_].sh collector" required: false type: choice options: [vllm, sglang] default: 'vllm' + collector-variant: + description: "Collector variant suffix; non-empty selects _fp4_b300__.sh (e.g. probabilistic_sample_method_block_rejection_sample_method)" + required: false + type: string + default: '' image: - # vLLM collectors need vLLM >= 0.30: thinking-on cells rely on --chat-template-kwargs (vllm-project/vllm#44244). description: "Container image (vLLM >= 0.30 for vLLM collectors)" required: false type: string @@ -68,10 +65,15 @@ on: # zizmor: ignore[concurrency-limits] type: string default: '480' open-pr: - description: "Open a PR updating benchmarks/speedbench-reference-al.yaml (default off: artifact-only, paste values in manually)" + description: "Open a PR writing the matrix to infx/golden_al_distribution/.yaml (default off: artifact-only)" required: false type: boolean default: false + golden-curve: + description: "Golden curve file stem that open-pr writes, e.g. dsv4_mtp or kimik3_dspark (required with open-pr)" + required: false + type: string + default: '' ref: description: "Git ref (branch/sha) to checkout" required: false @@ -86,15 +88,11 @@ env: DRAFT_SAMPLE_METHOD: greedy PORT: '8888' INFMAX_CONTAINER_WORKSPACE: '/workspace' - DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' - MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' ENROOT_IMPORT_TIME_LIMIT: '120' - KEEP_LOGS: '0' IS_MULTINODE: 'false' EVAL_ONLY: 'false' RUN_EVAL: 'false' IS_AGENTIC: '0' - SCENARIO_SUBDIR: fixed_seq_len/ HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} HF_HUB_CACHE: '/mnt/hf_hub_cache/' MODEL: ${{ inputs.model }} @@ -107,9 +105,7 @@ env: EP_SIZE: '1' DP_ATTENTION: 'false' SPEC_DECODING: mtp - BENCH_SCRIPT_OVERRIDE: benchmarks/single_node/speedbench/${{ inputs.model-prefix }}_fp4_b300_${{ inputs.framework }}.sh SALLOC_TIME_LIMIT: ${{ inputs.salloc-time }} - # Matrix-collector tunables (propagated into the container via srun --export=ALL). MTP_LIST: ${{ inputs.mtp-list }} THINKING_MODES: ${{ inputs.thinking-modes }} CATEGORY: ${{ inputs.category }} @@ -127,6 +123,15 @@ jobs: priority: ${{ steps.score.outputs.priority }} queue-token: ${{ steps.score.outputs.queue-token }} steps: + - name: Validate golden curve target + if: ${{ inputs.open-pr }} + env: + GOLDEN_CURVE: ${{ inputs.golden-curve }} + run: | + if [[ ! "$GOLDEN_CURVE" =~ ^[a-z0-9_.-]+$ ]]; then + echo "::error::open-pr needs golden-curve, a curve file stem such as dsv4_mtp (got '$GOLDEN_CURVE')." >&2 + exit 1 + fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: { clean: true, persist-credentials: false } - name: Checkout the measured revision's launcher @@ -200,7 +205,6 @@ jobs: steps: - name: Resource cleanup (pre-run) run: &resource-cleanup | - # Cleanup SLURM resources if command -v squeue >/dev/null 2>&1; then for job_name in "${RUNNER_NAME}" "inferencex-${RUNNER_NAME}"; do echo "[Slurm] Cleaning up jobs with name: ${job_name} ..." @@ -212,13 +216,9 @@ jobs: done fi - # Cleanup AL-matrix outputs from a prior job on this runner so a stale - # matrix from a previous run is never picked up as this job's output. rm -rf "${GITHUB_WORKSPACE}/speedbench_results" \ "${GITHUB_WORKSPACE}/inferencex-e2e/speedbench_results" 2>/dev/null || true - # A cancelled or crashed job can leave git state in this persistent - # workspace that breaks checkout; re-cloning fixes it and is idempotent. - id: checkout continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -244,12 +244,28 @@ jobs: rm -f speedbench-reference-al.yaml || true rm -rf speed_bench_data || true + - name: Resolve AL collector + env: + COLLECTOR_VARIANT: ${{ inputs.collector-variant }} + run: | + if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi + collector_dir=benchmarks/single_node/speedbench + collector="${MODEL_PREFIX}_fp4_b300_${FRAMEWORK}" + if [[ -n "$COLLECTOR_VARIANT" ]]; then + collector+="_${COLLECTOR_VARIANT}" + fi + if [[ "$collector" == */* || "$collector" == *$'\n'* || ! -f "$collector_dir/$collector.sh" ]]; then + echo "::error::No SPEED-Bench collector $collector_dir/$collector.sh at this ref. Available collectors are listed below." >&2 + ls "$collector_dir" >&2 + exit 1 + fi + echo "BENCH_SCRIPT_OVERRIDE=$collector_dir/$collector.sh" >> "$GITHUB_ENV" + - name: Checkout launch tooling uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} path: .launch-tooling - # Only inferencex-e2e/ is read from this checkout. sparse-checkout: inferencex-e2e persist-credentials: false @@ -304,33 +320,38 @@ jobs: path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench-reference-al.yaml if-no-files-found: warn - - name: Open PR updating reference yaml + - name: Open PR updating golden AL curve if: ${{ inputs.open-pr && success() }} env: - # Repository write credential for authorized reference-data collection. GH_TOKEN: ${{ secrets.SPEEDBENCH_PAT }} # zizmor: ignore[secrets-outside-env] + GOLDEN_CURVE: ${{ inputs.golden-curve }} run: | if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi set -eo pipefail - # NOTE: the reference yaml is keyed by model at the top level. This - # overwrites it with the current model's matrix; when more than one - # model is collected, replace this cp with a per-model-key YAML merge. - cp speedbench-reference-al.yaml benchmarks/speedbench-reference-al.yaml + source benchmarks/check_env.sh + check_env_vars GOLDEN_CURVE INFERENCEX_LAUNCH_PYTHON BENCH_SCRIPT_OVERRIDE + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + curve_file="infx/golden_al_distribution/${GOLDEN_CURVE}.yaml" + { + echo "# Source GitHub Actions run: ${run_url}" + cat speedbench-reference-al.yaml + } > "$curve_file" + PYTHONPATH="$PWD" "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.golden_al_distribution show "$GOLDEN_CURVE" BRANCH="speedbench-al/${MODEL_PREFIX}-auto-${GITHUB_RUN_ID}" git config user.name "github-actions" git config user.email "github-actions@github.com" git checkout -b "$BRANCH" - git add benchmarks/speedbench-reference-al.yaml + git add "$curve_file" if git diff --cached --quiet; then - echo "No change in reference yaml; skipping PR." + echo "No change in ${curve_file}; skipping PR." exit 0 fi - git commit -m "Update SpeedBench AL reference matrix for ${MODEL} (auto, run ${GITHUB_RUN_ID})" + git commit -m "Update golden AL curve ${GOLDEN_CURVE} for ${MODEL} (auto, run ${GITHUB_RUN_ID})" git -c credential.helper= -c 'credential.helper=!gh auth git-credential' push -u origin "$BRANCH" gh pr create \ - --title "Update SpeedBench AL reference matrix for ${MODEL_PREFIX} (auto)" \ - --body "Auto-generated by the SpeedBench AL Collection workflow (run ${GITHUB_RUN_ID}). Model: \`${MODEL}\`, category: \`${CATEGORY}\`, MTP: \`${MTP_LIST}\`, thinking: \`${THINKING_MODES}\`, output_len: \`${SPEEDBENCH_OUTPUT_LEN}\`. Please review the measured values before merging." \ + --title "Update golden AL curve ${GOLDEN_CURVE} (auto)" \ + --body "Auto-generated by the SpeedBench AL Collection workflow (${run_url}). Curve: \`${curve_file}\`, collector: \`${BENCH_SCRIPT_OVERRIDE}\`, model: \`${MODEL}\`, category: \`${CATEGORY}\`, MTP: \`${MTP_LIST}\`, thinking: \`${THINKING_MODES}\`, output_len: \`${SPEEDBENCH_OUTPUT_LEN}\`. Review the measured values against the checklist in infx/golden_al_distribution/README.md before merging. A new curve also needs its row in that README's curve table." \ --base main \ --head "$BRANCH" @@ -342,9 +363,6 @@ jobs: path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench_results/server_*.log if-no-files-found: ignore - # Per-request benchmark detail (vllm bench serve --save-detailed): includes - # the model's response text per request, so the AL of each cell can be - # audited for output correctness (sensible text + correct thinking mode). - name: Upload detailed benchmark results if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index 9c86ca2c10..2d2db195dc 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -86,7 +86,6 @@ jobs: REQUESTED_BY: ${{ steps.request.outputs.requested-by }} COMMENT_ID: ${{ steps.acknowledge.outputs.comment-id }} with: - # Cross-repository dispatch credential; trusted control code validates maintainer authorization. github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] script: | await github.rest.repos.createDispatchEvent({ diff --git a/.github/workflows/trusted-external-sweep.yml b/.github/workflows/trusted-external-sweep.yml index 6543dace46..6af540f2b8 100644 --- a/.github/workflows/trusted-external-sweep.yml +++ b/.github/workflows/trusted-external-sweep.yml @@ -1,13 +1,7 @@ name: Trusted External Sweep Dispatch run-name: Trusted external sweep - PR #${{ github.event.pull_request.number }} @ ${{ github.event.pull_request.head.sha }} -# This workflow is intentionally control-plane only. It never checks out or -# executes pull-request code. A write-authorized collaborator applies one primary -# sweep label to approve the exact external head SHA, then this trusted event -# dispatches the existing e2e workflow from main with repository secrets. -# Each authorized label event approves an exact PR head; no event may cancel another. on: # zizmor: ignore[concurrency-limits] - # Control plane only: verifies writer permission and the exact labeled head, never executes PR code. pull_request_target: # zizmor: ignore[dangerous-triggers] branches: - main diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index e17a07545b..2cf40a553a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -16,8 +16,8 @@ glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml ``` -- Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, and `tilert`; frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. -- Workloads are `1k1k`, `8k1k`, or `agentx`. Existing bundles spanning several fixed sequence lengths use `fixed-seq-len`; keep their override selectors intact. +- Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, `atom`, and `tilert`. Frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. +- Workloads are `1k1k`, `8k1k`, or `agentx`. - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. - A master-config entry with two or more rows of one recipe design keeps them in one `[-]-variants.yaml`; the axis only tells bundles in one directory apart. `base` holds everything the points share, with `schema: 2` beside it, and each row gets one plain `override__c` block (or a similarly point-identifying name, without dates or hardware) holding only its deltas. An override's `null` deletes the base key, so a null that a point needs, such as TensorRT-LLM's `cuda_graph_config:`, stays in `base`. Rows select it with `srt-recipe: .yaml:override_`, relative to `srt-recipe-dir`. Do not add `zip_override_*` blocks or share a bundle across entries, and keep one override per row: recipe references participate in eval grouping. An entry that reruns another entry's points selects that entry's overrides instead of copying them. Standalone files remain for single-row entries, adapted upstream recipes whose `# Source:` base materially differs, and points too different to share a base, such as a TP low-latency recipe beside wide-EP ones. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index e87b83bb98..27d8fb28ff 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -8,13 +8,9 @@ schema: 2 base: - model: - path: deepseek-v4-pro-0813 - container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 - precision: fp4 identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} - container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + model: {} + container: {} dynamo: install: false slurm: {time_limit: "8:00:00"} @@ -134,12 +130,7 @@ base: command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic PORT: "8000" - IS_MULTINODE: "true" - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index fd6c847285..a5f7b5dcee 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -10,13 +10,9 @@ schema: 2 base: - model: - path: deepseek-v4-pro-0813 - container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 - precision: fp4 identity: - model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} - container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + model: {} + container: {} dynamo: install: false slurm: {time_limit: "8:00:00"} @@ -248,12 +244,7 @@ base: command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic PORT: "8000" - IS_MULTINODE: "true" - MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index dc1fda9cd1..c152524ace 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -188,7 +188,6 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" SERVED_MODEL_NAME: GLM-5.2-NVFP4 AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml index d00b45710c..b41e608430 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-variants.yaml @@ -109,7 +109,6 @@ base: benchmark: env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml index d300a0d8d8..fd9a30d641 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -141,7 +141,6 @@ benchmark: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml index 4f48b45148..e2e2d65f20 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -137,7 +137,6 @@ benchmark: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml index cfa1500d5e..925dd6aae7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-variants.yaml @@ -111,7 +111,6 @@ base: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml index 92137c0e88..d35b8a2478 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -106,7 +106,6 @@ base: container-remap-root: "" benchmark: env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml index 37760ff43f..2a6d5f8f83 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/disagg-variants.yaml @@ -170,7 +170,6 @@ base: placement: node: head env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml index 73d3456bed..d34c126929 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml @@ -77,7 +77,3 @@ srun_options: telemetry: collector_join_timeout_seconds: 12 - -benchmark: - env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml index c4c2b380db..ae474b2072 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -78,7 +78,3 @@ srun_options: telemetry: collector_join_timeout_seconds: 12 - -benchmark: - env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml index 017ca3c7a8..cf5c57b9bd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -79,7 +79,3 @@ srun_options: telemetry: collector_join_timeout_seconds: 12 - -benchmark: - env: - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml index 4457b20714..4b6ba59dd7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml @@ -109,7 +109,6 @@ base: benchmark: env: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/inferencex-e2e/benchmarks/runtime_settings.sh b/inferencex-e2e/benchmarks/runtime_settings.sh index 9b8f0790e3..c674e646ae 100644 --- a/inferencex-e2e/benchmarks/runtime_settings.sh +++ b/inferencex-e2e/benchmarks/runtime_settings.sh @@ -2,6 +2,7 @@ # Workflow-owned common settings, loaded before recipe-specific overrides. # Benchmark receivers validate these inputs; they do not choose missing values. +# Jobs inherit these exports. Post-eval gets only WORKLOAD_ENV names (infx/launch/drivers/srt/submit.py). export OPENAI_API_KEY='EMPTY' export EVAL_ENDPOINT_READY_TIMEOUT_SECONDS='1800' export EVAL_MODEL_STABILIZATION_SECONDS='30' @@ -12,6 +13,3 @@ export ENABLE_AGENTX_POWER='1' export VLLM_ENGINE_READY_TIMEOUT_S='3600' export SGLANG_TORCH_PROFILER_DIR='/workspace' export VLLM_TORCH_PROFILER_DIR='/workspace' - -# Explicitly forward these settings across container boundaries. -export INFERENCEX_RUNTIME_ENV_VARS="OPENAI_API_KEY EVAL_ENDPOINT_READY_TIMEOUT_SECONDS EVAL_MODEL_STABILIZATION_SECONDS AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS AIPERF_EXPERIMENTAL_FAST ENABLE_AGENTX_POWER REQUIRE_POWER VLLM_ENGINE_READY_TIMEOUT_S SGLANG_TORCH_PROFILER_DIR VLLM_TORCH_PROFILER_DIR" diff --git a/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh b/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh index d01cbc2ad5..926a1735fb 100755 --- a/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh +++ b/inferencex-e2e/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh @@ -25,7 +25,7 @@ SERVE_MODEL="${MODEL_PATH}" MODEL_KEY="$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')" CONCURRENCY="1" TEMPERATURE="1.0" -# MUST match the golden config: benchmarks/speedbench-reference-al.yaml was measured +# MUST match the golden config: infx/golden_al_distribution/dsv4_mtp.yaml was measured # with reasoning_effort=high. SPEEDBENCH_DIR="/workspace/speed_bench_data" diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 1b6d9bb4d7..881ce3cb68 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -45,8 +45,6 @@ base: AITER_LOG_LEVEL: WARNING benchmark: env: - # The upstream one-hour profile warms five requests per lane. - AIPERF_WARMUP_REQUESTS_PER_LANE: '5' # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml index 95cd487ae7..89fc5d35ec 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/h100-fp8-mtp/agentic.yaml @@ -111,8 +111,8 @@ override_tp8_c5: KV_OFFLOADING: none # DRAM points offload KV to an embedded Mooncake store. Per rank: the host budget -# (1731 GB = 1612 GiB) less the 414 GiB checkpoint page cache, over TP8, less the -# 26 GiB CPU weight offload and the 4 GiB local buffer = 119 GB. +# (1607 GB = 1496 GiB) less the 414 GiB checkpoint page cache, over TP8, less the +# 26 GiB CPU weight offload and the 4 GiB local buffer = 105 GB. override_tp8_c6_dram: # The worker's Mooncake client and the mooncake-master both install the master config's release. setup_script: vllm-mooncake.sh @@ -131,7 +131,7 @@ override_tp8_c6_dram: store_config: mode: embedded metadata_server: P2PHANDSHAKE - global_segment_size: 119GB + global_segment_size: 105GB local_buffer_size: 4GB protocol: rdma device_name: '' @@ -151,7 +151,7 @@ override_tp8_c6_dram: env: CONC: '6' KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' override_tp8_c8_dram: # The worker's Mooncake client and the mooncake-master both install the master config's release. @@ -171,7 +171,7 @@ override_tp8_c8_dram: store_config: mode: embedded metadata_server: P2PHANDSHAKE - global_segment_size: 119GB + global_segment_size: 105GB local_buffer_size: 4GB protocol: rdma device_name: '' @@ -191,4 +191,4 @@ override_tp8_c8_dram: env: CONC: '8' KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml index 033f7562c3..9a480d8bcb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8-mtp/agentic.yaml @@ -47,18 +47,18 @@ override_gpu_kv: env: KV_OFFLOADING: none # HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one -# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +# Mamba host pool per rank: 1607 GB / TP8 / 2 pools = 100 GB each. override_hicache: roles: agg: args: page-size: 64 enable-hierarchical-cache: true - hicache-size: 108 + hicache-size: 100 hicache-io-backend: kernel hicache-mem-layout: page_first hicache-write-policy: write_through_selective benchmark: env: KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml index 0cffc53fec..1314bed3b5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h100-fp8/agentic.yaml @@ -44,18 +44,18 @@ override_gpu_kv: env: KV_OFFLOADING: none # HiCache extends RadixAttention. Hybrid GDN/Mamba allocates one KV and one -# Mamba host pool per rank: 1731 GB / TP8 / 2 pools = 108 GB each. +# Mamba host pool per rank: 1607 GB / TP8 / 2 pools = 100 GB each. override_hicache: roles: agg: args: page-size: 64 enable-hierarchical-cache: true - hicache-size: 108 + hicache-size: 100 hicache-io-backend: kernel hicache-mem-layout: page_first hicache-write-policy: write_through_selective benchmark: env: KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1731' + TOTAL_CPU_DRAM_GB: '1607' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml index 3c21ee1306..119a4e953e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml @@ -98,7 +98,7 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # Native single-node binding verifies these against the generated point. - TOTAL_CPU_DRAM_GB: '225' + TOTAL_CPU_DRAM_GB: '241' KV_OFFLOADING: dram # Only client concurrency changes; serving/admission/graph/cache settings stay fixed. diff --git a/inferencex-e2e/configs/CONFIGS.md b/inferencex-e2e/configs/CONFIGS.md index ce1fb997f8..9a1aaca4cd 100644 --- a/inferencex-e2e/configs/CONFIGS.md +++ b/inferencex-e2e/configs/CONFIGS.md @@ -12,9 +12,11 @@ entry-name: runner: string precision: string framework: string + multinode: bool + disagg: bool # optional, multinode only # Optional defaults for every search-space entry in this config. router: { name: string, version: string } - kv-p2p-transfer: string + kv-p2p-transfer: string # multinode only # Recipe directory under benchmarks/_node/srt-slurm-recipes/. srt-recipe-dir: string scenarios: @@ -25,16 +27,15 @@ entry-name: - { tp: int, conc-start: int, conc-end: int, srt-recipe: string } # Optionally, specify pipeline/expert/data-attention/context parallelism. - { tp: int, pp: int, ep: int, dp-attn: bool, dcp-size: int, pcp-size: int, conc-start: int, conc-end: int } - # Optionally, declare router metadata and the P2P KV transfer engine. + # Optionally, declare router metadata. - tp: int router: { name: string, version: string } - kv-p2p-transfer: string conc-start: int conc-end: int - ... - ... agentic-coding: # optional - - trace-source: string + - dram-utilization: float # required by kv-offloading: dram search-space: - tp: int kv-offloading: dram @@ -82,13 +83,14 @@ The below list describes what each field is: - `image`: The image used to serve the benchmark, e.g., `vllm/vllm-openai:v0.10.2` - `model`: The model to serve, e.g., `deepseek-ai/DeepSeek-R1-0528` -- `model-prefix`: The canonical InferenceX model prefix reference, i.e., `dsr1` for DeepSeek-R1 or `qwen3.5` for Qwen3.5. Consult `docs/MODELS.md` for supported model/scenario combinations. This value is used to decipher which script in `benchmarks/` should be used in order to launch the benchmark. +- `model-prefix`: The canonical InferenceX model prefix reference, i.e., `dsr1` for DeepSeek-R1 or `qwen3.5` for Qwen3.5. Consult `docs/MODELS.md` for supported model/scenario combinations. The launcher's workload policy rows in [`infx/launch/policy.py`](../infx/launch/policy.py) match on this value. - `runner`: This is the runner label on which to run the benchmark. This must be a valid key under `labels` in `runners.yaml`. Agentic configs must use an exact `cluster:` runner label, not a broad SKU or capacity label, so every search-space point runs on the same hardware fleet. -- `precision`: The precision to run the benchmark. Again, this is used to find which script to run in `benchmarks/`. +- `precision`: The precision to run the benchmark. Launcher workload policy rows may also match on it. - `framework`: The framework (serving runtime) to serve the benchmark, e.g., `vllm`, `sglang`, `trt`. +- `multinode`: Required. `true` selects the multinode schema with `prefill`/`decode` or `worker` blocks, and `false` the single-node schema with `tp` entries. - `disagg`: Enables disaggregated serving and may only be `true` when `multinode` is also `true`. - `hardware`: Optional metadata within each `prefill` and `decode` worker block @@ -98,10 +100,11 @@ The below list describes what each field is: scheduling. - `scenarios`: A dictionary of benchmark scenario types. At least one must be specified. Currently supported: - `fixed-seq-len`: Fixed input/output sequence length benchmarks. Each entry must have: - - `isl`: An integer representing the input sequence length, e.g., `1024` - - `osl`: An integer representing the output sequence length, e.g., `8192` + - `isl`: An integer representing the input sequence length, e.g., `8192` + - `osl`: An integer representing the output sequence length, e.g., `1024` + - PR sweeps run only `isl`/`osl` pairs 1024/1024 and 8192/1024; the changelog planner rejects any other pair. - `search-space`: A list of configurations to run with respective `isl` and `osl`, each entry must be a dict with the following fields: - - `tp`: An integer representing the tensor parallelism level that the configuration will be served at. + - `tp`: A positive integer representing the tensor parallelism level that the configuration will be served at. - `conc-start`: An integer representing the starting level of concurrency e.g., `4` - `conc-end`: An integer representing the ending level of concurrency (inclusive) e.g., `128` - Note: the step factor between `conc-start` and `conc-end` is 2, so if `conc-start` is 4 and `conc-end` is 128, all concurrencies `4, 8, 16, 32, ..., 128` will be run. @@ -114,9 +117,9 @@ The below list describes what each field is: - (Optional) `pcp-size`: Prefill context-parallel size. Default is 1. A topology consumes `tp * pp * pcp-size` GPUs per worker. DCP does not add GPUs. - For single-node entries, set `pp`, `dcp-size`, and `pcp-size` directly in the search-space entry. - For multinode entries, set them independently inside each `prefill` and `decode` worker block. A worker pool allocates `num-worker * tp * pp * pcp-size` GPUs. - - `agentic-coding`: Agentic trace replay benchmarks using real conversation traces. Each entry must have: - - `trace-source`: Identifier for the trace dataset to use. - - `search-space`: Same structure as `fixed-seq-len` search-space entries. + - `agentic-coding`: Agentic trace replay benchmarks using real conversation traces. Each entry has: + - (Optional) `dram-utilization`: The fraction of each node's available CPU DRAM that DRAM KV offloading may use, greater than 0 and at most 1. It is required when any search-space entry sets `kv-offloading: dram`. + - `search-space`: Same structure as `fixed-seq-len` search-space entries, plus `kv-offloading` (`none` or `dram`, required for single-node entries) and the optional `kv-offload-backend`. `router` and `kv-p2p-transfer` may be omitted independently. Router metadata requires non-empty `name` and `version` strings. Its `version` must be the diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ef46e58983..f5c11b8652 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1021,15 +1021,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - isl: 8192 osl: 1024 search-space: - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8] - srt-recipe: disagg-variants.yaml:override_mtp3_1p1d_tp8 - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" conc-list: [2, 4, 8, 16, 32] srt-recipe: disagg-variants.yaml:override_mtp3_1p2d_tp8_narrow @@ -1057,15 +1048,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: ep: 8 dp-attn: true decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - - spec-decoding: "mtp" - conc-list: [1024, 2048, 4096] - srt-recipe: disagg-variants.yaml:override_mtp1_2p1d_dep8 - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} minimaxm3-fp8-mi300x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.32.0@sha256:14f6a575e1fb0a580b0f633c09e06f648709d60405b85f6a7ff3f80d49f39905 diff --git a/inferencex-e2e/configs/ci-priority.yaml b/inferencex-e2e/configs/ci-priority.yaml index 435bf2ebd4..d986316bd3 100644 --- a/inferencex-e2e/configs/ci-priority.yaml +++ b/inferencex-e2e/configs/ci-priority.yaml @@ -19,8 +19,7 @@ adjustments: fp4: 0.1 spec-decoding: mtp: 0.1 - eagle: 0.1 - eagle3: 0.1 + draft_model: 0.1 framework-prefix: sglang: 1.5 vllm: 1.5 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 5ef2f719c1..820a2c6718 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9791,44 +9791,6 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: agentic-coding: - dram-utilization: 0.9 search-space: - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 1000 - srt-recipe: disagg-variants.yaml:override_5p3d_pp4_dep4_c1000 - prefill: - num-worker: 5 - tp: 1 - pp: 4 - ep: 1 - dp-attn: false - decode: - num-worker: 3 - tp: 4 - pp: 1 - ep: 4 - dp-attn: true - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 200 - srt-recipe: disagg-variants.yaml:override_1p1d_pp4_dep4_c200 - prefill: - num-worker: 1 - tp: 1 - pp: 4 - ep: 1 - dp-attn: false - decode: - num-worker: 1 - tp: 4 - pp: 1 - ep: 4 - dp-attn: true - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -9886,62 +9848,6 @@ qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve: pp: 1 ep: 16 dp-attn: true -qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve: - image: lmsysorg/sglang:nightly-dev-cu13-20260913-fa663e72 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:gb300-nv - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: false - srt-recipe-dir: qwen3.5/sglang/gb300-fp8/agentx - scenarios: - agentic-coding: - - dram-utilization: 0.9 - search-space: - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 44 - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c44_b1 - worker: - num-worker: 1 - tp: 4 - pp: 1 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 1 - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c1_b1 - worker: - num-worker: 1 - tp: 4 - pp: 1 - ep: 1 - dp-attn: false - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: - name: hicache - conc-list: - - 1 - num-nodes: 2 - srt-recipe: agg-variants.yaml:override_tp8_c1_b1 - worker: - num-worker: 1 - tp: 8 - pp: 1 - ep: 1 - dp-attn: false # GLM-5.2 NVFP4 AgentX on B300 behind Dynamo (KV router) + SGLang: one # aggregated TP4 engine on 4 of the node's 8 GPUs per point, so tok/s/GPU is diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index c293cbc359..1cb32dda0d 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -10,7 +10,7 @@ Following the principles of fairness and equitableness, the service-level object InferenceX-e2e runs on a fixed GPU pool with a small team, so scenarios, precisions, and recipe variants that no longer earn their cluster hours are retired. The support matrix below is the current record of what is active and retired; git history and `perf-changelog.yaml` hold past settings. -- **Pending A/B baseline retirement (not yet enacted):** standalone non-spec-decode baselines kept only for comparison with speculative decoding are slated for retirement: `dsv4`, `qwen3.5`, and `glm5.2` non-MTP; `minimaxm3` non-EAGLE3; `kimik3` non-DSpark. Retire one only once replacement coverage exists for the affected model, hardware, and engine. A spec-decode counterpart existing is not by itself a reason to remove a config. +- **Pending A/B baseline retirement (not yet enacted):** standalone non-spec-decode baselines kept only for comparison with speculative decoding are slated for retirement. This covers `dsv4` and `qwen3.5` non-MTP and `kimik3` non-DSpark, whose active configs still contain non-spec-decode points. `glm5.2` and `minimaxm3` have no active non-spec-decode configs. Retire one only once replacement coverage exists for the affected model, hardware, and engine. A spec-decode counterpart existing is not by itself a reason to remove a config. - **No separate A/B tracks for new models:** [`infx/golden_al_distribution/`](../infx/golden_al_distribution) pins speculative AgentX runs to one measured acceptance-length curve, so spec-decode results are comparable on their own. New models, including Kimi-K3, do not need a non-spec-decode baseline from day 0. - **Publish the best Pareto points, with or without speculative decoding.** Recipes may disable MTP, EAGLE/EAGLE3, DSpark, or another draft method when that gives a better operating point. Valid non-spec-decode results stay eligible under the [North-star Pareto policy](#north-star-pareto-policy), so a frontier may mix both. diff --git a/inferencex-e2e/infx/bench/agentic/run.py b/inferencex-e2e/infx/bench/agentic/run.py index e4fda327af..0926883c18 100644 --- a/inferencex-e2e/infx/bench/agentic/run.py +++ b/inferencex-e2e/infx/bench/agentic/run.py @@ -77,7 +77,7 @@ def from_env(cls, env: Mapping[str, str]) -> Plan: def _require_single_point(env: Mapping[str, str], conc: str) -> None: - """AgentX measures one concurrency per fresh server deployment.""" + """Raise ``InputError`` unless ``CONC_LIST`` is unset or equals ``conc``, as AgentX measures one concurrency per server.""" points = env.get("CONC_LIST") if points is not None and points != conc: raise inputs.InputError( @@ -88,7 +88,7 @@ def _require_single_point(env: Mapping[str, str], conc: str) -> None: def _validate_kv_offload(env: Mapping[str, str]) -> None: - """The served KV-offload configuration, as the matrix ``kv-offloading`` field allows.""" + """Raise ``InputError`` unless ``env``'s KV-offload settings match a matrix ``kv-offloading`` value.""" mode = inputs.require("KV_OFFLOADING", env=env)["KV_OFFLOADING"] backend = env.get("KV_OFFLOAD_BACKEND") if mode == "none": @@ -105,6 +105,7 @@ def _validate_kv_offload(env: Mapping[str, str]) -> None: def _power_mode(switches: Mapping[str, str], env: Mapping[str, str]) -> PowerMode: + """Pick the power mode from ``ENABLE_AGENTX_POWER`` and the srt-slurm window directory.""" if switches.get("ENABLE_AGENTX_POWER") not in TRUE_VALUES: return "off" return "window" if env.get("SRT_MEASUREMENT_WINDOW_DIR") else "missing" @@ -144,6 +145,7 @@ def execute(plan: Plan, runtime: Runtime, environ: Mapping[str, str]) -> int: def _download_traces(cfg: ReplayConfig, runtime: Runtime, env: Mapping[str, str]) -> int: + """Download ``cfg``'s trace dataset with the runtime's ``hf`` and return its status.""" print(f"Loading traces via aiperf public-dataset: {cfg.loader} ({cfg.dataset})", flush=True) rc = proc.call([str(runtime.hf), "download", "--repo-type", "dataset", cfg.dataset], env) if rc: @@ -195,21 +197,24 @@ def _power_audit(plan: Plan, replay_rc: int) -> list[str] | None: def _window(plan: Plan, state: str) -> list[str]: + """Power adapter arguments that write the multinode window in ``state``.""" return ["--concurrency", str(plan.replay.concurrency), "--write-multinode-window", state] def _power_adapter(plan: Plan, python: str, env: Mapping[str, str], *args: str) -> int: + """Run the power adapter for ``plan`` with ``args`` and return its status.""" strict = ["--require-power"] if plan.require_power else [] adapter = ["agentic.power_adapter", "--result-dir", str(plan.result_dir), *args, *strict] return _results(python, env, *adapter) def _results(python: str, env: Mapping[str, str], module: str, *args: str) -> int: + """Run ``infx.results.`` with ``args`` under ``python`` and return its status.""" return proc.call([python, "-m", f"infx.results.{module}", *args], env) def _check_server_metrics(artifact_dir: Path, prefix: str | None) -> None: - """Opt-in: fail rather than publish trace charts without the engine's metrics.""" + """When ``prefix`` is set, raise ``BenchError`` unless ``artifact_dir`` has non-empty metrics with that prefix.""" if not prefix: return exported = artifact_dir / "server_metrics_export.json" @@ -235,6 +240,7 @@ def _contains(path: Path, needle: bytes) -> bool: def _nonempty(path: Path) -> bool: + """Whether ``path`` is a file with at least one byte.""" return path.is_file() and path.stat().st_size > 0 diff --git a/inferencex-e2e/infx/bench/agentic/venv.py b/inferencex-e2e/infx/bench/agentic/venv.py index 779df64348..9f256a8a87 100644 --- a/inferencex-e2e/infx/bench/agentic/venv.py +++ b/inferencex-e2e/infx/bench/agentic/venv.py @@ -35,18 +35,22 @@ def for_job(cls, environ: Mapping[str, str]) -> Runtime: @property def venv(self) -> Path: + """The venv directory under ``root``.""" return self.root / "venv" @property def python(self) -> Path: + """The venv's python interpreter.""" return self.venv / "bin" / "python" @property def agentx(self) -> Path: + """The venv's ``agentx`` executable.""" return self.venv / "bin" / "agentx" @property def hf(self) -> Path: + """The venv's ``hf`` executable.""" return self.venv / "bin" / "hf" def active(self) -> bool: @@ -83,4 +87,5 @@ def bootstrap(runtime: Runtime) -> int: def _executable(path: Path) -> bool: + """Whether ``path`` is a file this process may execute.""" return path.is_file() and os.access(path, os.X_OK) diff --git a/inferencex-e2e/infx/bench/eval/__init__.py b/inferencex-e2e/infx/bench/eval/__init__.py index 6e91b056cb..ad947efad3 100644 --- a/inferencex-e2e/infx/bench/eval/__init__.py +++ b/inferencex-e2e/infx/bench/eval/__init__.py @@ -53,6 +53,7 @@ def main(argv: list[str]) -> int: def _server_root(endpoint: str) -> str: + """Strip trailing slashes from ``endpoint``, raising ``InputError`` unless it is an http(s) root.""" root = endpoint.rstrip("/") try: parts = urllib.parse.urlsplit(root) @@ -65,6 +66,7 @@ def _server_root(endpoint: str) -> str: def _concurrencies(values: str) -> list[int]: + """Parse the space-separated ``values`` into positive ints, raising ``InputError`` if none.""" concurrencies = [env.parse_positive_int("--concurrency", value) for value in values.split()] if not concurrencies: raise env.InputError("--concurrency must name at least one concurrency") @@ -144,6 +146,7 @@ def _record( suite: str, batch: Mapping[str, Sequence[int]] | None = None, ) -> bool: + """Write ``meta_env.json`` into ``destination`` and return whether that succeeded.""" path = destination / "meta_env.json" try: destination.mkdir(parents=True, exist_ok=True) @@ -161,6 +164,7 @@ def _single( destination: Path, environ: Mapping[str, str], ) -> int: + """Run one eval at ``conc``, stage and record it, and return the exit code.""" outcome, staged = _run(framework, context, conc, destination, "") recorded = _record(destination, environ, conc, outcome.suite) if outcome.returncode: @@ -179,6 +183,7 @@ def _batched( destination: Path, environ: Mapping[str, str], ) -> int: + """Run lm-eval at each concurrency and record the batch manifest, returning 1 if that fails.""" completed: list[int] = [] failed: list[int] = [] suite = "" diff --git a/inferencex-e2e/infx/bench/eval/lm_eval.py b/inferencex-e2e/infx/bench/eval/lm_eval.py index 98eafba81e..7f146b7fe8 100644 --- a/inferencex-e2e/infx/bench/eval/lm_eval.py +++ b/inferencex-e2e/infx/bench/eval/lm_eval.py @@ -24,7 +24,10 @@ def install(environ: Mapping[str, str]) -> None: - """Install lm-eval at ``REF`` into this interpreter; failures only warn, as images may ship it.""" + """Install lm-eval at ``REF`` into this interpreter. Failures only warn, as images may ship it. + + The ``huggingface-hub`` range keeps lm-eval from upgrading it past what transformers 5.x accepts. + """ try: uv.find() except env.BenchError as error: @@ -32,8 +35,6 @@ def install(environ: Mapping[str, str]) -> None: return if "atom" in environ.get("IMAGE", ""): _pip(environ, "uninstall", "torchvision") - # lm-eval does not depend on transformers, so without this range its resolution - # upgrades huggingface-hub past what transformers 5.x accepts. _pip(environ, "install", "lm-eval[api]", "huggingface-hub>=1.5,<2") pinned = ("install", "--no-deps", "--reinstall") git = shutil.which("git", path=environ.get("PATH")) @@ -43,6 +44,7 @@ def install(environ: Mapping[str, str]) -> None: def _pip(environ: Mapping[str, str], command: str, *args: str) -> bool: + """Run ``uv pip ``, warning on failure, and return whether it succeeded.""" argv = uv.pip(command, "-q", "--no-cache", "--break-system-packages", *args) rc = proc.call(argv, environ) if rc: @@ -98,6 +100,7 @@ def max_output_tokens(context: int) -> int: def tasks(environ: Mapping[str, str]) -> str: + """``EVAL_TASKS_DIR``, else the default task set.""" return env.optional("EVAL_TASKS_DIR", env=environ) or DEFAULT_TASKS diff --git a/inferencex-e2e/infx/bench/eval/meta.py b/inferencex-e2e/infx/bench/eval/meta.py index 81e672930e..a410195b1a 100644 --- a/inferencex-e2e/infx/bench/eval/meta.py +++ b/inferencex-e2e/infx/bench/eval/meta.py @@ -19,10 +19,12 @@ def _first(values: Mapping[str, str], *names: str, default: str) -> str: + """The first non-empty value among ``names``, else ``default``.""" return next((values[name] for name in names if values.get(name)), default) def _is_true(value: str) -> bool: + """Whether ``value`` spells true, ignoring case.""" return value.lower() in _TRUE @@ -60,6 +62,7 @@ def build( values.update(_disaggregated(values)) def size(field: str, *names: str) -> int: + """The first set of ``names`` as an int, else 1, raising ``InputError`` if it is not a digit string.""" name = next((name for name in names if values.get(name)), None) if name is None: return 1 @@ -70,6 +73,7 @@ def size(field: str, *names: str) -> int: return int(values[name]) def dp(*names: str) -> bool: + """Whether the first set of ``names`` is true, defaulting to false.""" return _is_true(_first(values, *names, default="false")) framework, precision = values.get("FRAMEWORK", ""), values.get("PRECISION", "") @@ -120,6 +124,7 @@ def dp(*names: str) -> bool: def write(path: Path, document: Mapping[str, object]) -> None: + """Write ``document`` to ``path`` as indented JSON.""" path.write_text(json.dumps(document, indent=2) + "\n") diff --git a/inferencex-e2e/infx/bench/eval/stage.py b/inferencex-e2e/infx/bench/eval/stage.py index 9511298f62..a14d83f880 100644 --- a/inferencex-e2e/infx/bench/eval/stage.py +++ b/inferencex-e2e/infx/bench/eval/stage.py @@ -29,6 +29,7 @@ def copy(results_dir: Path, destination: Path, *, suffix: str = "") -> list[Path def _target(destination: Path, name: str, suffix: str) -> Path: + """Where ``name`` lands in ``destination``, with ``suffix`` before its extension and a clash counter.""" if not suffix: return destination / name extension = next(extension for extension in _EXTENSIONS if name.endswith(extension)) diff --git a/inferencex-e2e/infx/bench/eval/vendor.py b/inferencex-e2e/infx/bench/eval/vendor.py index d55ad8749f..f025333b3a 100644 --- a/inferencex-e2e/infx/bench/eval/vendor.py +++ b/inferencex-e2e/infx/bench/eval/vendor.py @@ -44,6 +44,7 @@ class StepError(Exception): """A setup step failed; ``returncode`` becomes the eval's exit code.""" def __init__(self, message: str, returncode: int) -> None: + """Keep ``message`` and the exit code ``returncode`` the eval should return.""" super().__init__(message) self.returncode = returncode @@ -112,6 +113,7 @@ def step( raise StepError(f"{self.spec.label} {stage} failed with exit code {rc}", rc) def adapter_argv(self, command: tuple[str, ...], *args: str) -> list[str]: + """The adapter command line with ``command`` and ``args``, plus model, output, and suite.""" suite = [self.provider.suite_flag, self.suite] if self.provider.suite_flag else [] return [ self.python, str(EVALS / self.spec.adapter), *command, *args, @@ -119,6 +121,7 @@ def adapter_argv(self, command: tuple[str, ...], *args: str) -> list[str]: ] # fmt: skip def published(self) -> bool: + """Whether every result glob of the suite matches a file in the results directory.""" results = self.ctx.results_dir return all(any(p.is_file() for p in results.glob(glob)) for glob in self.spec.results) @@ -192,6 +195,7 @@ def _provision(job: Job) -> str: def _provision_failed(job: Job, detail: str) -> StepError: + """Print ``detail`` and return the provisioning ``StepError`` with exit code 1.""" print(f"ERROR: {detail}", file=sys.stderr) return StepError(f"{job.spec.label} {_PROVISION} failed with exit code 1", 1) @@ -205,6 +209,7 @@ def _pip_target(job: Job, target: Path) -> list[str]: def _prepare_kimi(job: Job) -> tuple[list[str], dict[str, str]]: + """Install Kimi's runtime and verifier checkout, then return adapter args and ``PYTHONPATH``.""" runtime = job.scratch / "kimi-runtime" job.step("dependency installation", _pip_target(job, runtime)) checkout = job.scratch / "kimi-verifier" @@ -220,6 +225,7 @@ def _prepare_kimi(job: Job) -> tuple[list[str], dict[str, str]]: def _prepare_minimax(job: Job) -> tuple[list[str], dict[str, str]]: + """Fetch the pinned MiniMax source and dependencies, then return adapter args and no env.""" source = job.scratch / "minimax-source" dependencies = job.scratch / "minimax-deps" stage = "pinned runtime preparation" @@ -234,10 +240,12 @@ def _prepare_minimax(job: Job) -> tuple[list[str], dict[str, str]]: def _bfcl_project(job: Job) -> Path: + """The BFCL project directory inside the job's scratch space.""" return job.scratch / "bfcl-project" def _prepare_bfcl(job: Job) -> tuple[list[str], dict[str, str]]: + """Install the BFCL runtime, create its project directory, and return adapter args.""" job.step("dependency installation", [ job.python, str(EVALS / job.spec.adapter), "--install-runtime", str(job.scratch / "bfcl-wheel"), "--uv", uv.find(), diff --git a/inferencex-e2e/infx/bench/fixed_seq.py b/inferencex-e2e/infx/bench/fixed_seq.py index 15704cfe11..43d1657780 100644 --- a/inferencex-e2e/infx/bench/fixed_seq.py +++ b/inferencex-e2e/infx/bench/fixed_seq.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import math from dataclasses import dataclass from pathlib import Path @@ -31,7 +32,7 @@ class Point: def client_argv(point: Point) -> list[str]: - """An unthrottled burst that ignores EOS, after 2x-concurrency warmups.""" + """The client argv for ``point``, an unthrottled burst that ignores EOS after 2x-concurrency warmups.""" argv = [ PYTHON, "-m", "infx.bench_serving.benchmark_serving", "--model", point.model, @@ -61,6 +62,7 @@ def client_argv(point: Point) -> list[str]: def _run_client(point: Point) -> int: + """Run the benchmark client for ``point``, relaying signals, and return its status.""" with proc.RelaySignals() as relay: return relay.run(client_argv(point)) @@ -77,7 +79,7 @@ def served_model(base_url: str) -> str: def srt_single(args: argparse.Namespace) -> int: - """One single-node point; srt-slurm owns the server and power sampling.""" + """Benchmark one single-node point from the environment and return its status. srt-slurm owns the server and power.""" values = env.require( "MODEL", "CONC", "ISL", "OSL", "RANDOM_RANGE_RATIO", "RESULT_FILENAME", "RESULT_DIR", "SRT_FRONTEND_HOST", "SRT_FRONTEND_PORT", "RUN_EVAL", "EVAL_ONLY", "USE_CHAT_TEMPLATE", @@ -117,16 +119,21 @@ def srt_single(args: argparse.Namespace) -> int: def srt_sweep(args: argparse.Namespace) -> int: - """Every ``CONC_LIST`` point of an srt-slurm multi-node job; srt-slurm owns the servers.""" + """Benchmark every ``CONC_LIST`` point of an srt-slurm multi-node job and return the first failure, else 0.""" values = env.require( "ISL", "OSL", "RANDOM_RANGE_RATIO", "SRT_FRONTEND_HOST", "SRT_FRONTEND_PORT", - "CONC_LIST", "PREFILL_NUM_WORKERS", "PREFILL_TP", "DECODE_NUM_WORKERS", "DECODE_TP", + "CONC_LIST", "PREFILL_NUM_WORKERS", "PREFILL_TP", "PREFILL_PP_SIZE", "PREFILL_PCP_SIZE", + "PREFILL_DWDP_SIZE", "DECODE_NUM_WORKERS", "DECODE_TP", "DECODE_PP_SIZE", "DECODE_PCP_SIZE", ) # fmt: skip isl = env.positive_int("ISL") osl = env.positive_int("OSL") base_url = f"http://{values['SRT_FRONTEND_HOST']}:{env.positive_int('SRT_FRONTEND_PORT')}" - ctx = env.non_negative_int("PREFILL_NUM_WORKERS") * env.positive_int("PREFILL_TP") - gen = env.non_negative_int("DECODE_NUM_WORKERS") * env.positive_int("DECODE_TP") + ctx = env.non_negative_int("PREFILL_NUM_WORKERS") * math.prod( + env.positive_int(f"PREFILL_{size}") for size in ("TP", "PP_SIZE", "PCP_SIZE", "DWDP_SIZE") + ) + gen = env.non_negative_int("DECODE_NUM_WORKERS") * math.prod( + env.positive_int(f"DECODE_{size}") for size in ("TP", "PP_SIZE", "PCP_SIZE") + ) concs = [env.parse_positive_int("CONC_LIST", word) for word in values["CONC_LIST"].split()] windows = env.optional("SRT_MEASUREMENT_WINDOW_DIR") model = served_model(base_url) @@ -160,7 +167,7 @@ def srt_sweep(args: argparse.Namespace) -> int: def explicit_point(args: argparse.Namespace) -> int: - """One point described entirely by flags.""" + """Benchmark one point described entirely by ``args`` and return the client's status.""" return _run_client( Point( base_url=args.base_url, diff --git a/inferencex-e2e/infx/bench/proc.py b/inferencex-e2e/infx/bench/proc.py index de3f558f9b..e5c20c7cdd 100644 --- a/inferencex-e2e/infx/bench/proc.py +++ b/inferencex-e2e/infx/bench/proc.py @@ -18,7 +18,7 @@ from typing_extensions import Self REPO_ROOT = Path(__file__).resolve().parents[2] -_SECRET_NAME = re.compile(r"TOKEN|SECRET", re.IGNORECASE) +_SECRET_NAME = re.compile(r"TOKEN(?!S|IZER)|SECRET|API_?KEY|PASSWORD", re.IGNORECASE) def status(returncode: int) -> int: @@ -27,7 +27,10 @@ def status(returncode: int) -> int: def echo(argv: Sequence[str | os.PathLike[str]], env: Mapping[str, str] | None = None) -> None: - """Print ``+ `` to stderr, masking the values of ``*TOKEN*`` and ``*SECRET*`` variables.""" + """Print ``+ `` to stderr, masking the values of credential variables. + + Names like ``TOKENS`` and ``TOKENIZER`` hold counts and paths, so they are not masked. + """ text = shlex.join(map(os.fspath, argv)) secrets = { value @@ -63,17 +66,22 @@ def call( def tee(argv: Sequence[str], log: Path, env: Mapping[str, str] | None = None) -> int: - """``argv 2>&1 | tee log``; return argv's status.""" + """``argv 2>&1 | tee log``; return argv's status, 127 if it cannot start.""" echo(argv, env) - with ( - log.open("wb") as sink, - subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) as child, - ): - output = cast("io.BufferedReader", child.stdout) - while chunk := output.read1(1 << 16): - for stream in (sys.stdout.buffer, sink): - stream.write(chunk) - stream.flush() + with log.open("wb") as sink: + try: + child = subprocess.Popen( + argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env + ) + except OSError as error: + print(f"ERROR: cannot run {argv[0]}: {error}", file=sys.stderr, flush=True) + return 127 + with child: + output = cast("io.BufferedReader", child.stdout) + while chunk := output.read1(1 << 16): + for stream in (sys.stdout.buffer, sink): + stream.write(chunk) + stream.flush() return status(child.returncode) @@ -93,19 +101,23 @@ class _Signals: """Handle ``SIGNALS`` for the ``with`` block and remember the first one received.""" def __init__(self) -> None: + """Start with no signal received and no saved handlers.""" self.received: int | None = None self._previous: dict[int, Any] = {} def __enter__(self) -> Self: + """Install ``_handle`` for every signal in ``SIGNALS`` and return self.""" for signum in SIGNALS: self._previous[signum] = signal.signal(signum, self._handle) return self def __exit__(self, *_: object) -> None: + """Restore the handlers that were in place before the block.""" for signum, handler in self._previous.items(): signal.signal(signum, handler) def _handle(self, signum: int, _frame: FrameType | None) -> None: + """Record ``signum`` unless an earlier signal was already recorded.""" self.received = self.received or signum @@ -117,10 +129,12 @@ class RelaySignals(_Signals): """Run commands one at a time, relaying signals; after one, nothing new starts.""" def __init__(self) -> None: + """Start with no running child.""" super().__init__() self._child: subprocess.Popen[bytes] | None = None def _handle(self, signum: int, frame: FrameType | None) -> None: + """Record ``signum`` and forward it to the running child, if any.""" super()._handle(signum, frame) if self._child is not None: self._child.send_signal(signum) diff --git a/inferencex-e2e/infx/bench/server.py b/inferencex-e2e/infx/bench/server.py index 6c19ff983a..9d4e62ef07 100644 --- a/inferencex-e2e/infx/bench/server.py +++ b/inferencex-e2e/infx/bench/server.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import codecs import http.client import json import os @@ -44,7 +45,7 @@ def http_json(url: str) -> object | None: def process_alive(pid: int) -> bool: - """A zombie waiting to be reaped is not a live server.""" + """Whether ``pid`` is running, counting a zombie waiting to be reaped as dead.""" try: os.kill(pid, 0) except ProcessLookupError: @@ -63,18 +64,32 @@ def wait_ready( ) -> None: """Poll ``url`` until it answers below 400, streaming ``log``; fail once ``pid`` exits.""" offset = 0 + inode: int | None = None + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + + def stream() -> None: + """Copy new bytes of ``log`` to stdout, restarting when the file is replaced or truncated.""" + nonlocal offset, inode + if log is None or not log.exists(): + return + with log.open("rb") as source: + stat = os.fstat(source.fileno()) + if stat.st_ino != inode or stat.st_size < offset: + offset, inode = 0, stat.st_ino + decoder.reset() + source.seek(offset) + chunk = source.read() + offset += len(chunk) + sys.stdout.write(decoder.decode(chunk)) + sys.stdout.flush() + while True: - if log is not None and log.exists(): - with log.open("rb") as stream: - stream.seek(offset) - chunk = stream.read() - offset += len(chunk) - sys.stdout.write(chunk.decode(errors="replace")) - sys.stdout.flush() + stream() status = http_status(url) if status is not None and status < 400: return if pid is not None and not process_alive(pid): + stream() raise NotReadyError(f"process {pid} died before {url} became ready") time.sleep(poll_s) @@ -96,13 +111,15 @@ def wait_chat_route( healthy_since: float | None = None next_report = 0.0 while True: - now = time.monotonic() models = http_json(f"{base_url}/v1/models") - entries = models.get("data", []) if isinstance(models, dict) else [] - served = any(isinstance(entry, dict) and entry.get("id") == model for entry in entries) + entries = models.get("data") if isinstance(models, dict) else None + served = isinstance(entries, list) and any( + isinstance(entry, dict) and entry.get("id") == model for entry in entries + ) if served and http_status(chat_url) in {401, 403, 405}: break status = http_status(f"{base_url}/health") + now = time.monotonic() if status is None or status >= 400: healthy_since = None elif healthy_since is None: diff --git a/inferencex-e2e/infx/bench_serving/backend_request_func.py b/inferencex-e2e/infx/bench_serving/backend_request_func.py index 426a54b6dc..c038b09bc2 100644 --- a/inferencex-e2e/infx/bench_serving/backend_request_func.py +++ b/inferencex-e2e/infx/bench_serving/backend_request_func.py @@ -33,13 +33,15 @@ class RequestFuncInput: @dataclass class RequestFuncOutput: + """One request's result. ``ttft`` is time to first token, ``itl`` the inter-token gaps.""" + generated_text: str = "" success: bool = False latency: float = 0.0 output_tokens: int = 0 - ttft: float = 0.0 # Time to first token - itl: list[float] = field(default_factory=list) # List of inter-token latencies - tpot: float = 0.0 # avg next-token latencies + ttft: float = 0.0 + itl: list[float] = field(default_factory=list) + tpot: float = 0.0 prompt_len: int = 0 error: str = "" @@ -48,6 +50,10 @@ async def async_request_tgi( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one TGI ``generate_stream`` request and return its timings and text. + + TGI rejects temperature 0.0, top_p 1.0, and ``ignore_eos``, so near values are sent and EOS is honored. + """ api_url = request_func_input.api_url assert api_url.endswith("generate_stream") # noqa: S101 @@ -56,10 +62,9 @@ async def async_request_tgi( "best_of": request_func_input.best_of, "max_new_tokens": request_func_input.output_len, "do_sample": True, - "temperature": 0.01, # TGI does not accept 0.0 temperature. - "top_p": 0.99, # TGI does not accept 1.0 top_p. + "temperature": 0.01, + "top_p": 0.99, "truncate": request_func_input.prompt_len, - # TGI does not accept ignore_eos flag. } payload = { "inputs": request_func_input.prompt, @@ -80,19 +85,16 @@ async def async_request_tgi( continue chunk_bytes = chunk_bytes.decode("utf-8") - # TGI pings may contain no data. if chunk_bytes.startswith(":"): continue chunk = chunk_bytes.removeprefix("data:") data = json.loads(chunk) timestamp = time.perf_counter() - # First token if ttft == 0.0: ttft = time.perf_counter() - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -118,6 +120,7 @@ async def async_request_trt_llm( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one TensorRT-LLM ``generate_stream`` request and return its timings and text.""" api_url = request_func_input.api_url assert api_url.endswith("generate_stream") # noqa: S101 @@ -152,12 +155,10 @@ async def async_request_trt_llm( data = json.loads(chunk) output.generated_text += data["text_output"] timestamp = time.perf_counter() - # First token if ttft == 0.0: ttft = timestamp - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -183,21 +184,22 @@ async def async_request_deepspeed_mii( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Send one non-streaming DeepSpeed-MII request and return its latency and text. + + MII rejects temperature 0.0 and cannot stream, so ``ttft`` stays 0 (see DeepSpeed-MII PR 311). + """ async with aiohttp.ClientSession(trust_env=True, timeout=AIOHTTP_TIMEOUT) as session: assert request_func_input.best_of == 1 # noqa: S101 payload = { "prompt": request_func_input.prompt, "max_tokens": request_func_input.output_len, - "temperature": 0.01, # deepspeed-mii does not accept 0.0 temp. + "temperature": 0.01, "top_p": 1.0, } output = RequestFuncOutput() output.prompt_len = request_func_input.prompt_len - # NOTE: DeepSpeed-MII doesn't support streaming as of Jan 28 2024, - # will use 0 as placeholder. - # See https://github.com/microsoft/DeepSpeed-MII/pull/311 output.ttft = 0 st = time.perf_counter() @@ -225,6 +227,10 @@ async def async_request_openai_completions( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one OpenAI completions request and return its timings, text, and token count. + + Skips SSE comment and event lines, and treats a choice-less chunk as the final usage report. + """ api_url = request_func_input.api_url assert api_url.endswith(("completions", "profile")), ( # noqa: S101 "OpenAI Completions API URL must end with 'completions' or 'profile'." @@ -264,22 +270,21 @@ async def async_request_openai_completions( if not chunk_bytes: continue - chunk = chunk_bytes.decode("utf-8").removeprefix("data: ") + chunk = chunk_bytes.decode("utf-8") + if chunk.startswith((":", "event:")): + continue + chunk = chunk.removeprefix("data: ") if chunk != "[DONE]": data = json.loads(chunk) - # The final usage-only chunk may contain no token. if choices := data.get("choices"): - # Special tokens may have empty text. text = choices[0].get("text") timestamp = time.perf_counter() - # First token if not first_chunk_received: first_chunk_received = True ttft = time.perf_counter() - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) @@ -287,7 +292,12 @@ async def async_request_openai_completions( generated_text += text or "" elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") - if first_chunk_received: + elif error := data.get("error"): + output.error = json.dumps(error) + break + if output.error: + output.success = False + elif first_chunk_received: output.success = True else: output.success = False @@ -314,6 +324,7 @@ async def async_request_openai_chat_completions( request_func_input: RequestFuncInput, pbar: tqdm | None = None, ) -> RequestFuncOutput: + """Stream one OpenAI chat completions request and return its timings, text, and token count.""" api_url = request_func_input.api_url assert api_url.endswith("chat/completions"), ( # noqa: S101 "OpenAI Chat Completions API URL must end with 'chat/completions'." @@ -360,30 +371,39 @@ async def async_request_openai_chat_completions( if not chunk_bytes: continue - chunk = chunk_bytes.decode("utf-8").removeprefix("data: ") + chunk = chunk_bytes.decode("utf-8") + if chunk.startswith((":", "event:")): + continue + chunk = chunk.removeprefix("data: ") if chunk != "[DONE]": timestamp = time.perf_counter() data = json.loads(chunk) if choices := data.get("choices"): content = choices[0]["delta"].get("content") - # First token if ttft == 0.0: ttft = timestamp - st output.ttft = ttft - # Decoding phase else: output.itl.append(timestamp - most_recent_timestamp) generated_text += content or "" elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") + elif error := data.get("error"): + output.error = json.dumps(error) + break most_recent_timestamp = timestamp output.generated_text = generated_text - output.success = True + if not output.error and ttft == 0.0: + output.error = ( + "Never received a valid chunk to calculate TTFT." + "This response will be marked as failed!" + ) + output.success = not output.error output.latency = most_recent_timestamp - st else: output.error = response.reason or "" @@ -399,6 +419,7 @@ async def async_request_openai_chat_completions( def get_model(pretrained_model_name_or_path: str) -> str: + """A ModelScope snapshot path when ``VLLM_USE_MODELSCOPE`` is true, else the name unchanged.""" if os.getenv("VLLM_USE_MODELSCOPE", "False").lower() == "true": from modelscope import snapshot_download @@ -500,6 +521,10 @@ def get_tokenizer( trust_remote_code: bool = False, **kwargs: Any, ) -> PreTrainedTokenizer | PreTrainedTokenizerFast: + """Load the tokenizer for a model path or hub id in ``tokenizer_mode`` with sglang's v5 fixes applied. + + Raises ``ValueError`` for slow mode with ``use_fast`` and ``ImportError`` for mistral mode without vllm. + """ if pretrained_model_name_or_path is not None and not os.path.exists( pretrained_model_name_or_path ): diff --git a/inferencex-e2e/infx/bench_serving/benchmark_outcome.py b/inferencex-e2e/infx/bench_serving/benchmark_outcome.py index a1b4ebfbfb..cfa9272708 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_outcome.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_outcome.py @@ -16,7 +16,10 @@ class BenchmarkOutcome(TypedDict): def benchmark_outcome(requested: int, completed: int) -> BenchmarkOutcome: - """Record the same five-percent gate before cleanup or artifact upload.""" + """Apply the five-percent failure gate to ``requested`` and ``completed`` and return the outcome. + + Raises ``ValueError`` unless both are ints with ``0 <= completed <= requested`` and ``requested > 0``. + """ if ( type(requested) is not int or type(completed) is not int diff --git a/inferencex-e2e/infx/bench_serving/benchmark_serving.py b/inferencex-e2e/infx/bench_serving/benchmark_serving.py index 6c8a185d05..8239e70b44 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_serving.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_serving.py @@ -1,21 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 r"""Benchmark online serving throughput. -On the server side, run one of the following commands: - vLLM OpenAI API server +On the server side, start an OpenAI-compatible server, for example: vllm serve \ --swap-space 16 \ --disable-log-requests - (TGI backend) - ./launch_tgi_server.sh - On the client side, run: python -m infx.bench_serving.benchmark_serving \ --backend \ --model \ - --dataset-name sharegpt \ - --dataset-path \ + --dataset-name random \ + --random-input-len \ + --random-output-len \ --request-rate \ # By default is inf --num-prompts # By default is 1000 @@ -36,6 +33,7 @@ from dataclasses import dataclass from datetime import datetime from multiprocessing import Pool, cpu_count +from pathlib import Path from typing import Any import numpy as np @@ -51,9 +49,6 @@ try: from .backend_request_func import get_tokenizer except ImportError: - # vLLM moved get_tokenizer from vllm.transformers_utils.tokenizer to - # vllm.tokenizers; the old path was a deprecation alias in v0.20 and - # was removed by v0.23. Try the new location first, fall back to old. try: from vllm.tokenizers import get_tokenizer except ImportError: @@ -73,6 +68,8 @@ @dataclass class BenchmarkMetrics: + """Aggregate results of one run. ``e2el`` is client latency from request send to full response.""" + completed: int total_input: int total_output: int @@ -92,14 +89,12 @@ class BenchmarkMetrics: median_itl_ms: float std_itl_ms: float percentiles_itl_ms: list[tuple[float, float]] - # Client latency from request send to complete response. mean_e2el_ms: float median_e2el_ms: float std_e2el_ms: float percentiles_e2el_ms: list[tuple[float, float]] -# --- Multiprocessing helpers for sample_random_requests --- _worker_tokenizer = None @@ -112,9 +107,9 @@ def _load_tokenizer( LlamaTokenizer.all_special_tokens_extended (e.g. Qwen3.5 with newer transformers). Prefer backend_request_func.get_tokenizer on fallback so client tokenization stays aligned with the sglang server (#1381, #1428). + ``deepseek_v4`` always uses vLLM's loader, since HF AutoTokenizer may not know it. """ if tokenizer_mode == "deepseek_v4": - # HF AutoTokenizer may not recognize deepseek_v4; use vLLM's loader. try: from vllm.tokenizers import get_tokenizer as _vllm_get_tokenizer except ImportError: @@ -185,7 +180,7 @@ def _apply_chat_template(prompt: str, tokenizer: PreTrainedTokenizerBase, dsv4: def _process_prompt_chunk(chunk_args: tuple) -> list[tuple[str, int, int, None, int]]: - """Generate a chunk of random prompts in a worker process.""" + """Generate one chunk of random prompts in a worker, as ``(prompt, len, output_len, None, mismatch)`` tuples.""" ( indices, prefix_token_ids, @@ -246,6 +241,13 @@ def sample_random_requests( trust_remote_code: bool = False, num_workers: int = 0, ) -> list[tuple[str, int, int]]: + """Build ``num_prompts`` random-token prompts, each length between ``range_ratio`` times its target and the target. + + Returns ``(prompt, prompt_len, output_len, None)`` tuples. Raises ``ValueError`` for no prompts or + ``dsv4`` without a chat template. Padding uses a local RNG so arrival times ignore ``num_workers``. + """ + if num_prompts <= 0: + raise ValueError(f"--num-prompts must be positive, got {num_prompts}") vocab_size = tokenizer.vocab_size prefix_token_ids = np.random.randint(0, vocab_size, size=prefix_len).tolist() # noqa: NPY002 @@ -261,19 +263,19 @@ def sample_random_requests( input_len = input_len - chat_template_len def sample_uniform(seq_len: int) -> list[int]: + """Draw ``num_prompts`` lengths uniformly from ``range_ratio * seq_len`` to ``seq_len``.""" lower = int(seq_len * range_ratio) upper = seq_len return np.random.randint(lower, upper + 1, size=num_prompts).tolist() # noqa: NPY002 input_lens = sample_uniform(input_len) output_lens = sample_uniform(output_len) - offsets = np.random.randint(0, vocab_size, size=num_prompts) # noqa: NPY002 + offsets = np.random.randint(0, vocab_size, size=num_prompts).tolist() # noqa: NPY002 - # Local padding draws keep downstream arrival times independent of num_workers. local_rng = np.random.RandomState( np.random.get_state()[1][ # noqa: NPY002 :4 - ].tolist() # derive seed from current state without advancing it + ].tolist() ) if num_workers <= 0: @@ -294,7 +296,7 @@ def sample_uniform(seq_len: int) -> list[int]: prefix_token_ids, input_lens[start:end], output_lens[start:end], - offsets[start:end].tolist(), + offsets[start:end], prefix_len, vocab_size, use_chat_template, @@ -406,10 +408,14 @@ def calculate_metrics( outputs: list[RequestFuncOutput], dur_s: float, tokenizer: PreTrainedTokenizerBase, - selected_percentile_metrics: list[str], # noqa: ARG001 selected_percentiles: list[float], goodput_config_dict: dict[str, float], ) -> tuple[BenchmarkMetrics, list[int]]: + """Summarize ``outputs`` over ``dur_s`` seconds into metrics, plus each request's output length. + + Retokenizes text when the server sent no usage count, which may overcount slightly. Goodput counts + a request whose TTFT, TPOT (0 for one token), and E2EL all meet ``goodput_config_dict``. + """ actual_output_lens: list[int] = [] total_input = 0 completed = 0 @@ -423,8 +429,7 @@ def calculate_metrics( if outputs[i].success: output_len = outputs[i].output_tokens - if output_len is None: - # Chunks can bundle tokens; retokenization may slightly overcount. + if not output_len: output_len = len( tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids ) @@ -435,7 +440,6 @@ def calculate_metrics( latency_minus_ttft = outputs[i].latency - outputs[i].ttft tpot = latency_minus_ttft / (output_len - 1) tpots.append(tpot) - # Note: if output_len <= 1, we regard tpot as 0 for goodput all_tpots.append(tpot) itls += outputs[i].itl ttfts.append(outputs[i].ttft) @@ -469,6 +473,7 @@ def calculate_metrics( "on the benchmark arguments.", stacklevel=2, ) + ttfts, tpots, itls, e2els = (np.asarray(v or 0) for v in (ttfts, tpots, itls, e2els)) metrics = BenchmarkMetrics( completed=completed, total_input=total_input, @@ -477,29 +482,22 @@ def calculate_metrics( request_goodput=good_completed / dur_s, output_throughput=sum(actual_output_lens) / dur_s, total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s, - mean_ttft_ms=np.mean(ttfts or 0) - * 1000, # ttfts is empty if streaming is not supported by backend - std_ttft_ms=np.std(ttfts or 0) * 1000, - median_ttft_ms=np.median(ttfts or 0) * 1000, - percentiles_ttft_ms=[ - (p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles - ], - mean_tpot_ms=np.mean(tpots or 0) * 1000, - std_tpot_ms=np.std(tpots or 0) * 1000, - median_tpot_ms=np.median(tpots or 0) * 1000, - percentiles_tpot_ms=[ - (p, np.percentile(tpots or 0, p) * 1000) for p in selected_percentiles - ], - mean_itl_ms=np.mean(itls or 0) * 1000, - std_itl_ms=np.std(itls or 0) * 1000, - median_itl_ms=np.median(itls or 0) * 1000, - percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000) for p in selected_percentiles], - mean_e2el_ms=np.mean(e2els or 0) * 1000, - std_e2el_ms=np.std(e2els or 0) * 1000, - median_e2el_ms=np.median(e2els or 0) * 1000, - percentiles_e2el_ms=[ - (p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles - ], + mean_ttft_ms=np.mean(ttfts) * 1000, + std_ttft_ms=np.std(ttfts) * 1000, + median_ttft_ms=np.median(ttfts) * 1000, + percentiles_ttft_ms=[(p, np.percentile(ttfts, p) * 1000) for p in selected_percentiles], + mean_tpot_ms=np.mean(tpots) * 1000, + std_tpot_ms=np.std(tpots) * 1000, + median_tpot_ms=np.median(tpots) * 1000, + percentiles_tpot_ms=[(p, np.percentile(tpots, p) * 1000) for p in selected_percentiles], + mean_itl_ms=np.mean(itls) * 1000, + std_itl_ms=np.std(itls) * 1000, + median_itl_ms=np.median(itls) * 1000, + percentiles_itl_ms=[(p, np.percentile(itls, p) * 1000) for p in selected_percentiles], + mean_e2el_ms=np.mean(e2els) * 1000, + std_e2el_ms=np.std(e2els) * 1000, + median_e2el_ms=np.median(e2els) * 1000, + percentiles_e2el_ms=[(p, np.percentile(e2els, p) * 1000) for p in selected_percentiles], ) return metrics, actual_output_lens @@ -527,6 +525,10 @@ async def benchmark( max_concurrency: int | None, lora_modules: list[str] | None, ) -> dict[str, Any]: + """Warm up, then send ``input_requests`` to ``api_url`` at ``request_rate`` and return the result dict. + + Raises ``ValueError`` for an unknown backend or multi-modal input outside ``openai-chat``. + """ if backend in ASYNC_REQUEST_FUNCS: request_func = ASYNC_REQUEST_FUNCS[backend] else: @@ -535,7 +537,6 @@ async def benchmark( print("Starting initial single prompt test run...") test_prompt, test_prompt_len, test_output_len, test_mm_content = input_requests[0] if backend != "openai-chat" and test_mm_content is not None: - # multi-modal benchmark is only available on OpenAI Chat backend. raise ValueError("Multi-modal content is only supported on 'openai-chat' backend.") test_input = RequestFuncInput( model=model_id, @@ -556,6 +557,7 @@ async def benchmark( warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None async def warmup_limited_req_fn() -> RequestFuncOutput: + """Send the test request, under the warmup concurrency limit if one is set.""" if warmup_semaphore is None: return await request_func(request_func_input=test_input, pbar=warmup_pbar) async with warmup_semaphore: @@ -612,6 +614,7 @@ async def warmup_limited_req_fn() -> RequestFuncOutput: async def limited_request_func( request_func_input: RequestFuncInput, pbar: tqdm | None ) -> RequestFuncOutput: + """Send ``request_func_input``, under the concurrency limit if one is set.""" if semaphore is None: return await request_func(request_func_input=request_func_input, pbar=pbar) async with semaphore: @@ -648,6 +651,12 @@ async def limited_request_func( ) outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks) + if pbar is not None: + pbar.close() + + benchmark_duration = time.perf_counter() - benchmark_start_time + benchmark_end_time_unix = time.time() + if profile: print("Stopping profiler...") profile_input = RequestFuncInput( @@ -663,18 +672,11 @@ async def limited_request_func( if profile_output.success: print("Profiler stopped") - if pbar is not None: - pbar.close() - - benchmark_duration = time.perf_counter() - benchmark_start_time - benchmark_end_time_unix = time.time() - metrics, actual_output_lens = calculate_metrics( input_requests=input_requests, outputs=outputs, dur_s=benchmark_duration, tokenizer=tokenizer, - selected_percentile_metrics=selected_percentile_metrics, selected_percentiles=selected_percentiles, goodput_config_dict=goodput_config_dict, ) @@ -700,7 +702,7 @@ async def limited_request_func( "total_input_tokens": metrics.total_input, "total_output_tokens": metrics.total_output, "request_throughput": metrics.request_throughput, - "request_goodput:": metrics.request_goodput if goodput_config_dict else None, + "request_goodput": metrics.request_goodput if goodput_config_dict else None, "output_throughput": metrics.output_throughput, "total_token_throughput": metrics.total_token_throughput, "input_lens": [output.prompt_len for output in outputs], @@ -716,6 +718,7 @@ def process_one_metric( metric_name: str, metric_header: str, ) -> None: + """Print the selected metric's mean, median, and percentiles and copy them into ``result``.""" if metric_attribute_name not in selected_percentile_metrics: return print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-")) @@ -756,6 +759,7 @@ def process_one_metric( def check_goodput_args(args: argparse.Namespace) -> dict[str, float]: + """Parse ``args.goodput`` into SLOs in ms, raising ``ValueError`` for unknown names or negative values.""" goodput_config_dict = {} valid_names = ["ttft", "tpot", "e2el"] if args.goodput: @@ -777,6 +781,7 @@ def check_goodput_args(args: argparse.Namespace) -> dict[str, float]: def parse_goodput(slo_pairs: list[str]) -> dict[str, float]: + """Parse ``KEY:VALUE`` strings into a dict, raising ``ArgumentTypeError`` on bad format.""" goodput_config_dict = {} try: for slo_pair in slo_pairs: @@ -795,6 +800,7 @@ def parse_goodput(slo_pairs: list[str]) -> dict[str, float]: def save_to_pytorch_benchmark_format( args: argparse.Namespace, results: dict[str, Any], file_name: str ) -> None: + """Write the latency metrics of ``results`` as PyTorch benchmark records beside ``file_name``.""" metrics = [ "median_ttft_ms", "mean_ttft_ms", @@ -812,7 +818,7 @@ def save_to_pytorch_benchmark_format( ignored_metrics = ["ttfts", "itls", "generated_texts", "errors"] pt_records = convert_to_pytorch_benchmark_format( args=args, - metrics={k: [results[k]] for k in metrics}, + metrics={k: [results[k]] for k in metrics if k in results}, extra_info={ k: results[k] for k in results if k not in metrics and k not in ignored_metrics }, @@ -824,7 +830,15 @@ def save_to_pytorch_benchmark_format( def main(args: argparse.Namespace) -> None: + """Run the benchmark from parsed ``args``, save results if asked, and exit nonzero when the run fails. + + Records the request outcome before saving so cleanup cannot turn a failed run into a pass. + """ print(args) + if args.request_rate <= 0: + raise ValueError(f"--request-rate must be positive, got {args.request_rate}") + if args.save_result and args.result_dir and not Path(args.result_dir).is_dir(): + raise ValueError(f"--result-dir {args.result_dir} is not an existing directory") random.seed(args.seed) np.random.seed(args.seed) # noqa: NPY002 @@ -868,7 +882,6 @@ def main(args: argparse.Namespace) -> None: goodput_config_dict = check_goodput_args(args) - # Avoid GC processing "static" data - reduce pause times. gc.collect() gc.freeze() @@ -897,12 +910,9 @@ def main(args: argparse.Namespace) -> None: ) ) - # Preserve the request gate before writing results: cleanup or a surrounding - # launcher must not turn an error-containing benchmark into a passed point. try: outcome = benchmark_outcome(args.num_prompts, benchmark_result["completed"]) except ValueError as exc: - # Preserve invalid counts for diagnostics; the processor rejects them. outcome = { "status": "failed", "requested": args.num_prompts, @@ -925,7 +935,7 @@ def main(args: argparse.Namespace) -> None: if args.metadata: for item in args.metadata: if "=" in item: - kvstring = item.split("=") + kvstring = item.split("=", 1) result_json[kvstring[0].strip()] = kvstring[1].strip() else: raise ValueError("Invalid metadata format. Please use KEY=VALUE format.") @@ -939,7 +949,6 @@ def main(args: argparse.Namespace) -> None: result_json = {**result_json, **benchmark_result} if not args.save_detailed: - # Remove fields with too many data points for field in [ "ttfts", "itls", @@ -988,7 +997,6 @@ def main(args: argparse.Namespace) -> None: default=None, help="Server or API base url if not using http host and port.", ) - # Use 127.0.0.1 here instead of localhost to force the use of ipv4 parser.add_argument("--host", type=str, default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) parser.add_argument( diff --git a/inferencex-e2e/infx/bench_serving/benchmark_utils.py b/inferencex-e2e/infx/bench_serving/benchmark_utils.py index db746acafc..c4180e3f84 100644 --- a/inferencex-e2e/infx/bench_serving/benchmark_utils.py +++ b/inferencex-e2e/infx/bench_serving/benchmark_utils.py @@ -8,9 +8,9 @@ def convert_to_pytorch_benchmark_format( args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any] ) -> list: - """ - Save the benchmark results in the format used by PyTorch OSS benchmark with - on metric per record + """Build one PyTorch OSS benchmark record per entry of ``metrics``, tagged with ``args`` and ``extra_info``. + + Returns an empty list unless ``SAVE_TO_PYTORCH_BENCHMARK_FORMAT`` is set. Format reference is at https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database """ records = [] diff --git a/inferencex-e2e/infx/bench_serving/encoding_dsv4.py b/inferencex-e2e/infx/bench_serving/encoding_dsv4.py index a83c9d504b..1d7bac4ddb 100644 --- a/inferencex-e2e/infx/bench_serving/encoding_dsv4.py +++ b/inferencex-e2e/infx/bench_serving/encoding_dsv4.py @@ -10,8 +10,6 @@ import re from typing import Any -# Special Tokens - bos_token: str = "<|begin▁of▁sentence|>" # noqa: RUF001, S105 eos_token: str = "<|end▁of▁sentence|>" # noqa: RUF001, S105 thinking_start_token: str = "" # noqa: S105 @@ -22,7 +20,6 @@ ASSISTANT_SP_TOKEN = "<|Assistant|>" # noqa: RUF001, S105 LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" # noqa: RUF001, S105 -# Task special tokens for internal classification tasks DS_TASK_SP_TOKENS = { "action": "<|action|>", # noqa: RUF001 "query": "<|query|>", # noqa: RUF001 @@ -33,8 +30,6 @@ } VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) -# Templates - system_msg_template: str = "{content}" user_msg_template: str = "{content}" latest_reminder_msg_template: str = "{content}" @@ -84,8 +79,6 @@ You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. """ -# Utility Functions - def to_json(value: Any) -> str: """Serialize a value to JSON string.""" @@ -159,6 +152,7 @@ def decode_dsml_to_arguments( """Convert param_name -> (value, is_string) pairs to OpenAI tool-call arguments.""" def _decode_value(key: str, value: str, string: str) -> str: + """Render one ``"key": value`` JSON member, quoting ``value`` when ``string`` is "true".""" if string == "true": value = to_json(value) return f"{to_json(key)}: {value}" @@ -193,9 +187,6 @@ def find_last_user_index(messages: list[dict[str, Any]]) -> int: return last_user_index -# Message Rendering - - def render_message( index: int, messages: list[dict[str, Any]], @@ -206,6 +197,8 @@ def render_message( """ Render a single message at the given index into its encoded string form. + An assistant message that follows a task message is a task output and renders without thinking. + Args: index: Index of the message to render. messages: Full list of messages in the conversation. @@ -323,7 +316,6 @@ def render_message( summary_content = content or "" rc = reasoning_content or "" - # Check if previous message has a task - if so, this is a task output (no thinking) prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None if thinking_mode == "thinking" and not prev_has_task: @@ -379,9 +371,6 @@ def render_message( return prompt -# Preprocessing - - def merge_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: """Merge OpenAI tool messages into user-message content_blocks. @@ -423,7 +412,6 @@ def merge_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: "content": msg.get("content", ""), "content_blocks": [text_block], } - # Preserve extra fields (task, wo_eos, mask, etc.) for key in ("task", "wo_eos", "mask"): if key in msg: new_msg[key] = msg[key] @@ -469,9 +457,6 @@ def sort_tool_results_by_call_order( return messages -# Main Encoding Function - - def encode_messages( messages: list[dict[str, Any]], thinking_mode: str, @@ -513,8 +498,6 @@ def encode_messages( if thinking_mode == "thinking" and effective_drop_thinking: full_messages = _drop_thinking_messages(full_messages) - # After dropping, recalculate how many messages to render - # (context may have shrunk too) num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) context_len = len(full_messages) - num_to_render else: @@ -555,14 +538,10 @@ def _drop_thinking_messages(messages: list[dict[str, Any]]) -> list[dict[str, An msg = copy.copy(msg) msg.pop("reasoning_content", None) result.append(msg) - # developer and other roles before last_user_idx are dropped return result -# Parsing (Decoding model output) - - def _read_until_stop(index: int, text: str, stop: list[str]) -> tuple[int, str, str | None]: """ Read text from index until one of the stop strings is found. diff --git a/inferencex-e2e/infx/cli.py b/inferencex-e2e/infx/cli.py index 6ad09e8691..79d2fc1ef2 100644 --- a/inferencex-e2e/infx/cli.py +++ b/inferencex-e2e/infx/cli.py @@ -12,6 +12,7 @@ def main(argv: list[str] | None = None) -> int: + """Run the ``infx`` command line on ``argv`` and return the exit status.""" parser = argparse.ArgumentParser(prog="infx") commands = parser.add_subparsers(dest="command", required=True) generate = commands.add_parser( diff --git a/inferencex-e2e/infx/clusters/__init__.py b/inferencex-e2e/infx/clusters/__init__.py index e9138b12a2..305b67e948 100644 --- a/inferencex-e2e/infx/clusters/__init__.py +++ b/inferencex-e2e/infx/clusters/__init__.py @@ -11,7 +11,6 @@ from pathlib import Path, PurePosixPath from typing import Annotated, Any, Literal, Self -import yaml from pydantic import ( Field, PrivateAttr, @@ -21,6 +20,7 @@ model_validator, ) +from infx import yamlio from infx.clusters.base import Record, SchedulerSettings from infx.clusters.slurm import Fabric, SlurmSettings from infx.config import RUNNER_CONFIG, repository_root @@ -83,14 +83,17 @@ class Cluster(Record): @property def id(self) -> str: + """Return the cluster id this record was bound to, or an empty string.""" return self._id def bind_id(self, cluster_id: str) -> None: + """Record ``cluster_id`` as this cluster's id.""" self._id = cluster_id @field_validator("env") @classmethod def _exportable(cls, env: dict[str, str]) -> dict[str, str]: + """Return ``env`` unchanged, or raise ValueError if any value contains a comma.""" if names := sorted(name for name, value in env.items() if "," in value): raise ValueError(f"env values cannot contain ',' (srun --export splits on it): {names}") return env @@ -220,7 +223,7 @@ def load_inventory(runner_config: str | Path | Mapping[str, Any] | None = None) data = runner_config else: with open(runner_config) as file: - data = yaml.safe_load(file) + data = yamlio.safe_load(file) if not isinstance(data, Mapping): raise ValueError(f"{runner_config}: runner config must be a mapping") return RunnerInventory.model_validate(data) diff --git a/inferencex-e2e/infx/config.py b/inferencex-e2e/infx/config.py index fca8964433..2a0f0df46f 100644 --- a/inferencex-e2e/infx/config.py +++ b/inferencex-e2e/infx/config.py @@ -9,6 +9,7 @@ def repository_root() -> Path: + """Return the project root from ``INFERENCEX_REPOSITORY_ROOT``, the source tree, or the working directory.""" if root := os.environ.get("INFERENCEX_REPOSITORY_ROOT"): return project_root(Path(root)) source_root = Path(__file__).resolve().parent.parent diff --git a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py index da20cbcb6c..193048c1b9 100644 --- a/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/datasets/build_weka_hf_dataset.py @@ -32,6 +32,7 @@ import json import os import shlex +import shutil import subprocess import sys from collections.abc import Callable @@ -48,6 +49,7 @@ def parse_args() -> argparse.Namespace: + """Parse the command line and return the options, rejecting ``--dwbug-min-peak`` alone.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -101,9 +103,15 @@ def parse_args() -> argparse.Namespace: "--limit", type=int, default=None, - help="Cap session count (for smoke tests). Implies --sampling top.", + help="Cap session count (smoke tests), taking the first sessions in --sampling order.", ) p.add_argument("--sampling", choices=("top", "recent", "random"), default="top") + p.add_argument( + "--seed", + type=int, + default=None, + help="Salt for --sampling random (required by the sampler in that mode).", + ) p.add_argument( "--exclude-dynamic-workflow-bug", action="store_true", @@ -125,11 +133,13 @@ def parse_args() -> argparse.Namespace: help="HF write token (else $HF_TOKEN or cached login).", ) - p.add_argument("--skip-sample", action="store_true", help="Reuse work-dir/proxy/ if present.") + p.add_argument( + "--skip-sample", action="store_true", help="Reuse a completed work-dir/proxy/ sample." + ) p.add_argument( "--skip-convert", action="store_true", - help="Reuse work-dir/per_trace/ if present.", + help="Reuse work-dir/per_trace/ if it holds exactly the sampled sessions.", ) p.add_argument( "--skip-upload", @@ -137,21 +147,37 @@ def parse_args() -> argparse.Namespace: help="Build payloads but don't push to HF.", ) - return p.parse_args() + args = p.parse_args() + if args.dwbug_min_peak is not None and not args.exclude_dynamic_workflow_bug: + p.error("--dwbug-min-peak requires --exclude-dynamic-workflow-bug") + return args def _run(cmd: list, **kw: Any) -> None: + """Echo ``cmd`` and run it, raising ``CalledProcessError`` on failure.""" print(f"$ {' '.join(shlex.quote(str(c)) for c in cmd)}", flush=True) subprocess.run(cmd, check=True, **kw) +def _recreate(stage_dir: Path) -> None: + """Empty a work-dir stage directory so earlier runs' traces are not republished.""" + if stage_dir.exists(): + shutil.rmtree(stage_dir) + stage_dir.mkdir(parents=True) + + def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: + """Dump sampled proxy sessions into ``work_dir/proxy`` and return that directory. + + With ``--skip-sample`` a complete earlier dump is reused. A missing manifest.json marks a crashed dump. + """ proxy_dir = work_dir / "proxy" - if args.skip_sample and proxy_dir.exists() and any(proxy_dir.glob("*.jsonl")): + complete = (proxy_dir / "manifest.json").is_file() + if args.skip_sample and complete and any(proxy_dir.glob("*.jsonl")): n = sum(1 for _ in proxy_dir.glob("*.jsonl")) print(f"[sample] reusing {n} cached session JSONLs in {proxy_dir}") return proxy_dir - proxy_dir.mkdir(parents=True, exist_ok=True) + _recreate(proxy_dir) cmd: list = [ sys.executable, str(SAMPLER), @@ -169,6 +195,7 @@ def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: ("--require-cli-min", args.require_cli_min), ("--max-parallel-subagents", args.max_parallel_subagents), ("--limit", args.limit), + ("--seed", args.seed), ]: if val is not None: cmd += [flag, str(val)] @@ -176,19 +203,19 @@ def stage_sample(args: argparse.Namespace, work_dir: Path) -> Path: cmd.append("--exclude-dynamic-workflow-bug") if args.dwbug_min_peak is not None: cmd += ["--dwbug-min-peak", str(args.dwbug_min_peak)] - if args.db_url: - cmd += ["--db-url", args.db_url] _run(cmd) return proxy_dir def stage_convert(args: argparse.Namespace, proxy_dir: Path, work_dir: Path) -> Path: + """Convert the proxy JSONLs in ``proxy_dir`` to weka JSONs under ``work_dir`` and return that directory.""" per_trace = work_dir / "per_trace" - if args.skip_convert and per_trace.exists() and any(per_trace.glob("*.json")): - n = sum(1 for _ in per_trace.glob("*.json")) + traces = {p.stem for p in per_trace.glob("*.json")} + if args.skip_convert and traces and traces == {p.stem for p in proxy_dir.glob("*.jsonl")}: + n = len(traces) print(f"[convert] reusing {n} cached per-trace JSONs in {per_trace}") return per_trace - per_trace.mkdir(parents=True, exist_ok=True) + _recreate(per_trace) _run( [ sys.executable, @@ -257,6 +284,7 @@ def _compute_stats(per_trace_dir: Path) -> dict: def _format_stats(stats: dict) -> str: + """Return ``stats`` as one aligned ``key: value`` line per entry.""" lines = [] for k, v in stats.items(): lines.append(f"{k}: {v:>15,}") @@ -285,6 +313,7 @@ def _build_readme( pretty_date: str | None = None, isl_cap: int | None = None, ) -> str: + """Return the dataset card README for ``repo_id`` from its stats, filters, and sampler command.""" now = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC") pretty_label = ( f"CC Traces — Weka, With Subagents, " @@ -389,6 +418,7 @@ def _build_readme( def _filters_block(args: argparse.Namespace, *, cap_256k: bool = False) -> str: + """Return the Markdown bullet list of the sampling filters in ``args``.""" bits = [] if ( args.min_trace_version is not None @@ -451,7 +481,7 @@ def _build_payload( parent_repo_id: str | None = None, ) -> dict: """Assemble traces.jsonl + stats + plots + README into payload_dir.""" - payload_dir.mkdir(parents=True, exist_ok=True) + _recreate(payload_dir) n = _concat_traces_jsonl(per_trace_dir, payload_dir / "traces.jsonl") print(f"[payload] wrote traces.jsonl with {n} traces") @@ -490,6 +520,7 @@ def _build_payload( def _is_oversize(req: dict, cap: int = CAP_TOKENS) -> bool: + """Whether the request's input plus output tokens exceed ``cap``.""" return (req.get("in") or 0) + (req.get("out") or 0) > cap @@ -554,8 +585,16 @@ def _filter_trace(trace: dict, is_oversize: Callable[[dict], bool]) -> dict | No entry["t"] = first_t entry["duration_ms"] = round((last_end - first_t) * 1000.0) entry["total_tokens"] = sum((r.get("in") or 0) + (r.get("out") or 0) for r in inners) + entry["models"] = sorted({r["model"] for r in inners}) out["requests"] = surviving + out["models"] = sorted( + { + m + for e in surviving + for m in (e["models"] if e.get("type") == "subagent" else [e["model"]]) + } + ) return out @@ -577,7 +616,7 @@ def _stage_filter( tag: str, ) -> Path: """Apply ``filter_fn`` to every per-trace JSON; write survivors to out_dir.""" - out_dir.mkdir(parents=True, exist_ok=True) + _recreate(out_dir) kept = 0 dropped = 0 for p in sorted(per_trace_dir.glob("*.json")): @@ -617,6 +656,7 @@ def stage_isl_filter(per_trace_dir: Path, work_dir: Path, cap: int) -> Path: def stage_upload( args: argparse.Namespace, payload_dir: Path, repo_id: str, commit_msg: str ) -> None: + """Create the dataset repo ``repo_id`` and upload ``payload_dir`` to it, unless ``--skip-upload``.""" if args.skip_upload: print(f"[upload] --skip-upload: payload ready at {payload_dir}") return @@ -656,6 +696,7 @@ def _reconstruct_sampler_cmd(args: argparse.Namespace) -> list: ("--require-cli-min", args.require_cli_min), ("--max-parallel-subagents", args.max_parallel_subagents), ("--limit", args.limit), + ("--seed", args.seed), ]: if val is not None: cmd += [flag, str(val)] @@ -667,6 +708,7 @@ def _reconstruct_sampler_cmd(args: argparse.Namespace) -> list: def main() -> int: + """Sample, convert, build, and upload the base dataset and the optional 256k one, returning 0.""" args = parse_args() work_dir = args.work_dir work_dir.mkdir(parents=True, exist_ok=True) diff --git a/inferencex-e2e/infx/datasets/plot_subagent_distributions.py b/inferencex-e2e/infx/datasets/plot_subagent_distributions.py index d9239d89d8..e57d17e49b 100644 --- a/inferencex-e2e/infx/datasets/plot_subagent_distributions.py +++ b/inferencex-e2e/infx/datasets/plot_subagent_distributions.py @@ -90,6 +90,7 @@ def parse_args() -> argparse.Namespace: + """Parse the command line and return the input, output, and binning options.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -119,6 +120,7 @@ def parse_args() -> argparse.Namespace: def collect_metrics(in_dir: Path) -> dict[str, list[float]]: + """Read every weka trace in ``in_dir`` and return the sub-agent fan-out samples by metric name.""" groups_per_trace: list[int] = [] inners_per_group: list[int] = [] duration_sec: list[float] = [] @@ -178,6 +180,10 @@ def _draw_histogram( value_fmt: str, linear_clip_pct: float, ) -> None: + """Draw a histogram of ``values`` with percentile markers on ``ax``, or an empty panel if none. + + A log axis centres a single repeated value, since ``geomspace(v, v)`` can round to non-monotonic edges. + """ if not values: ax.set_title(f"{title}\n(no values)") ax.axis("off") @@ -191,9 +197,11 @@ def _draw_histogram( ax.set_title(f"{title}\n(no positive values)") ax.axis("off") return - lo = max(positive.min(), 1e-3) hi = float(positive.max()) - edges = np.geomspace(lo, hi, bins + 1) + lo = max(float(positive.min()), 1e-3) if hi > 1e-3 else float(positive.min()) + edges = ( + np.geomspace(lo, hi, bins + 1) if hi > lo else np.geomspace(lo / 2, hi * 2, bins + 1) + ) ax.hist(positive, bins=edges, color="#888", edgecolor="#222", linewidth=0.4) ax.set_xscale("log") else: @@ -239,6 +247,7 @@ def plot_combined( use_log: bool, linear_clip_pct: float, ) -> None: + """Draw every metric of ``metrics`` into one 3x2 figure and save it to ``out_path``.""" fig, axes = plt.subplots(3, 2, figsize=(20, 16)) for spec, ax in zip(PLOT_SPECS, axes.flat, strict=False): log_x = use_log and spec["log_in_log_fig"] @@ -264,6 +273,7 @@ def plot_combined( def main() -> int: + """Collect the metrics from ``--in-dir`` and write the log and linear figures, returning 0.""" args = parse_args() args.out_dir.mkdir(parents=True, exist_ok=True) metrics = collect_metrics(args.in_dir) diff --git a/inferencex-e2e/infx/datasets/plot_weka_distributions.py b/inferencex-e2e/infx/datasets/plot_weka_distributions.py index b8795b94cd..d97a015c33 100644 --- a/inferencex-e2e/infx/datasets/plot_weka_distributions.py +++ b/inferencex-e2e/infx/datasets/plot_weka_distributions.py @@ -91,6 +91,7 @@ def parse_args() -> argparse.Namespace: + """Parse the command line and return the input, output, and binning options.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -126,6 +127,7 @@ def parse_args() -> argparse.Namespace: def collect_metrics(in_dir: Path) -> dict[str, list[float]]: + """Read every weka trace in ``in_dir`` and return the per-request and per-session samples by metric name.""" think_times: list[float] = [] isls: list[int] = [] osls: list[int] = [] @@ -201,6 +203,10 @@ def _draw_histogram( value_fmt: str, linear_clip_pct: float, ) -> None: + """Draw a histogram of ``values`` with percentile markers on ``ax``, or an empty panel if none. + + A log axis centres a single repeated value, since ``geomspace(v, v)`` can round to non-monotonic edges. + """ if not values: ax.set_title(f"{title}\n(no values)") ax.axis("off") @@ -214,9 +220,11 @@ def _draw_histogram( ax.set_title(f"{title}\n(no positive values)") ax.axis("off") return - lo = max(positive.min(), 1e-3) hi = float(positive.max()) - edges = np.geomspace(lo, hi, bins + 1) + lo = max(float(positive.min()), 1e-3) if hi > 1e-3 else float(positive.min()) + edges = ( + np.geomspace(lo, hi, bins + 1) if hi > lo else np.geomspace(lo / 2, hi * 2, bins + 1) + ) ax.hist(positive, bins=edges, color="#888", edgecolor="#222", linewidth=0.4) ax.set_xscale("log") else: @@ -268,6 +276,7 @@ def plot_combined( use_log: bool, linear_clip_pct: float, ) -> None: + """Draw every metric of ``metrics`` into one 3x2 figure and save it to ``out_path``.""" fig, axes = plt.subplots(3, 2, figsize=(20, 16)) for spec, ax in zip(PLOT_SPECS, axes.flat, strict=False): log_x = use_log and spec["log_in_log_fig"] @@ -293,6 +302,7 @@ def plot_combined( def main() -> int: + """Collect the metrics from ``--in-dir`` and write the log and linear figures, returning 0.""" args = parse_args() args.out_dir.mkdir(parents=True, exist_ok=True) diff --git a/inferencex-e2e/infx/datasets/proxy_to_weka.py b/inferencex-e2e/infx/datasets/proxy_to_weka.py index 1e5436d78e..b713e2874d 100644 --- a/inferencex-e2e/infx/datasets/proxy_to_weka.py +++ b/inferencex-e2e/infx/datasets/proxy_to_weka.py @@ -1,7 +1,7 @@ """Convert flat per-session JSONL dumps into weka-format trace JSON. Reads /.jsonl produced by `sample_proxy_traces.py` -and writes /..//.json in the v1 weka trace +and writes /.json in the v1 weka trace format consumed by the kv-cache-tester replayer (see agentx-harness: src/aiperf/dataset/loader/weka_trace_models.py). @@ -38,6 +38,7 @@ def _dump_trace_inline_hash_ids(trace: dict, path: Path) -> None: placeholders: list[list[Any]] = [] def _substitute(obj: Any) -> Any: + """Return a copy of ``obj`` with every ``hash_ids`` list swapped for a numbered placeholder.""" if isinstance(obj, dict): out: dict[str, Any] = {} for k, v in obj.items(): @@ -63,6 +64,7 @@ def _substitute(obj: Any) -> Any: def parse_args() -> argparse.Namespace: + """Parse the command line and return the input and output directories.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -88,10 +90,12 @@ def parse_args() -> argparse.Namespace: def slugify(label: str) -> str: + """Return ``label`` lowercased with non-alphanumeric runs as underscores, or ``subagent`` if empty.""" return _SLUG_RE.sub("_", label.lower()).strip("_") or "subagent" def load_session_rows(path: Path) -> list[dict]: + """Read the JSONL rows at ``path`` and return them sorted by timestamp with exact duplicates dropped.""" rows: list[dict] = [] with path.open() as f: for line in f: @@ -125,12 +129,13 @@ def load_session_rows(path: Path) -> list[dict]: def remap_hash(h: str, m: dict[str, int]) -> int: + """Return the dense integer id of hash ``h``, assigning the next one in ``m`` when it is new.""" if h not in m: m[h] = len(m) return m[h] -def infer_block_size(rows: list[dict]) -> int: # noqa: ARG001 +def infer_block_size() -> int: """Return the proxy's 64-token block size. Partial trailing blocks make hash_token_count / len(hash_ids) unreliable. @@ -218,6 +223,7 @@ def compute_think_times(rows: list[dict]) -> list[float | None]: def _parse_cli_version(s: str | None) -> tuple[int, int, int] | None: + """Parse a ``major.minor.patch`` CLI version string, returning None when it is missing or malformed.""" if not s: return None parts = s.split(".") @@ -268,6 +274,7 @@ def build_subagent_entry( items: list[tuple[dict, float | None]], hash_map: dict[str, int], ) -> dict: + """Return the weka sub-agent entry for one ``label`` instance built from its rows and think times.""" inner = [build_normal_request(row, hash_map, tt) for row, tt in items] first_row = items[0][0] last_row = items[-1][0] @@ -296,6 +303,7 @@ def build_subagent_entry( def session_to_weka(session_id: str, rows: list[dict]) -> dict: + """Convert one session's proxy ``rows`` into a weka trace dict for ``session_id``.""" if not rows: return { "id": session_id, @@ -323,7 +331,7 @@ def session_to_weka(session_id: str, rows: list[dict]) -> dict: think_times = compute_think_times(rows) hash_map: dict[str, int] = {} - block_size = infer_block_size(rows) + block_size = infer_block_size() out_requests: list[dict] = [] instance_count: dict[str, int] = {} @@ -386,6 +394,7 @@ def session_to_weka(session_id: str, rows: list[dict]) -> dict: def main() -> int: + """Convert every session JSONL in ``--in-dir`` into a weka trace in ``--out-dir``, returning 0.""" args = parse_args() in_files = sorted(p for p in args.in_dir.glob("*.jsonl")) diff --git a/inferencex-e2e/infx/datasets/sample_proxy_traces.py b/inferencex-e2e/infx/datasets/sample_proxy_traces.py index 18a622af93..20d9ee2f6d 100644 --- a/inferencex-e2e/infx/datasets/sample_proxy_traces.py +++ b/inferencex-e2e/infx/datasets/sample_proxy_traces.py @@ -47,9 +47,13 @@ def _redact_url(url: str) -> str: - """Show host + db, drop credentials.""" + """Show host + db, drop credentials. + + A libpq ``key=value`` conninfo has no URL netloc, so it is shown as a placeholder.""" try: p = urlsplit(url) + if not p.netloc: + return "" return f"{p.hostname}{p.path}" except Exception: # noqa: BLE001 return "" @@ -110,6 +114,7 @@ def _redact_url(url: str) -> str: def parse_args() -> argparse.Namespace: + """Parse and validate the command line, exiting on bad flag combinations, and return the options.""" p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -260,7 +265,7 @@ def parse_args() -> argparse.Namespace: "--seed", type=int, default=None, - help="Salt for --sampling random (server-side md5). Required when --sampling random.", + help="Salt for --sampling random (md5 of session_id + seed). Required when --sampling random.", ) p.add_argument( "--db-url", @@ -308,6 +313,8 @@ def parse_args() -> argparse.Namespace: p.error("--sampling random requires --seed") if args.limit is not None and args.sampling is None: p.error("--limit requires --sampling so the LIMIT is deterministic") + if args.limit is not None and args.limit < 1: + p.error("--limit must be >= 1") if args.max_parallel_subagents is not None: if args.max_parallel_subagents < 0: p.error("--max-parallel-subagents must be >= 0") @@ -348,6 +355,7 @@ def parse_args() -> argparse.Namespace: def get_db_url(args: argparse.Namespace) -> str: + """Return the connection string from ``--db-url`` or the environment, exiting when neither is set.""" url = args.db_url or os.environ.get(DEFAULT_ENV) if not url: sys.exit(f"ERROR: connection string missing. Set ${DEFAULT_ENV} or pass --db-url.") @@ -464,8 +472,10 @@ def _sort_and_limit(rows: list[dict], args: argparse.Namespace) -> list[dict]: md5(session_id || seed) is the same expression Postgres uses for the 'random' mode, so deterministic-random output is identical to the - one-pass version. + one-pass version. Rows are first sorted by session_id because phase 1 + has no ORDER BY, which makes ties reproducible. """ + rows = sorted(rows, key=lambda r: r["session_id"]) if args.sampling == "top": rows = sorted(rows, key=lambda r: r["req_count"], reverse=True) elif args.sampling == "recent": @@ -487,7 +497,10 @@ def _sort_and_limit(rows: list[dict], args: argparse.Namespace) -> list[dict]: def find_sessions( conn: psycopg.Connection, args: argparse.Namespace, model_like: str ) -> list[dict]: + """Return the sampled summary rows of sessions passing every filter in ``args``.""" + def _encode_cli_int(s: str | None) -> int | None: + """Encode ``X.Y.Z`` as one comparable int, None when empty, exiting when malformed.""" if not s: return None parts = s.split(".") @@ -627,6 +640,7 @@ def _encode_cli_int(s: str | None) -> int | None: def find_one_session( conn: psycopg.Connection, session_id: str, privacy_mode: str, model_like: str ) -> list[dict]: + """Return the summary row of session ``session_id``, or an empty list when no rows match.""" logger.info("looking up session %s (--session-id mode)", session_id) t = time.time() with conn.cursor(row_factory=dict_row) as cur: @@ -696,6 +710,7 @@ def find_one_session( def _parse_cli_tuple(s: str | None) -> tuple[int, int, int] | None: + """Parse a ``major.minor.patch`` CLI version, returning None when it is missing or malformed.""" if not s: return None parts = s.split(".") @@ -708,6 +723,7 @@ def _parse_cli_tuple(s: str | None) -> tuple[int, int, int] | None: def _hash_lcp(a: list, b: list) -> int: + """Return the length of the common prefix of hash lists ``a`` and ``b``.""" n = 0 for x, y in zip(a, b, strict=False): if x == y: @@ -779,6 +795,7 @@ def fetch_session_rows( def write_session_rows(rows: list[dict], out_path: Path) -> int: + """Write ``rows`` to ``out_path`` as JSONL and return how many were written.""" with out_path.open("w") as f: for row in rows: f.write(json.dumps(row, default=str) + "\n") @@ -786,6 +803,7 @@ def write_session_rows(rows: list[dict], out_path: Path) -> int: def main() -> int: + """Find the requested sessions, dump each to a JSONL file plus manifest.json, and return 0.""" args = parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, diff --git a/inferencex-e2e/infx/evals/EVALS.md b/inferencex-e2e/infx/evals/EVALS.md index 88d0c63020..33d643a60d 100644 --- a/inferencex-e2e/infx/evals/EVALS.md +++ b/inferencex-e2e/infx/evals/EVALS.md @@ -7,9 +7,10 @@ from throughput. Selection lives in `mark_eval_entries()` in ## Selection -- **Fixed-sequence, single-node:** 8k1k only, at the highest and median - concurrency for every model, runner, framework, precision, TP, and decoding - configuration. +- **Fixed-sequence, single-node:** 8k1k only. Each group of model, runner, + framework, precision, spec-decoding, and dp-attn runs evals at the highest + and the median of its concurrencies of 16 or above. Every row at those concurrencies + gets an eval, and TP/EP do not split groups. - **Fixed-sequence, multi-node:** 8k1k only, with one job per parallelism topology at its highest eligible concurrency. Rows differing only by concurrency share a topology. @@ -461,8 +462,8 @@ mismatched parser/chat template. staging, the workflow upload, the collector, and the score validator. It projects the four-case aggregate as task `bfcl_smoke` and the four one-case diagnostic tasks shown above. Every row uses lm-eval-compatible `acc,none` (plus -`acc_stderr,none`); BFCL workflows therefore validate with metric prefix -`acc,` rather than the default exact-match prefix. +`acc_stderr,none`), and every task config declares the `acc` metric, so the +score validator selects `acc,` without a metric override. The `bfcl_smoke` and four `bfcl_` thresholds are `0.0`, so all five scores remain diagnostic. Dependency, endpoint, timeout, malformed-output, @@ -502,10 +503,11 @@ adapter rejects missing, unexpected, or duplicate result IDs and score headers whose counts or accuracy do not reconcile with the selected corpus. The compatibility result publishes `bfcl_vllm_minimax_m3` or -`bfcl_vllm_kimi` as the aggregate task and preserves per-category -`bfcl_` tasks. Kimi also publishes a combined `bfcl_multi_turn` -task. Full-suite thresholds are `0.0`; they are diagnostic until repeated -hardware runs establish model, precision, and backend baselines. A completed +`bfcl_vllm_kimi` as the aggregate task, plus one `_` task per +category, such as `bfcl_vllm_kimi_simple_python`. Kimi also publishes a +combined `bfcl_vllm_kimi_multi_turn` task. Full-suite thresholds are `0.0`, so +they stay diagnostic until repeated hardware runs establish model, precision, +and backend baselines. A completed zero-score run therefore passes threshold validation, while dependency, transport, timeout, malformed-output, and integration failures still fail. diff --git a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py index 29973d952e..2bb37c0416 100644 --- a/inferencex-e2e/infx/evals/_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/evals/_kimi_verifier_archive.py @@ -6,6 +6,7 @@ import tempfile import time from hashlib import sha256 +from http.client import IncompleteRead from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import quote, urlsplit, urlunsplit @@ -13,6 +14,10 @@ def archive_member_parts(name: str) -> tuple[str, ...]: + """Split an archive member name into its path parts. + + Raises ValueError if the name is absolute, empty or escapes the archive root. + """ if not name or "\x00" in name or "\\" in name or name.startswith("/"): raise ValueError(f"unsafe archive member path: {name!r}") normalized = name.rstrip("/") @@ -23,6 +28,11 @@ def archive_member_parts(name: str) -> tuple[str, ...]: def main() -> None: + """Download the pinned verifier archive, check its SHA256 and extract the required files. + + Takes repo URL, commit SHA, archive SHA256 and checkout directory from argv. Exits 1 after + printing the failed stage. + """ repo_url, verifier_ref, expected_archive_sha256, checkout_dir_arg = sys.argv[1:] checkout_dir = Path(checkout_dir_arg) stage = "derive the pinned archive URL" @@ -34,6 +44,7 @@ def main() -> None: raise ValueError( f"expected a 64-character archive SHA256, got {expected_archive_sha256!r}" ) + expected_archive_sha256 = expected_archive_sha256.lower() parsed_repo_url = urlsplit(repo_url) if parsed_repo_url.scheme not in ("http", "https") or not parsed_repo_url.netloc: @@ -89,7 +100,6 @@ def main() -> None: ) archive_file.write(chunk) digest.update(chunk) - break except HTTPError as error: if error.code not in (408, 429) and not 500 <= error.code < 600: raise @@ -101,7 +111,7 @@ def main() -> None: file=sys.stderr, ) time.sleep(attempt) - except (TimeoutError, URLError, ConnectionError) as error: + except (TimeoutError, URLError, ConnectionError, IncompleteRead) as error: if attempt == 3: raise print( @@ -110,6 +120,15 @@ def main() -> None: file=sys.stderr, ) time.sleep(attempt) + else: + if digest.hexdigest() == expected_archive_sha256 or attempt == 3: + break + print( + f"WARN: Kimi-Vendor-Verifier archive download attempt " + f"{attempt}/3 failed: archive SHA256 mismatch; retrying", + file=sys.stderr, + ) + time.sleep(attempt) if downloaded == 0: raise ValueError("downloaded archive is empty") actual_archive_sha256 = digest.hexdigest() diff --git a/inferencex-e2e/infx/evals/bfcl_adapter.py b/inferencex-e2e/infx/evals/bfcl_adapter.py index 4a37e31f27..8646526a60 100644 --- a/inferencex-e2e/infx/evals/bfcl_adapter.py +++ b/inferencex-e2e/infx/evals/bfcl_adapter.py @@ -75,13 +75,16 @@ class SuiteSpec: @property def leaf_categories(self) -> tuple[str, ...]: + """Leaf category names in suite order.""" return tuple(category for category, _ in self.expected_leaf_counts) @property def expected_sample_count(self) -> int: + """Total number of cases across all leaf categories.""" return sum(count for _, count in self.expected_leaf_counts) def projected_task(self, category: str) -> str: + """Return the projected lm-eval task name for category.""" if self.name == TASK_NAME: return f"bfcl_{category}" return f"{self.name}_{category}" @@ -162,7 +165,9 @@ def __call__( base_url: str, api_key: str, num_threads: int, - ) -> None: ... + ) -> None: + """Run BFCL generation and evaluation for model against base_url under project_root.""" + ... @dataclass(frozen=True) @@ -177,6 +182,7 @@ class CategoryScore: total_count: int def as_dict(self) -> dict[str, Any]: + """Return the score as a JSON-ready dict with a per-case pass or fail entry.""" failure_ids = {record["id"] for record in self.records} return { "category": self.category, @@ -196,12 +202,17 @@ def as_dict(self) -> dict[str, Any]: def _nonempty_string(value: str) -> str: + """Return value stripped of whitespace, raising ArgumentTypeError if it is blank.""" if not value.strip(): raise argparse.ArgumentTypeError("must be a non-empty string") return value.strip() def _absolute_http_url(value: str) -> str: + """Return value without trailing slashes. + + Raises ArgumentTypeError unless it is an absolute HTTP(S) API root URL. + """ normalized = _nonempty_string(value).rstrip("/") parsed = urllib.parse.urlsplit(normalized) if ( @@ -221,6 +232,7 @@ def _absolute_http_url(value: str) -> str: def _positive_int(value: str) -> int: + """Parse value as an int, raising ArgumentTypeError unless it is greater than 0.""" try: parsed = int(value) except ValueError as exc: @@ -231,9 +243,9 @@ def _positive_int(value: str) -> int: def _source_details( - suite: SuiteSpec, # noqa: ARG001 case_ids_by_category: Mapping[str, tuple[str, ...]], ) -> dict[str, Any]: + """Return the pinned upstream source metadata plus the case IDs selected per category.""" return { "url": UPSTREAM_SOURCE, "repository": UPSTREAM_REPOSITORY, @@ -250,12 +262,14 @@ def _source_details( def _error_dict(error: BaseException) -> dict[str, str]: + """Return the type name and message of error as a dict.""" return {"type": type(error).__name__, "message": str(error)} def _expected_category_details( case_ids_by_category: Mapping[str, tuple[str, ...]], ) -> list[dict[str, Any]]: + """Return placeholder diagnostics that mark every expected case in each category as failed.""" return [ { "category": category, @@ -272,12 +286,12 @@ def _expected_category_details( def _diagnostics( - suite: SuiteSpec, case_ids_by_category: Mapping[str, tuple[str, ...]], scores: Sequence[CategoryScore] | None = None, ) -> dict[str, Any]: + """Return source metadata and per-category diagnostics, with placeholders if scores is None.""" return { - "source": _source_details(suite, case_ids_by_category), + "source": _source_details(case_ids_by_category), "categories": ( [score.as_dict() for score in scores] if scores is not None @@ -296,6 +310,7 @@ def _native_report( scores: Sequence[CategoryScore] | None, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the native BFCL report for model from scores, recording integration_error if given.""" correct_count = sum(score.correct_count for score in scores or ()) total_count = sum(score.total_count for score in scores or ()) accuracy = correct_count / total_count if total_count else 0.0 @@ -317,7 +332,7 @@ def _native_report( "total_count": total_count, "expected_count": suite.expected_sample_count, }, - "bfcl": _diagnostics(suite, case_ids_by_category, scores), + "bfcl": _diagnostics(case_ids_by_category, scores), } if integration_error is not None: report["integration_error"] = _error_dict(integration_error) @@ -332,6 +347,10 @@ def _compatibility_result( scores: Sequence[CategoryScore] | None, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with suite and per-category accuracies. + + Records integration_error in the result when given. + """ score_by_category = {score.category: score for score in scores or ()} total_count = sum(score.total_count for score in scores or ()) correct_count = sum(score.correct_count for score in scores or ()) @@ -390,7 +409,7 @@ def _compatibility_result( "results": results, "configs": configs, "n-samples": task_samples, - "bfcl": _diagnostics(suite, case_ids_by_category, scores), + "bfcl": _diagnostics(case_ids_by_category, scores), } if integration_error is not None: result["integration_error"] = _error_dict(integration_error) @@ -398,6 +417,7 @@ def _compatibility_result( def _write_json(path: Path, value: Mapping[str, Any]) -> None: + """Write value to path as indented UTF-8 JSON.""" path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -437,6 +457,7 @@ def _write_upstream_attribution(project_root: Path) -> None: def _prepare_output_paths(output_dir: Path) -> tuple[Path, Path]: + """Create output_dir, delete stale reports and return the (native, compatibility) paths.""" output_dir.mkdir(parents=True, exist_ok=True) native_path = output_dir / NATIVE_REPORT_FILENAME native_path.unlink(missing_ok=True) @@ -467,6 +488,7 @@ def _bounded_openai_handler(stock_handler: type[Any]) -> type[Any]: class BoundedOpenAICompletionsHandler(stock_handler): def _build_client_kwargs(self) -> dict[str, Any]: + """Return the stock client kwargs with the full-suite timeout and retry limits.""" kwargs = super()._build_client_kwargs() kwargs.update( timeout=FULL_SUITE_REQUEST_TIMEOUT_SECONDS, @@ -478,6 +500,7 @@ def _build_client_kwargs(self) -> dict[str, Any]: def _write_id_map(project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]]) -> None: + """Write the selected case IDs per category to project_root/test_case_ids_to_generate.json.""" project_root.mkdir(parents=True, exist_ok=True) _write_json( project_root / "test_case_ids_to_generate.json", @@ -518,6 +541,11 @@ def _load_dataset_helpers() -> tuple[ def _build_suite_case_ids( suite: SuiteSpec, ) -> dict[str, tuple[str, ...]]: + """Return the case IDs per leaf category that suite evaluates. + + Splits each category limit evenly across its leaves. Raises ValueError on malformed or duplicate + dataset entries or unexpected counts. + """ if suite is SMOKE_SUITE: return dict(SMOKE_CASE_IDS) @@ -561,6 +589,10 @@ def _build_suite_case_ids( def _read_selected_suite( project_root: Path, ) -> tuple[SuiteSpec, dict[str, tuple[str, ...]]]: + """Read the case ID map under project_root and return the suite it matches with those IDs. + + Raises ValueError if the map is malformed or matches no supported suite. + """ raw = json.loads((project_root / "test_case_ids_to_generate.json").read_text(encoding="utf-8")) if not isinstance(raw, Mapping): raise ValueError("BFCL test-case ID map must be a JSON object") @@ -590,7 +622,11 @@ def _run_upstream( api_key: str, num_threads: int, ) -> None: - """Lazily load and invoke the pinned BFCL API against an existing server.""" + """Lazily load and invoke the pinned BFCL API against an existing server. + + Also registers the model config under its result-dir name with every "_" turned + into "/", since that is how BFCL evaluation looks configs up. + """ suite, case_ids_by_category = _read_selected_suite(project_root) _apply_suite_runtime_limits(suite) os.environ["BFCL_PROJECT_ROOT"] = str(project_root) @@ -610,7 +646,7 @@ def _run_upstream( else _bounded_openai_handler(OpenAICompletionsHandler) ) - bfcl_model_config.MODEL_CONFIG_MAPPING[model] = ModelConfig( + model_config = ModelConfig( model_name=model, display_name=f"{model} (FC) (InferenceX)", url="", @@ -622,6 +658,8 @@ def _run_upstream( is_fc_model=True, underscore_to_dot=True, ) + bfcl_model_config.MODEL_CONFIG_MAPPING[model] = model_config + bfcl_model_config.MODEL_CONFIG_MAPPING[model.replace("/", "_").replace("_", "/")] = model_config categories = list(suite.generation_categories) generation_kwargs = _function_defaults(generate) @@ -650,6 +688,10 @@ def _validate_generated_results( project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]] = SMOKE_CASE_IDS, ) -> None: + """Check each category has one result file under project_root with the expected case IDs. + + Raises ValueError on malformed files and RuntimeError if any case hit an inference error. + """ for category, case_ids in case_ids_by_category.items(): matches = sorted(project_root.glob(f"result/**/BFCL_v4_{category}_result.json")) if len(matches) != 1: @@ -690,6 +732,11 @@ def _validate_generated_results( def _validate_header(*, category: str, header: Any, expected_count: int) -> tuple[float, int, int]: + """Check a category score header against expected_count and return its counts. + + Returns (accuracy, correct count, total count). Raises ValueError if fields are missing, + mistyped or inconsistent. + """ if not isinstance(header, Mapping): raise ValueError(f"{category} score header must be a JSON object") missing = {"accuracy", "correct_count", "total_count"} - header.keys() @@ -724,6 +771,11 @@ def _collect_scores( project_root: Path, case_ids_by_category: Mapping[str, tuple[str, ...]] = SMOKE_CASE_IDS, ) -> list[CategoryScore]: + """Read and validate each category's BFCL score file under project_root. + + Returns one CategoryScore per category. Raises ValueError if a file is missing, malformed or + disagrees with its header. + """ scores: list[CategoryScore] = [] for category, case_ids in case_ids_by_category.items(): matches = sorted(project_root.glob(f"score/**/BFCL_v4_{category}_score.json")) @@ -958,6 +1010,10 @@ def install_runtime(download_dir: Path, uv: str) -> None: def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse CLI arguments from argv. + + Enforces the options each mode needs and defaults --num-threads per suite. + """ parser = argparse.ArgumentParser(description="Run a pinned BFCL V4 OpenAI completions suite.") parser.add_argument( "--install-runtime", @@ -997,6 +1053,10 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Install the runtime, record an integration error or run the suite. + + Returns 0 on pass, 1 on failure or after recording an error. + """ args = parse_args(argv) if args.install_runtime is not None: try: diff --git a/inferencex-e2e/infx/evals/kimi_vendor_eval.py b/inferencex-e2e/infx/evals/kimi_vendor_eval.py index 9cc9d5f4be..711ea19620 100755 --- a/inferencex-e2e/infx/evals/kimi_vendor_eval.py +++ b/inferencex-e2e/infx/evals/kimi_vendor_eval.py @@ -85,12 +85,17 @@ def build_pytest_command( def _mapping(value: Any, name: str) -> Mapping[str, Any]: + """Return value if it is a mapping, else raise ValueError naming it as name.""" if not isinstance(value, Mapping): raise ValueError(f"{name} must be an object") return value def _expected_total(task_name: str) -> int: + """Return the expected record count for task_name. + + Raises ValueError for an unsupported task. + """ try: return EXPECTED_TOTALS[task_name] except KeyError as exc: @@ -122,6 +127,11 @@ def _project_report( *, task_name: str = TASK_NAME, ) -> tuple[dict[str, Any], bool]: + """Validate a native verifier report and convert it to a compatibility result. + + Returns (result, True if every record passed). Raises ValueError if the report is malformed or + incomplete. + """ expected_total = _expected_total(task_name) root = _mapping(report, "report") summary = _mapping(root.get("summary"), "report.summary") @@ -244,6 +254,10 @@ def _compatibility_result( n_samples: int, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with score over n_samples. + + Records integration_error in the result when given. + """ expected_total = _expected_total(task_name) result: dict[str, Any] = { "result_format": RESULT_FORMAT, @@ -310,6 +324,7 @@ def _write_native_failure( def _write_compatibility(path: Path, result: Mapping[str, Any]) -> None: + """Write result to path as indented JSON.""" path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") @@ -420,6 +435,7 @@ def run_evaluation( def _positive_int(value: str) -> int: + """Parse value as an int, raising ArgumentTypeError unless it is at least 1.""" parsed = int(value) if parsed < 1: raise argparse.ArgumentTypeError("must be a positive integer") @@ -427,6 +443,10 @@ def _positive_int(value: str) -> int: def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse CLI arguments from argv. + + Requires --verifier-dir and --base-url unless --integration-error is set. + """ parser = argparse.ArgumentParser( description="Run the pinned stock Kimi Vendor Verifier tool-schema evaluation." ) @@ -463,6 +483,10 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Run the Kimi verifier eval, or record the given integration error. + + Returns 0 on pass or after recording an error, 1 on failure. + """ args = parse_args(argv) if args.integration_error is not None: args.output_dir.mkdir(parents=True, exist_ok=True) diff --git a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py index 303bc5c401..fcb04545c9 100755 --- a/inferencex-e2e/infx/evals/minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/evals/minimax_m3_full_eval.py @@ -58,7 +58,8 @@ class FullSuiteError(RuntimeError): class _RejectRedirects(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args: Any, **kwargs: Any) -> None: # noqa: ARG002 - return None + """Refuse every redirect by returning None.""" + return _NO_REDIRECT_OPENER = urllib.request.build_opener(_RejectRedirects()) @@ -91,6 +92,10 @@ def verify_source_content(relative_path: str, content: bytes) -> None: def _fetch_source(relative_path: str) -> bytes: + """Download the pinned source file at relative_path, verify its hash and return its bytes. + + Retries transient failures and raises FullSuiteError once every attempt fails. + """ url = source_url(relative_path) request = urllib.request.Request( # noqa: S310 url, @@ -111,6 +116,7 @@ def _fetch_source(relative_path: str) -> bytes: if declared_size is not None and int(declared_size) > MAX_SOURCE_BYTES: raise FullSuiteError(f"pinned source {relative_path} exceeds the size limit") content = response.read(MAX_SOURCE_BYTES + 1) + verify_source_content(relative_path, content) except ( FullSuiteError, OSError, @@ -122,7 +128,6 @@ def _fetch_source(relative_path: str) -> bytes: time.sleep(DOWNLOAD_RETRY_DELAY_SECONDS) continue break - verify_source_content(relative_path, content) return content raise FullSuiteError( f"failed to download pinned source {relative_path} after " @@ -131,6 +136,10 @@ def _fetch_source(relative_path: str) -> bytes: def _validate_sample(content: bytes) -> None: + """Check that pinned sample.jsonl content has the expected number of JSON object rows. + + Raises ValueError otherwise. + """ try: text = content.decode("utf-8") except UnicodeDecodeError as exc: @@ -226,6 +235,7 @@ def build_verifier_command( def _error_dict(error: BaseException) -> dict[str, str]: + """Return the type name and message of error as a dict.""" return {"type": type(error).__name__, "message": str(error)} @@ -236,6 +246,10 @@ def _compatibility_result( effective: int, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with score over effective samples. + + Records integration_error in the result when given. + """ result: dict[str, Any] = { "result_format": RESULT_FORMAT, "eval_adapter": ADAPTER_NAME, @@ -272,6 +286,7 @@ def _compatibility_result( def _compatibility_path(output_dir: Path) -> Path: + """Delete stale compatibility results in output_dir and return a new timestamped result path.""" for stale_path in output_dir.glob(COMPATIBILITY_GLOB): stale_path.unlink() timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%f") @@ -279,10 +294,15 @@ def _compatibility_path(output_dir: Path) -> Path: def _write_json(path: Path, value: Mapping[str, Any]) -> None: + """Write value to path as indented UTF-8 JSON.""" path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _read_native_report(path: Path) -> dict[str, Any]: + """Load the native summary JSON object at path. + + Raises FullSuiteError if it is missing or invalid. + """ try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: @@ -293,8 +313,12 @@ def _read_native_report(path: Path) -> dict[str, Any]: def _read_native_results(path: Path) -> list[dict[str, Any]]: + """Load the native JSONL result rows at path. + + Raises FullSuiteError unless there are exactly the expected number of object rows. + """ try: - lines = path.read_text(encoding="utf-8").splitlines() + lines = path.read_text(encoding="utf-8").removesuffix("\n").split("\n") except OSError as exc: raise FullSuiteError(f"native results are unavailable: {exc}") from exc if len(lines) != EXPECTED_RESULT_COUNT or any(not line.strip() for line in lines): @@ -314,12 +338,17 @@ def _read_native_results(path: Path) -> list[dict[str, Any]]: def _count(value: Any, name: str) -> int: + """Return value if it is a non-negative int, else raise FullSuiteError naming field name.""" if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise FullSuiteError(f"native summary {name} must be a non-negative integer") return value def _rate(value: Any, name: str) -> float: + """Return value as a float if it is a finite rate in [0, 1]. + + Raises FullSuiteError naming field name otherwise. + """ if ( isinstance(value, bool) or not isinstance(value, (int, float)) @@ -460,6 +489,7 @@ def run_full_suite( def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse argv into the prepare-source, run, project or failure subcommand and its options.""" parser = argparse.ArgumentParser( description="Run or project the pinned full MiniMax M3 verifier." ) @@ -488,6 +518,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Run the chosen subcommand and return 0 on success or 1 on failure.""" args = parse_args(argv) if args.command == "prepare-source": try: diff --git a/inferencex-e2e/infx/evals/minimax_provider_eval.py b/inferencex-e2e/infx/evals/minimax_provider_eval.py index 5fd71f7a26..8f1a0c49cb 100755 --- a/inferencex-e2e/infx/evals/minimax_provider_eval.py +++ b/inferencex-e2e/infx/evals/minimax_provider_eval.py @@ -48,6 +48,7 @@ class SmokeSuiteError(RuntimeError): def _mapping(value: Any, name: str) -> Mapping[str, Any]: + """Return value if it is a mapping, else raise ValueError naming it as name.""" if not isinstance(value, Mapping): raise ValueError(f"{name} must be an object") return value @@ -82,6 +83,10 @@ def load_fixture(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: def _normalized_base_url(value: str) -> str: + """Return value stripped of whitespace and trailing slashes. + + Raises ValueError unless it is an absolute HTTP(S) URL without query or fragment. + """ if not isinstance(value, str) or not value.strip(): raise ValueError("base_url must be a non-empty string") normalized = value.strip().rstrip("/") @@ -147,10 +152,15 @@ def build_verifier_command( def _error_dict(error: BaseException) -> dict[str, str]: + """Return the type name and message of error as a dict.""" return {"type": type(error).__name__, "message": str(error)} def _rate(value: Any, name: str) -> float: + """Return value as a float if it is a finite rate in [0, 1]. + + Raises SmokeSuiteError naming field name otherwise. + """ if ( isinstance(value, bool) or not isinstance(value, (int, float)) @@ -162,12 +172,14 @@ def _rate(value: Any, name: str) -> float: def _nonnegative_count(value: Any, name: str) -> int: + """Return value if it is a non-negative int, else raise SmokeSuiteError naming field name.""" if isinstance(value, bool) or not isinstance(value, int) or value < 0: raise SmokeSuiteError(f"native summary {name} must be a non-negative integer") return value def _compatibility_path(output_dir: Path) -> Path: + """Delete stale compatibility results in output_dir and return a new timestamped result path.""" for stale_path in output_dir.glob(COMPATIBILITY_GLOB): stale_path.unlink() timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S.%f") @@ -181,6 +193,10 @@ def _compatibility_result( effective: int, integration_error: BaseException | None = None, ) -> dict[str, Any]: + """Build the lm-eval style result for model with score over effective samples. + + Records integration_error in the result when given. + """ result: dict[str, Any] = { "result_format": RESULT_FORMAT, "eval_adapter": ADAPTER_NAME, @@ -218,6 +234,7 @@ def _compatibility_result( def _write_json(path: Path, value: Mapping[str, Any]) -> None: + """Write value to path as indented UTF-8 JSON.""" path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -227,7 +244,8 @@ def project_native_artifacts(*, output_dir: Path, model: str) -> Path: json.loads((output_dir / NATIVE_REPORT_FILENAME).read_text(encoding="utf-8")), "native summary", ) - result_lines = (output_dir / NATIVE_RESULTS_FILENAME).read_text(encoding="utf-8").splitlines() + result_text = (output_dir / NATIVE_RESULTS_FILENAME).read_text(encoding="utf-8") + result_lines = result_text.removesuffix("\n").split("\n") if len(result_lines) != 1 or not result_lines[0].strip(): raise SmokeSuiteError("native results must contain exactly one row") result = _mapping(json.loads(result_lines[0]), "native result") @@ -350,6 +368,7 @@ def run_evaluation( def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse argv into the run or failure subcommand and its options.""" parser = argparse.ArgumentParser( description="Run the pinned MiniMax M3 smoke through stock verify.py." ) @@ -372,6 +391,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: + """Run the smoke eval, or record the given failure, and return 0 on success or 1 on failure.""" args = parse_args(argv) if args.command == "failure": publish_failure( diff --git a/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py b/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py index 881a7b6c8a..fe8c17325e 100644 --- a/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py +++ b/inferencex-e2e/infx/evals/patches/lm_eval_sitecustomize.py @@ -7,6 +7,10 @@ def _parse_generations(outputs: Any, **kwargs: Any) -> list[Any]: # noqa: ARG001 + """Return the text of every choice in chat completion outputs. + + Falls back to reasoning_content when content is empty. + """ results = [] if not isinstance(outputs, list): outputs = [outputs] @@ -42,6 +46,11 @@ def _apply_chat_template( chat_history: list[dict[str, Any]], add_generation_prompt: bool = True, ) -> Any: + """Render chat_history for the configured tokenizer backend. + + Returns a templated string for local HF tokenizers, the raw messages for remote ones, or a + JSON chat string otherwise. + """ if self.tokenizer_backend == "huggingface" and self.tokenized_requests: return self.tokenizer.apply_chat_template( chat_history, diff --git a/inferencex-e2e/infx/evals/utils.py b/inferencex-e2e/infx/evals/utils.py index 29f55f8f5b..997927281d 100644 --- a/inferencex-e2e/infx/evals/utils.py +++ b/inferencex-e2e/infx/evals/utils.py @@ -8,6 +8,10 @@ def process_docs( dataset: datasets.Dataset, n_repeats: int = 2, seed: int = 3407 ) -> datasets.Dataset: + """Return dataset repeated n_repeats times with answers shuffled into A to D by seed. + + Each row gets its correct letter in answer and its repeat index in repeat_id. + """ rng = random.Random(seed) # noqa: S311 docs = list(dataset) diff --git a/inferencex-e2e/infx/evals/validate_scores.py b/inferencex-e2e/infx/evals/validate_scores.py index a50073c030..ad826b93e2 100644 --- a/inferencex-e2e/infx/evals/validate_scores.py +++ b/inferencex-e2e/infx/evals/validate_scores.py @@ -37,8 +37,33 @@ def load_config(path: str) -> dict: if not isinstance(cfg, dict): raise ValueError("thresholds config must be a mapping") if "default" not in cfg and "models" not in cfg: - return {"default": cfg, "models": {}} - return {"default": cfg.get("default", {}), "models": cfg.get("models", {})} + cfg = {"default": cfg} + if unknown := sorted(set(cfg) - {"default", "models"}, key=str): + raise ValueError( + f"thresholds config has unknown section(s): {', '.join(map(str, unknown))}" + ) + sections = {name: {} if cfg.get(name) is None else cfg[name] for name in ("default", "models")} + for name, section in sections.items(): + if not isinstance(section, dict): + raise ValueError(f"thresholds config {name!r} must be a mapping") + sections["models"] = { + prefix: {} if tasks is None else tasks for prefix, tasks in sections["models"].items() + } + tables = {"default": sections["default"]} + tables.update((f"models.{prefix}", tasks) for prefix, tasks in sections["models"].items()) + for name, tasks in tables.items(): + if not isinstance(tasks, dict): + raise ValueError(f"thresholds config {name!r} must be a mapping") + for task, value in tasks.items(): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError( + f"thresholds config {name}.{task} must be a finite number, got {value!r}" + ) + return sections def detect_model_prefix(meta_env_path: str, override: str | None) -> str | None: @@ -95,8 +120,15 @@ def metric_prefixes(data: dict, task: str, override: str | None) -> tuple[str, . """Resolve score metric prefixes from an explicit override or task config.""" if override is not None: return (override,) - task_config = data.get("configs", {}).get(task, {}) + configs = data.get("configs", {}) + if not isinstance(configs, dict): + raise ValueError("'configs' is not a mapping") + task_config = configs.get(task, {}) + if not isinstance(task_config, dict): + raise ValueError("'configs' entry is not a mapping") metric_list = task_config.get("metric_list", []) + if not isinstance(metric_list, list): + raise ValueError("'metric_list' is not a list") declared = tuple( f"{item['metric']}," for item in metric_list @@ -215,6 +247,10 @@ def validate_batch_manifest( def main() -> int: + """Check each eval result file against its score threshold, printing PASS or FAIL per metric. + + Returns 0 if all metrics pass, 1 if any fail or none matched. + """ for _stream in (sys.stdout, sys.stderr): with contextlib.suppress(AttributeError, ValueError): _stream.reconfigure(line_buffering=True) @@ -287,9 +323,10 @@ def main() -> int: print(f"Loaded thresholds from {thresholds_path}") except (json.JSONDecodeError, OSError, ValueError) as e: print( - f"WARN: could not load thresholds from {thresholds_path}: {e}", + f"FAIL: could not load thresholds from {thresholds_path}: {e}", file=sys.stderr, ) + return 1 prefix = detect_model_prefix(args.meta_env, args.model_prefix) if prefix and prefix in config.get("models", {}): @@ -301,6 +338,7 @@ def main() -> int: failed = False checked = 0 + selector = args.metric_prefix or "declared task metrics" result_files = sorted(glob.glob(args.results_glob)) manifest_errors = validate_batch_manifest( @@ -325,9 +363,18 @@ def main() -> int: for f in result_files: match = CONC_SUFFIX_RE.search(Path(f).name) conc_label = f"[conc={match.group(1)}] " if match else "" - with open(f) as fh: - data = json.load(fh) - if "integration_error" in data: + try: + with open(f) as fh: + data = json.load(fh) + except json.JSONDecodeError as exc: + print(f"FAIL: unreadable result file {f}: invalid JSON: {exc}", file=sys.stderr) + failed = True + continue + except (OSError, UnicodeDecodeError) as exc: + print(f"FAIL: unreadable result file {f}: {exc}", file=sys.stderr) + failed = True + continue + if isinstance(data, dict) and "integration_error" in data: print( f"FAIL: {conc_label}integration failure: " f"{integration_error_message(data['integration_error'])}", @@ -335,7 +382,17 @@ def main() -> int: ) failed = True continue - for task, metrics in data.get("results", {}).items(): + results = data.get("results", {}) if isinstance(data, dict) else None + if not isinstance(results, dict): + print( + f"FAIL: unreadable result file {f}: " + "expected a JSON object with a 'results' mapping", + file=sys.stderr, + ) + failed = True + continue + file_checked = checked + for task, metrics in results.items(): invalid_effective, effective = invalid_effective_count(data, task) if invalid_effective: print( @@ -344,15 +401,33 @@ def main() -> int: ) failed = True continue + if not isinstance(metrics, dict): + print( + f"FAIL: {conc_label}{task} malformed result: 'results' entry is not a mapping", + file=sys.stderr, + ) + failed = True + continue min_score, source = resolve_threshold(config, prefix, task, args.min_score) - prefixes = metric_prefixes(data, task, args.metric_prefix) + try: + prefixes = metric_prefixes(data, task, args.metric_prefix) + except ValueError as exc: + print(f"FAIL: {conc_label}{task} malformed result: {exc}", file=sys.stderr) + failed = True + continue for name, val in metrics.items(): if not name.startswith(prefixes) or "stderr" in name: continue if not isinstance(val, (int, float)): continue checked += 1 - if val < min_score: + if isinstance(val, bool) or not math.isfinite(val): + print( + f"FAIL: {conc_label}{task} {name} = {val} is not a finite number", + file=sys.stderr, + ) + failed = True + elif val < min_score: print( f"FAIL: {conc_label}{task} {name} = {val:.4f} (< {min_score} from {source})", file=sys.stderr, @@ -362,9 +437,11 @@ def main() -> int: print( f"PASS: {conc_label}{task} {name} = {val:.4f} (>= {min_score} from {source})" ) + if checked == file_checked: + print(f"FAIL: {conc_label}no metrics matched {selector!r} in {f}", file=sys.stderr) + failed = True if checked == 0: - selector = args.metric_prefix or "declared task metrics" print(f"WARN: no metrics matched {selector!r}", file=sys.stderr) return 1 if (failed or checked == 0) else 0 diff --git a/inferencex-e2e/infx/fastcopy.py b/inferencex-e2e/infx/fastcopy.py new file mode 100644 index 0000000000..194e5e53f6 --- /dev/null +++ b/inferencex-e2e/infx/fastcopy.py @@ -0,0 +1,33 @@ +"""``copy.deepcopy`` without its per-object dispatch for the dicts and lists YAML and JSON build.""" + +from __future__ import annotations + +import copy +from typing import Any + + +def deepcopy(value: Any) -> Any: + """Return what ``copy.deepcopy(value)`` returns, shared references included.""" + return _copy(value, {}) + + +def _copy(value: Any, memo: dict[int, Any]) -> Any: + """Return a deep copy of ``value``, reusing copies already recorded in ``memo`` by object id.""" + cls = type(value) + if cls is str or cls is int or cls is float or cls is bool or value is None: + return value + if (found := memo.get(id(value))) is not None: + return found + if cls is dict: + result: Any = {} + memo[id(value)] = result + for key, item in value.items(): + result[_copy(key, memo)] = _copy(item, memo) + return result + if cls is list: + result = [] + memo[id(value)] = result + for item in value: + result.append(_copy(item, memo)) + return result + return copy.deepcopy(value, memo) diff --git a/inferencex-e2e/infx/github.py b/inferencex-e2e/infx/github.py index 91a9a57a28..fe8f16cc1c 100644 --- a/inferencex-e2e/infx/github.py +++ b/inferencex-e2e/infx/github.py @@ -19,7 +19,8 @@ class APIError(RuntimeError): """Authenticated gh failure, with an HTTP status when gh reports one.""" def __init__(self, path: str, stderr: str) -> None: - match = re.search(r"\(HTTP ([1-5][0-9]{2})\)\s*$", stderr) + """Wrap a failed gh call on ``path``, parsing the HTTP status from ``stderr`` even when hints follow it.""" + match = re.search(r"\(HTTP ([1-5][0-9]{2})\)\s*$", stderr, re.MULTILINE) self.status = int(match[1]) if match else None super().__init__(f"GitHub API {path} failed: {stderr.strip()}") @@ -34,6 +35,10 @@ def api( data: dict[str, Any] | None = None, paginate: bool = False, ) -> Any: + """Call the GitHub REST endpoint ``path`` under ``repo`` through gh and return the decoded JSON. + + With ``token``, failures raise ``APIError`` and an empty DELETE reply returns None. + """ if token is not None and not token.strip(): raise RuntimeError("GitHub API requires a non-empty token") endpoint, _, query = path.partition("?") @@ -84,6 +89,10 @@ def paginate( item_key: str = "", params: dict[str, str] | None = None, ) -> list[dict[str, Any]]: + """Fetch every page of a listing and return its rows, read from ``item_key`` when pages are objects. + + Raises ``ListingError`` when the listing is missing, malformed, or shorter than ``total_count``. + """ pages = api( repo, path, token, {**(params or {}), "per_page": "100", "page": "1"}, paginate=True ) diff --git a/inferencex-e2e/infx/golden_al_distribution/README.md b/inferencex-e2e/infx/golden_al_distribution/README.md index 59c4f752b5..dfdc080cf5 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README.md +++ b/inferencex-e2e/infx/golden_al_distribution/README.md @@ -24,8 +24,8 @@ AL is workload-dependent: a draft model's predictions are easier to accept in so Under the AgentX Guidelines, each model, thinking mode, and draft length has one committed golden AL. Once synthetic acceptance is enabled for a benchmark scenario, a submission may choose any supported draft length, but it may not substitute a different acceptance target. Different models keep their own SPEED-Bench-derived curves. All submissions evaluating the same model and mode use the same curve. -For Qwen3.5 srt-slurm launches, `apply_srt_recipe` uses the -`infx/srt_slurm/synthetic_acceptance.py` connector to select the committed curve +For Qwen3.5 srt-slurm launches, the srt driver runs the +`infx/srt_slurm/synthetic_acceptance.py` connector, which selects the committed curve from the generation role's `speculative-num-steps` and `THINKING_MODE` after caller overrides. The verification token is not an additional MTP step. For SGLang MTP synthetic throughput, the connector rejects a generation role whose @@ -85,11 +85,11 @@ This policy follows the same broad principle as MLPerf Inference: prescribe the The push-button [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.yml) workflow, introduced in [InferenceX#1650](https://github.com/SemiAnalysisAI/InferenceX/pull/1650) and extended to additional MTP and EAGLE3 models in [InferenceX#1706](https://github.com/SemiAnalysisAI/InferenceX/pull/1706), performs the following process. It superseded the early manually assembled reference in [InferenceX#1592](https://github.com/SemiAnalysisAI/InferenceX/pull/1592), making the exact commands, logs, outputs, and generated YAML auditable from one run. -1. A maintainer dispatches the workflow with a model, model prefix, vLLM image, draft lengths (normally 1–8), thinking modes, `category=coding`, and `output-len=4096`. -2. The workflow launches the model on a B300 runner and selects the matching collector under [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench). -3. For every `(thinking mode, draft length)` cell, the collector starts a clean vLLM server with real MTP or EAGLE3 decoding and the model's production sampling/chat-template settings. -4. The collector snapshots vLLM's cumulative accepted-token and verification-draft counters, runs every prompt in the SPEED-Bench Qualitative `coding` category through `vllm bench serve`, and snapshots the counters again. -5. It computes the mean acceptance length as: +1. A maintainer dispatches the workflow with a model, model prefix, serving framework (`vllm` or `sglang`), image, draft lengths (normally 1–8), thinking modes, `category=coding`, and `output-len=4096`. +2. The workflow launches the model on a B300 runner and selects the matching `_fp4_b300_.sh` collector under [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench). A non-empty `collector-variant` input selects `_fp4_b300__.sh` instead. For example, `model-prefix=kimik3` with `collector-variant=probabilistic_sample_method_block_rejection_sample_method` runs the Kimi K3 probabilistic-draft collector. The run fails before launch if the selected script does not exist. +3. For every `(thinking mode, draft length)` cell, the collector starts a clean vLLM or SGLang server with real speculative decoding and the model's production sampling/chat-template settings. +4. A vLLM collector snapshots vLLM's cumulative accepted-token and verification-draft counters, runs every prompt in the SPEED-Bench Qualitative `coding` category through `vllm bench serve`, and snapshots the counters again. The SGLang collector runs the same prompts through `sglang.bench_serving` and records the `accept_length` it reports. +5. A vLLM collector computes the mean acceptance length as: ```text AL = 1 + (delta accepted draft tokens / delta verification drafts) @@ -98,7 +98,7 @@ The push-button [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.y The `1` is the target model's guaranteed verification token. Values are rounded to two decimal places. 6. The collector emits a YAML matrix. The workflow publishes it in the GitHub Actions step summary and uploads it as a `speedbench-reference-al-` artifact. 7. Server logs and detailed per-request results are retained so reviewers can confirm sensible output, correct thinking mode, and the absence of silent server or chat-template failures. -8. After review, the matrix is committed here with its exact sampling metadata and source Actions run URL. +8. After review, the matrix is committed here with its exact sampling metadata and source Actions run URL. With `open-pr=true` and `golden-curve=` (for example `dsv4_mtp`), the workflow writes `.yaml` here with the source run URL on its first line. It loads the file through `python -m infx.golden_al_distribution show` and then opens the PR. The whole file is replaced, so cells the run did not collect are dropped. The collection measures real head quality only to establish the golden curve. AgentX then uses that committed curve as the synthetic acceptance target for every comparable submission. @@ -166,7 +166,7 @@ python -m infx.golden_al_distribution lookup qwen3.5 mtp 3 # 3.39 (thin python -m infx.golden_al_distribution lookup kimik3 dspark 7 --draft-sample-method probabilistic --json ``` -`lookup` takes the InferenceX model prefix, the engine's method name (`mtp`, `eagle`, `nextn`, `eagle3`, `dspark`) and `num_speculative_tokens`. `--draft-model` selects the MiniMax GQA curve, and `--draft-sample-method` chooses between the two Kimi DSpark curves. A missing curve or cell exits non-zero, exactly as a launch would fail. +`lookup` takes the InferenceX model prefix, the engine's method name (`mtp`, `eagle`, `nextn`, `eagle3`, `dspark`, `dflash`) and `num_speculative_tokens`. `--draft-model` selects the MiniMax GQA curve, and `--draft-sample-method` chooses between the two Kimi DSpark curves. A missing curve or cell exits non-zero, exactly as a launch would fail. ## Primary references diff --git a/inferencex-e2e/infx/golden_al_distribution/README_zh.md b/inferencex-e2e/infx/golden_al_distribution/README_zh.md index 8e01779b71..e25bdd372c 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README_zh.md +++ b/inferencex-e2e/infx/golden_al_distribution/README_zh.md @@ -24,7 +24,7 @@ AL 取决于工作负载:草稿模型的预测在某些领域比其他领域 根据 AgentX 指南,每个模型、思考模式和草稿长度都有一个已提交的黄金 AL。当某个基准场景启用合成接受后,提交可以选择任意受支持的草稿长度,但不能替换为其他接受目标。不同模型保留各自基于 SPEED-Bench 测得的曲线。所有评估同一模型和模式的提交都使用同一条曲线。 -对于 Qwen3.5 的 srt-slurm 启动,`apply_srt_recipe` 通过 `infx/srt_slurm/synthetic_acceptance.py` 连接器,在应用调用方覆盖设置后,根据生成角色的 `speculative-num-steps` 和 `THINKING_MODE` 选择已提交的曲线。验证 token 不计为额外的 MTP 步数。对于采用合成接受的 SGLang MTP 吞吐测试,如果生成角色缺少 `speculative-num-draft-tokens`,或者其值不是等于 `speculative-num-steps + 1` 的整数,连接器会拒绝启动。无需手动设置接受长度。仅评测运行会移除合成设置并使用真实验证。 +对于 Qwen3.5 的 srt-slurm 启动,srt 驱动会运行 `infx/srt_slurm/synthetic_acceptance.py` 连接器,在应用调用方覆盖设置后,根据生成角色的 `speculative-num-steps` 和 `THINKING_MODE` 选择已提交的曲线。验证 token 不计为额外的 MTP 步数。对于采用合成接受的 SGLang MTP 吞吐测试,如果生成角色缺少 `speculative-num-draft-tokens`,或者其值不是等于 `speculative-num-steps + 1` 的整数,连接器会拒绝启动。无需手动设置接受长度。仅评测运行会移除合成设置并使用真实验证。 vLLM 通过合成拒绝采样支持这一策略。例如,EAGLE3 运行可以通过 `synthetic_acceptance_length` 注入所选 YAML 值: @@ -78,11 +78,11 @@ python -m atom.entrypoints.openai_server \ 一键触发的 [`speedbench-al.yml`](../../../.github/workflows/speedbench-al.yml) 工作流最初由 [InferenceX#1650](https://github.com/SemiAnalysisAI/InferenceX/pull/1650) 引入,随后在 [InferenceX#1706](https://github.com/SemiAnalysisAI/InferenceX/pull/1706) 中扩展到更多 MTP 和 EAGLE3 模型。它取代了 [InferenceX#1592](https://github.com/SemiAnalysisAI/InferenceX/pull/1592) 中早期手工整理的参考值,使精确命令、日志、输出和生成的 YAML 都可以从同一次运行中审计。其流程如下: -1. 维护者触发工作流,指定模型、模型前缀、vLLM 镜像、草稿长度(通常为 1–8)、思考模式、`category=coding` 和 `output-len=4096`。 -2. 工作流在 B300 runner 上启动模型,并选择 [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench) 下对应的收集脚本。 -3. 对每个“思考模式 × 草稿长度”组合,收集脚本使用真实 MTP 或 EAGLE3 解码以及该模型的生产采样和聊天模板设置,启动一个干净的 vLLM 服务。 -4. 收集脚本读取 vLLM 累计的已接受 token 和验证草稿计数器,通过 `vllm bench serve` 运行 SPEED-Bench Qualitative `coding` 类别中的全部提示词,然后再次读取计数器。 -5. 按以下公式计算平均接受长度: +1. 维护者触发工作流,指定模型、模型前缀、服务框架(`vllm` 或 `sglang`)、镜像、草稿长度(通常为 1–8)、思考模式、`category=coding` 和 `output-len=4096`。 +2. 工作流在 B300 runner 上启动模型,并选择 [`benchmarks/single_node/speedbench/`](../../benchmarks/single_node/speedbench) 下对应的 `_fp4_b300_.sh` 收集脚本。若 `collector-variant` 输入非空,则改为选择 `_fp4_b300__.sh`。例如 `model-prefix=kimik3` 加 `collector-variant=probabilistic_sample_method_block_rejection_sample_method` 会运行 Kimi K3 概率草稿收集脚本。所选脚本不存在时,工作流会在启动前失败。 +3. 对每个“思考模式 × 草稿长度”组合,收集脚本使用真实投机解码以及该模型的生产采样和聊天模板设置,启动一个干净的 vLLM 或 SGLang 服务。 +4. vLLM 收集脚本读取 vLLM 累计的已接受 token 和验证草稿计数器,通过 `vllm bench serve` 运行 SPEED-Bench Qualitative `coding` 类别中的全部提示词,然后再次读取计数器。SGLang 收集脚本通过 `sglang.bench_serving` 运行相同的提示词,并记录其报告的 `accept_length`。 +5. vLLM 收集脚本按以下公式计算平均接受长度: ```text AL = 1 + (delta accepted draft tokens / delta verification drafts) @@ -91,7 +91,7 @@ python -m atom.entrypoints.openai_server \ 其中 `1` 是目标模型保证生成的验证 token。结果四舍五入到小数点后两位。 6. 收集脚本生成 YAML 矩阵。工作流将其发布到 GitHub Actions step summary,并上传为 `speedbench-reference-al-` artifact。 7. 工作流保留服务日志和逐请求详细结果,以便审阅者确认输出合理、思考模式正确,并且没有静默的服务或聊天模板故障。 -8. 审阅完成后,将矩阵连同准确的采样元数据和源 Actions run URL 一并提交到本目录。 +8. 审阅完成后,将矩阵连同准确的采样元数据和源 Actions run URL 一并提交到本目录。设置 `open-pr=true` 和 `golden-curve=<曲线文件名>`(例如 `dsv4_mtp`)时,工作流会写入本目录下的 `<曲线文件名>.yaml`,首行为源 run URL,再通过 `python -m infx.golden_al_distribution show` 加载校验,然后开启 PR。它会替换整个文件,因此本次未收集的格子会被删除。 收集过程仅通过测量真实草稿头质量来建立黄金曲线。之后,AgentX 对所有可比较提交使用该已提交曲线作为合成接受目标。 @@ -159,7 +159,7 @@ python -m infx.golden_al_distribution lookup qwen3.5 mtp 3 # 3.39(默 python -m infx.golden_al_distribution lookup kimik3 dspark 7 --draft-sample-method probabilistic --json ``` -`lookup` 接受 InferenceX 模型前缀、引擎的方法名(`mtp`、`eagle`、`nextn`、`eagle3`、`dspark`)以及 `num_speculative_tokens`。`--draft-model` 用于选择 MiniMax GQA 曲线,`--draft-sample-method` 用于在两条 Kimi DSpark 曲线之间选择。缺少曲线或数值时以非零状态退出,与启动时的失败方式一致。 +`lookup` 接受 InferenceX 模型前缀、引擎的方法名(`mtp`、`eagle`、`nextn`、`eagle3`、`dspark`、`dflash`)以及 `num_speculative_tokens`。`--draft-model` 用于选择 MiniMax GQA 曲线,`--draft-sample-method` 用于在两条 Kimi DSpark 曲线之间选择。缺少曲线或数值时以非零状态退出,与启动时的失败方式一致。 ## 主要参考资料 diff --git a/inferencex-e2e/infx/golden_al_distribution/__main__.py b/inferencex-e2e/infx/golden_al_distribution/__main__.py index 7180afa74f..15a20a7dfa 100644 --- a/inferencex-e2e/infx/golden_al_distribution/__main__.py +++ b/inferencex-e2e/infx/golden_al_distribution/__main__.py @@ -26,6 +26,7 @@ def token_range(tokens: list[int]) -> str: + """Render draft lengths as ``a-b`` if contiguous, else comma-separated, or ``-`` if empty.""" if not tokens: return "-" if tokens == list(range(tokens[0], tokens[-1] + 1)) and len(tokens) > 1: @@ -34,6 +35,7 @@ def token_range(tokens: list[int]) -> str: def curve_json(curve: Curve) -> dict: + """Return a curve as a JSON-ready dict of its name, model and per-mode AL values.""" return { "curve": curve.name, "model": curve.model, @@ -45,6 +47,7 @@ def curve_json(curve: Curve) -> dict: def list_command(args: argparse.Namespace) -> None: + """Print every committed curve with its draft-length ranges, as a table or JSON.""" curves = list_curves(args.golden_dir) if args.json: print(json.dumps([curve_json(curve) for curve in curves], indent=2)) @@ -62,6 +65,7 @@ def list_command(args: argparse.Namespace) -> None: def show_command(args: argparse.Namespace) -> None: + """Print one curve's AL per draft length and thinking mode, as a table or JSON.""" curve = load_curve(args.curve, args.golden_dir) if args.json: print(json.dumps(curve_json(curve), indent=2)) @@ -71,7 +75,7 @@ def show_command(args: argparse.Namespace) -> None: print(" ".join(["tokens", *modes])) for tokens in sorted({token for mode in modes for token in curve.tokens(mode)}): cells = [ - f"{float(curve.modes[mode][tokens]):.2f}" if tokens in curve.modes[mode] else "-" + f"{float(curve.modes[mode][tokens]):.2f}" if tokens in curve.tokens(mode) else "-" for mode in modes ] print( @@ -85,6 +89,7 @@ def show_command(args: argparse.Namespace) -> None: def lookup_command(args: argparse.Namespace) -> None: + """Print the golden AL for the model, method, draft length and thinking mode in ``args``.""" spec = {"method": args.method, "num_speculative_tokens": args.tokens} if args.draft_model: spec["model"] = args.draft_model @@ -107,6 +112,7 @@ def lookup_command(args: argparse.Namespace) -> None: def main(argv: list[str] | None = None) -> int: + """Parse ``argv``, run the chosen subcommand and return the exit code.""" parser = argparse.ArgumentParser( prog="python -m infx.golden_al_distribution", description=__doc__, @@ -128,7 +134,9 @@ def main(argv: list[str] | None = None) -> int: "lookup", help="the AL synthetic acceptance applies for a model prefix and method" ) lookup_parser.add_argument("model", help="InferenceX model prefix, e.g. qwen3.5, dsv4, kimik3") - lookup_parser.add_argument("method", help="spec method: mtp, eagle, nextn, eagle3, dspark") + lookup_parser.add_argument( + "method", help="spec method: mtp, eagle, nextn, eagle3, dspark, dflash" + ) lookup_parser.add_argument("tokens", type=int, help="num_speculative_tokens") lookup_parser.add_argument("--thinking", choices=THINKING_MODES, default="thinking_on") lookup_parser.add_argument("--draft-model", help="draft checkpoint (selects MiniMax GQA)") diff --git a/inferencex-e2e/infx/golden_al_distribution/curves.py b/inferencex-e2e/infx/golden_al_distribution/curves.py index 64d6e6d713..4618bb1533 100644 --- a/inferencex-e2e/infx/golden_al_distribution/curves.py +++ b/inferencex-e2e/infx/golden_al_distribution/curves.py @@ -9,10 +9,11 @@ from pathlib import Path from typing import Any -import yaml +from infx import yamlio GOLDEN_DIR = Path(__file__).resolve().parent THINKING_MODES = ("thinking_on", "thinking_off") +CURVE_NAME = re.compile(r"[a-z0-9_.-]+") @dataclass(frozen=True) @@ -24,9 +25,11 @@ class Curve: modes: Mapping[str, Mapping[int, Any]] def tokens(self, thinking: str) -> list[int]: + """Return the sorted draft lengths measured for one thinking mode.""" return sorted(self.modes.get(thinking) or {}) def acceptance(self, thinking: str, tokens: int) -> float: + """Return the AL for a thinking mode and draft length, or raise ValueError if invalid.""" try: value = float(self.modes[thinking][tokens]) except (KeyError, TypeError, ValueError) as error: @@ -39,9 +42,12 @@ def acceptance(self, thinking: str, tokens: int) -> float: def curve_name(model: str, spec: Mapping[str, Any]) -> str: - """Map an InferenceX model prefix and speculative config to a curve file stem.""" + """Map an InferenceX model prefix and speculative config to a curve file stem. + + SGLang names native model MTP EAGLE or NEXTN, so those map to the model's own head curve. + Kimi DSpark has one curve per draft sampler and raises ValueError for any other sampler. + """ method = str(spec.get("method", "")).lower() - # SGLang calls native model MTP EAGLE/NEXTN; the curve describes the model's head. if method in ("eagle", "nextn"): method = "eagle3" if model == "minimaxm3" else "mtp" curve = f"{model}_{method}" @@ -51,31 +57,37 @@ def curve_name(model: str, spec: Mapping[str, Any]) -> str: if "gqa" in str(spec.get("model", "")).lower(): curve += "_gqa" elif model == "kimik3" and method == "dspark": - # Kimi has distinct measured curves; require the recipe to choose its sampler. sampling = spec.get("draft_sample_method") if sampling == "probabilistic": curve += "_probabilistic_sample_method_block_rejection_sample_method" elif sampling != "greedy": raise ValueError(f"No Kimi DSpark golden curve for draft sampling {sampling!r}") - if not re.fullmatch(r"[a-z0-9_.-]+", curve): + if not CURVE_NAME.fullmatch(curve): raise ValueError(f"Invalid golden curve identity: {curve!r}") return curve def load_curve(name: str, golden_dir: Path = GOLDEN_DIR) -> Curve: + """Load ``.yaml`` from ``golden_dir``, or raise ValueError if missing or malformed.""" + if not CURVE_NAME.fullmatch(name): + raise ValueError(f"Invalid golden curve identity: {name!r}") path = golden_dir / f"{name}.yaml" if not path.is_file(): raise ValueError(f"No committed golden curve {path.name} in {golden_dir}") - data = yaml.safe_load(path.read_text()) + data = yamlio.safe_load(path.read_text()) if not isinstance(data, dict) or len(data) != 1: raise ValueError(f"Expected one model in golden curve {path.name}") [(model, modes)] = data.items() if not isinstance(modes, dict): raise ValueError(f"Expected thinking modes in golden curve {path.name}") + for mode, cells in modes.items(): + if cells is not None and not isinstance(cells, dict): + raise ValueError(f"Expected draft lengths in {mode} of golden curve {path.name}") return Curve(name=name, model=str(model), modes=modes) def list_curves(golden_dir: Path = GOLDEN_DIR) -> list[Curve]: + """Return every committed curve in ``golden_dir``, sorted by name.""" return [load_curve(path.stem, golden_dir) for path in sorted(golden_dir.glob("*.yaml"))] diff --git a/inferencex-e2e/infx/klaud/__main__.py b/inferencex-e2e/infx/klaud/__main__.py index 54cbc447a4..f528e8d073 100644 --- a/inferencex-e2e/infx/klaud/__main__.py +++ b/inferencex-e2e/infx/klaud/__main__.py @@ -43,11 +43,11 @@ from infx.clusters import RunnerInventory PRODUCERS = "producers.json" -# An abort is lifted by removing its label, so it never delays the next attempt by itself. RETRY_NEXT_WAVE = frozenset({"capacity-deferred", "agent-not-started", "aborted"}) def observation_key(row: dict) -> tuple: + """Return the public identity tuple of feed ``row``, with its image normalized.""" image = row.get("image") if isinstance(image, str): image = normalized_image(image) @@ -69,6 +69,7 @@ def observation_key(row: dict) -> tuple: def target_clusters(entries: list[dict], inventory: RunnerInventory) -> frozenset[str]: + """Return the cluster IDs serving the runners of ``entries``, or none if one is unknown.""" try: return frozenset( inventory.cluster_for(node).id @@ -89,6 +90,7 @@ def live_families(root: Path) -> dict[tuple, dict[str, tuple[frozenset[str], int from infx.config import project_root from infx.matrix.generate import _hardware_family, generate_test_config_sweep from infx.matrix.validation import load_config_files, load_runner_file + from infx.results.identity import public_framework from .validation import in_scope @@ -122,7 +124,7 @@ def live_families(root: Path) -> dict[tuple, dict[str, tuple[frozenset[str], int row = { "model": entry["model-prefix"], "hardware": _hardware_family(entry["runner"]), - "framework": entry["framework"], + "framework": public_framework(entry["framework"]), "precision": entry["precision"], "spec_method": entry["spec-decoding"], "disagg": entry.get("disagg", False), @@ -200,16 +202,21 @@ def choose( def weights_key(source: dict) -> tuple: + """Return the (model, precision, hardware) key that identifies the weights ``source`` needs.""" return (source.get("model"), source.get("precision"), source.get("hardware")) def recent_candidate_outcomes( repository: str, cooldown_hours: int ) -> tuple[dict[str, str | None], set[tuple]]: + """Return each candidate's latest outcome within ``cooldown_hours`` and missing-weights keys. + + Resumed sessions record outcomes in later waves, so history reaches MAX_SESSIONS waves back. + The newest session artifact counts. Unreadable history means no cooldown. + """ from .lifecycle import MAX_SESSIONS, WAVE_HOURS, resume_parent cutoff = datetime.now(UTC) - timedelta(hours=cooldown_hours) - # Resumed sessions of an older planner run record their outcomes in later waves. since = cutoff - timedelta(hours=MAX_SESSIONS * WAVE_HOURS) try: runs = github_items( @@ -227,7 +234,6 @@ def recent_candidate_outcomes( created = utc(run["created_at"]) if created >= since: dated.append((created, int(run["id"]))) - # A resume run's candidates come from its planner run's plan. plans[int(run["id"])] = resume_parent(run.get("display_title") or "") or int( run["id"] ) @@ -253,7 +259,6 @@ def recent_candidate_outcomes( or utc(artifact["created_at"]) < cutoff ): continue - # Each session of a candidate uploads its own artifact; the newest one counts. if match[1] not in latest or artifact["id"] > latest[match[1]][0]["id"]: latest[match[1]] = (artifact, run_id) except (AttributeError, KeyError, TypeError, ValueError): @@ -282,6 +287,7 @@ def recent_candidate_outcomes( def planned_candidates(repository: str, artifacts: list[dict]) -> list[dict]: + """Return the candidates in the klaud-plan artifact among ``artifacts``, or [] if unreadable.""" plan = next((a for a in artifacts if a.get("name") == "klaud-plan"), None) if not plan: return [] @@ -297,6 +303,10 @@ def planned_candidates(repository: str, artifacts: list[dict]) -> list[dict]: def recorded_outcome(repository: str, artifact: dict) -> str | None: + """Return the outcome a diagnostics ``artifact`` recorded, or None if it is unreadable. + + A skipped agent yields ``agent-not-started`` and a missing-weights failure ``weights-missing``. + """ with tempfile.TemporaryDirectory() as scratch: try: download_json(repository, artifact, Path(scratch)) @@ -318,6 +328,10 @@ def recorded_outcome(repository: str, artifact: dict) -> str | None: def order_by_cooldown(candidates: list[dict], recent: dict[str, str | None]) -> list[dict]: + """Return ``candidates`` never tried recently, then those whose recent outcome retries now. + + An abort is lifted by removing its label, so it never delays the next attempt by itself. + """ fresh = [candidate for candidate in candidates if candidate["id"] not in recent] deferred = [ candidate for candidate in candidates if recent.get(candidate["id"], "") in RETRY_NEXT_WAVE @@ -379,6 +393,7 @@ def abort_released(repository: str, candidate_id: str) -> bool: def active_pulls(pulls: list[dict], now: datetime, stale_days: int) -> list[dict]: + """Return ``pulls`` on Klaud branches or updated within ``stale_days`` before ``now``.""" cutoff = now - timedelta(days=stale_days) return [ pr @@ -391,6 +406,11 @@ def active_pulls(pulls: list[dict], now: datetime, stale_days: int) -> list[dict def plan( root: Path, directory: Path, review_batch_size: int, cooldown_hours: int, stale_pr_days: int ) -> None: + """Write ``candidates.json`` and ``open-prs.json`` under ``directory`` for overlap review. + + Unclaimed, unaborted feed candidates at ``root``'s HEAD are cooled down by + ``cooldown_hours`` and capped at ``review_batch_size``. PRs idle ``stale_pr_days`` drop out. + """ from . import claims policy = Policy() @@ -421,12 +441,10 @@ def plan( if recovery_file.exists(): blocked.update(json.loads(recovery_file.read_text())) candidates = [candidate for candidate in candidates if candidate["family"] not in blocked] - # klaud-abort keeps a candidate out of the pool until someone removes the label. aborted = aborted_candidates(repository) candidates = [candidate for candidate in candidates if candidate["id"] not in aborted] recent, missing_weights = recent_candidate_outcomes(repository, cooldown_hours) for candidate in candidates: - # Only candidates the cooldown would hold back need their abort history. held = candidate["id"] in recent and recent[candidate["id"]] not in RETRY_NEXT_WAVE if held and abort_released(repository, candidate["id"]): recent[candidate["id"]] = "aborted" @@ -500,7 +518,10 @@ def plan( def denied_bash_category(denial: dict) -> str: - """Retain only fixed categories, never command text, arguments or paths.""" + """Return a fixed category for the Bash command in permission ``denial``. + + Command text, arguments and paths are never retained. + """ command = denial.get("tool_input", {}) command = command.get("command") if isinstance(command, dict) else None if not isinstance(command, str): @@ -538,6 +559,7 @@ def denied_bash_category(denial: dict) -> str: def execution_diagnostics(path: Path) -> dict: + """Return numeric metrics, termination and denial categories from the log at ``path``.""" try: messages = json.loads(path.read_text()) if not isinstance(messages, list): @@ -610,13 +632,15 @@ def execution_diagnostics(path: Path) -> dict: def check_stop() -> dict: - """Require a verified terminal outcome, not merely an agent's final response.""" + """Return the Stop hook decision, blocking unless a verified terminal outcome exists. + + An aborted candidate never blocks, since the recover-current step cleans it up. + """ from .lifecycle import current_session try: session = current_session() pulls = session.pulls() - # The job's recover-current step cancels an aborted candidate's runs and closes its PR. if pulls and session.aborted(pulls[0]): return {} outcome = CandidateOutcome.model_validate_json( @@ -662,7 +686,6 @@ def resume_check() -> dict[str, bool]: if run is None or int(run) not in same: return no jobs = [job for job in candidate_jobs(session) if started_session(job)] - # This job's own session is already one of the started ones. if len(jobs) > MAX_SESSIONS or any( job["run_id"] in same and job["run_id"] != int(run) for job in jobs ): @@ -678,6 +701,10 @@ def resume_check() -> dict[str, bool]: def save_diagnostics( execution_file: Path, outcome_file: Path, action_outcome: str, output: Path ) -> bool: + """Write sanitized session diagnostics to ``output`` and return False if unverified. + + The outcome comes from the posted receipt, the evidence outcome, then ``outcome_file``. + """ diagnostics = { "action-outcome": action_outcome, **execution_diagnostics(execution_file), @@ -846,6 +873,11 @@ def regenerate_producers(directory: Path) -> None: def select( directory: Path, max_candidates: int, execution_file: Path | None = None, resumed: int = 0 ) -> None: + """Claim reviewed candidates that fit capacity and have a verified baseline. + + Takes up to ``max_candidates`` less the ``resumed`` sessions sharing the cap, and writes + their context, ownership.json and selection.json under ``directory``. + """ from . import claims, validation from .reporting import BaselinePreflight, Prose, missing_baseline_points, resolve_baseline @@ -854,7 +886,6 @@ def select( review, deferred = reviewed(contexts) decisions = {decision.candidate_id: decision for decision in review.decisions} selected = [] - # Sessions that recovery resumed in this wave run beside its new candidates and share the cap. slots = max(0, max_candidates - resumed) headroom: dict[str, int] = {} if slots and any(decision.decision == "proceed" for decision in review.decisions): @@ -882,6 +913,9 @@ def select( {key: candidate[key] for key in ("id", "family", "base")} ) try: + matrix = validation.canonical_matrix( + os.environ["GITHUB_REPOSITORY"], owned.base, owned.family + ) baseline = resolve_baseline( os.environ["GITHUB_REPOSITORY"], owned, @@ -892,6 +926,7 @@ def select( zh="在调度候选任务前验证完整的已发布基线。", ), producers.get(candidate["family"], {}), + matrix, ) preflight = BaselinePreflight( candidate_id=candidate["id"], @@ -900,12 +935,7 @@ def select( source_identity=identity(candidate["source"]), baseline=baseline, ) - missing = missing_baseline_points( - validation.canonical_matrix( - os.environ["GITHUB_REPOSITORY"], owned.base, owned.family - ), - baseline, - ) + missing = missing_baseline_points(matrix, baseline) except (VerificationError, ReadError) as error: baseline_deferred.append(candidate["id"]) baseline_reasons[candidate["id"]] = str(error) @@ -1025,6 +1055,7 @@ def select( def main() -> int: + """Run the Klaud subcommand named on the command line and return its exit status.""" parser = argparse.ArgumentParser(prog="python -m infx.klaud", description=__doc__) parser.add_argument( "--root", @@ -1169,7 +1200,6 @@ def main() -> int: return 0 session = current_session() pulls = session.pulls() - # A resumed session continues with the baseline its PR already froze. record = baseline_for(session, pulls[0]) if pulls else None if record is None: context = json.loads( @@ -1195,7 +1225,10 @@ def main() -> int: from .validation import canonical_matrix, check_matrix session = current_session() - pull = session.pulls()[0] + pulls = session.pulls() + if not pulls: + raise VerificationError("Candidate PR not found") + pull = pulls[0] head = pull["head"]["sha"] matrix = json.loads(args.matrix_file.read_text()) canonical = canonical_matrix(session.repository, head, session.candidate.family) @@ -1315,7 +1348,6 @@ def main() -> int: if isinstance(error, VerificationError): reason = str(error) elif isinstance(error, ValidationError) and args.command == "report": - # Name the failing fields of the agent's own record without echoing its values. reason = "Invalid record: " + "; ".join( ".".join(str(part) for part in item["loc"]) + ": " + item["msg"] if item["loc"] diff --git a/inferencex-e2e/infx/klaud/api.py b/inferencex-e2e/infx/klaud/api.py index 707f6d18bf..b6023b9836 100644 --- a/inferencex-e2e/infx/klaud/api.py +++ b/inferencex-e2e/infx/klaud/api.py @@ -53,14 +53,17 @@ def redirect_request( headers: Any, # noqa: ARG002 newurl: str, # noqa: ARG002 ) -> None: - return None + """Refuse every redirect by returning None.""" + return def reject_nonfinite(value: str) -> Never: # noqa: ARG001 + """Raise ReadError for a NaN or Infinity constant ``value`` in JSON.""" raise ReadError("invalid-json-number") def finite_float(value: str) -> float: + """Return JSON number text ``value`` as a float, raising ReadError unless finite.""" result = float(value) if not math.isfinite(result): raise ReadError("invalid-json-number") @@ -78,6 +81,11 @@ def fetch( opener: Callable[..., Any] | None = None, sleeper: Callable[[float], None] = time.sleep, ) -> Feed: + """GET one ``resource`` from ENDPOINTS and return it as a Feed. + + ``model``, ``date``, ``benchmark`` and ``token`` fill the query or auth header where the + resource takes them. Retries transient failures, then raises ReadError with a fixed code. + """ query = {} if resource == "benchmark-siblings": if benchmark is None: @@ -177,6 +185,7 @@ def fetch( def feed_issues(feed: Feed | None, now: datetime, policy: Policy) -> list[str]: + """Return issue codes for a missing, failed or stale ``feed`` at time ``now``.""" if feed is None: return ["feed-unavailable"] issues = ["feed-refresh-failed"] if feed.error else [] @@ -187,6 +196,7 @@ def feed_issues(feed: Feed | None, now: datetime, policy: Policy) -> list[str]: def catalog(images: Feed | None, now: datetime, policy: Policy) -> tuple[list[dict], list[str]]: + """Return one catalog item per public image row plus the sorted feed issue codes.""" issues = [f"images:{issue}" for issue in feed_issues(images, now, policy)] if images is None or not isinstance(images.payload, list): return [], [*issues, "images:invalid-or-missing-payload"] @@ -221,6 +231,7 @@ def catalog(images: Feed | None, now: datetime, policy: Policy) -> tuple[list[di def fetch_catalog(policy: Policy) -> tuple[list[dict], list[str]]: + """Fetch the public image feed and return its catalog and issues, or raise ReadError.""" try: images = fetch("images") except ReadError as error: @@ -229,6 +240,7 @@ def fetch_catalog(policy: Policy) -> tuple[list[dict], list[str]]: def fresh(value: Any, now: datetime, policy: Policy) -> bool: + """Return True if timestamp ``value`` is within the response age limit at ``now``.""" try: return ( -policy.clock_skew_seconds @@ -243,7 +255,8 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int """Return the idle nodes of each cluster that waiting Klaud jobs have not already requested. Clusters a waiting human job could run on, and clusters without headroom, are omitted. - Malformed or duplicate scheduler status yields no clusters. + A queued job without a node count needs one node, and jobs already holding a runner are + counted in allocatedNodes. Malformed or duplicate scheduler status yields no clusters. """ try: raw = feed.payload @@ -262,7 +275,6 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int human: set[str] = set() klaud: Counter[str] = Counter() for job in snapshot["queue"]: - # A job without a nodes label runs on one runner. nodes = 1 if job["nodeCount"] is None else job["nodeCount"] labels, runner = job["clusterIds"], job["runnerName"] if ( @@ -273,7 +285,6 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int or not isinstance(runner, str | None) ): return {} - # A job holding a runner is already counted in allocatedNodes. if runner is not None: continue if KLAUD_JOB.search(job["name"]): @@ -300,6 +311,7 @@ def cluster_headroom(feed: Feed, policy: Policy, now: datetime) -> dict[str, int def fetch_capacity(policy: Policy) -> dict[str, int]: + """Return per-cluster idle headroom from the private scheduler status feed.""" return cluster_headroom( fetch("scheduler", token=os.environ.get("KLAUD_DASHBOARD_API_KEY")), policy, diff --git a/inferencex-e2e/infx/klaud/claims.py b/inferencex-e2e/infx/klaud/claims.py index e1be6002eb..ca1b8dd4b9 100644 --- a/inferencex-e2e/infx/klaud/claims.py +++ b/inferencex-e2e/infx/klaud/claims.py @@ -18,6 +18,7 @@ def ref(repository: str, name: str) -> dict | None: + """Return the ref record for branch ``name`` in ``repository``, or None if absent.""" return next( ( row @@ -29,6 +30,10 @@ def ref(repository: str, name: str) -> dict | None: def record(repository: str, value: dict) -> dict: + """Return the JSON owner stored in the commit message of ref ``value``. + + Raises VerificationError when the message is not readable JSON. + """ commit = github.read(repository, "git/commits/" + value["object"]["sha"]) try: return json.loads(commit["message"]) @@ -39,6 +44,11 @@ def record(repository: str, value: dict) -> dict: def create( repository: str, name: str, payload: dict, base: str, previous: dict | None = None ) -> bool: + """Point branch ``name`` at a new commit whose message is ``payload`` as JSON. + + The commit sits on ``previous`` (fast-forward, no force) or on ``base`` for a new ref. + Return True if this caller won the ref, False if a concurrent writer did. + """ parent = previous["object"]["sha"] if previous else base commit = github.read(repository, "git/commits/" + parent) new = github.write( @@ -75,10 +85,12 @@ def create( def family_name(candidate: OwnedCandidate) -> str: + """Return the claim branch name derived from the candidate's family.""" return "klaud/claim-" + identity(candidate.family)[:16] def claim_family(repository: str, candidate: OwnedCandidate, parent: int) -> bool: + """Claim the candidate's family for run ``parent`` and return True if this call won.""" name = family_name(candidate) if ref(repository, name): return False @@ -87,6 +99,7 @@ def claim_family(repository: str, candidate: OwnedCandidate, parent: int) -> boo def release_family(repository: str, candidate: OwnedCandidate, parent: int) -> None: + """Delete the candidate's family claim if run ``parent`` owns exactly that candidate.""" name = family_name(candidate) value = ref(repository, name) if not value: @@ -98,6 +111,7 @@ def release_family(repository: str, candidate: OwnedCandidate, parent: int) -> N def family_owners(repository: str) -> list[Ownership]: + """Return every family claim's ownership, raising VerificationError on a malformed one.""" result = [] for value in github.items(repository, "git/matching-refs/heads/klaud/claim-"): owner = Ownership.model_validate(record(repository, value)) @@ -110,6 +124,10 @@ def family_owners(repository: str) -> list[Ownership]: def recovery_lease(repository: str, parent: int, candidate: OwnedCandidate) -> bool: + """Take the recovery lease for ``candidate`` of run ``parent`` and return True if held. + + A prior holder hands over only once its planner run or its reconcile job has completed. + """ name = f"klaud/recovery-{parent}-{candidate.id}" owner = { "run-id": int(os.environ["GITHUB_RUN_ID"]), diff --git a/inferencex-e2e/infx/klaud/github.py b/inferencex-e2e/infx/klaud/github.py index bdb5ef5b2e..aa7ea707ff 100644 --- a/inferencex-e2e/infx/klaud/github.py +++ b/inferencex-e2e/infx/klaud/github.py @@ -17,33 +17,49 @@ class VerificationError(ValueError): def read(repository: str, path: str, *, paginate: bool = False) -> list | dict: + """Return the decoded JSON from a GET of ``path`` in ``repository``, optionally paginated.""" return github.api(repository, path, paginate=paginate) def items(repository: str, path: str, key: str | None = None) -> list[dict]: + """Return every listed item at ``path``, unwrapped from ``key``, or raise VerificationError.""" try: return github.paginate(repository, path, item_key=key or "") except github.ListingError as error: raise VerificationError(str(error)) from error +writes = 0 + + def write(repository: str, path: str, method: str, payload: dict | None = None) -> dict: + """Send ``payload`` to ``path`` with ``method`` and return the JSON reply. + + Each call bumps the module ``writes`` counter so listings cached before it are read again. + """ + global writes # noqa: PLW0603 + writes += 1 return github.api(repository, path, method=method, data=payload) def artifacts(repository: str, run_id: int) -> list[dict]: + """Return the artifact records of workflow run ``run_id``.""" return items(repository, f"actions/runs/{run_id}/artifacts?per_page=100", "artifacts") def file_at(repository: str, head: str, path: str) -> bytes: - """Read data at an immutable revision; callers never execute downloaded code.""" + """Return the bytes of ``path`` at commit ``head``, raising VerificationError if unsafe. + + ``head`` must be a full SHA and the file at most 4 MiB. Callers never execute the result. + """ import re if not re.fullmatch(r"[0-9a-f]{40}", head): raise VerificationError("Expected an immutable source revision") result = read(repository, f"contents/{quote(path, safe='/')}?ref={head}") if ( - result.get("type") != "file" + not isinstance(result, dict) + or result.get("type") != "file" or result.get("encoding") != "base64" or result["size"] > 4 * 1024 * 1024 ): @@ -52,7 +68,10 @@ def file_at(repository: str, head: str, path: str) -> bytes: def download_json(repository: str, artifact: dict, destination: Path) -> None: - """Read bounded JSON only; never extract archive paths or execute artifact code.""" + """Write the JSON members of ``artifact`` under ``destination``, raising VerificationError. + + Archives over 256 MiB, unsafe paths and duplicates are rejected. Artifact code never runs. + """ limit = 256 * 1024 * 1024 if artifact["expired"] or not 0 < artifact["size_in_bytes"] <= limit: raise VerificationError("Artifact unavailable or too large") diff --git a/inferencex-e2e/infx/klaud/lifecycle.py b/inferencex-e2e/infx/klaud/lifecycle.py index b51f2cde12..0b740900e1 100644 --- a/inferencex-e2e/infx/klaud/lifecycle.py +++ b/inferencex-e2e/infx/klaud/lifecycle.py @@ -1,4 +1,8 @@ -"""Finish owned sessions; resume or reconcile interrupted ones in the next autosweep.""" +"""Finish owned sessions; resume or reconcile interrupted ones in the next autosweep. + +A candidate gets at most ``MAX_SESSIONS`` sessions, its planner run's own plus three resume +runs, and planner waves start every ``WAVE_HOURS`` as scheduled in klaud-plan.yml. +""" from __future__ import annotations @@ -30,22 +34,14 @@ "evals-only", "agentx-fast", } -# Leaves the scheduler time to rerun what it preempted before recovery reruns it. RERUN_GRACE_SECONDS = 30 * 60 -# Recovery stops waiting for a preempted final sweep that stayed cancelled this long. RERUN_GIVE_UP_SECONDS = 24 * 60 * 60 MAX_SWEEP_ATTEMPTS = 10 -# Sessions per candidate: its planner run's session plus three resume runs. MAX_SESSIONS = 4 -# Planner waves start this often (klaud-plan.yml's schedule). WAVE_HOURS = 6 -# Recovery stops resuming, or keeping unreconciled sessions, this long after the last ended. RESUME_GIVE_UP_SECONDS = 24 * 60 * 60 -# The klaud-candidate.yml step that lets a resume run start only a session recovery granted. GATE_STEP = "Check whether this session should run" -# The klaud-candidate.yml step that runs the session's agent. AGENT_STEP = "Let Klaud Cold own the candidate PR" -# GitHub concludes a job stopped at its timeout-minutes as cancelled, with this annotation. TIMEOUT_ANNOTATION = "exceeded the maximum execution time" @@ -72,11 +68,13 @@ class ResumeSession(PendingCleanup): """Recovery may resume this unfinished session; keep its PR, branch, claim and lease.""" def __init__(self, pull: dict) -> None: + """Carry the resumable ``pull`` with a fixed public message.""" super().__init__("Unfinished session approved to resume in a new run") self.pull = pull def terminal(run: dict) -> bool: + """Return True if workflow ``run`` has completed with a conclusion.""" return run["status"] == "completed" and bool(run.get("conclusion")) @@ -89,14 +87,27 @@ def __init__( *, recovering: bool = False, ) -> None: + """Track ``candidate`` of planner run ``parent`` in ``repository``, as recovery if set.""" self.repository, self.parent, self.candidate = repository, parent, candidate self.recovering = recovering self.branch = f"klaud/auto-{candidate.id}" self.marker = f"", 1)[0]) def pending(self, pull: dict) -> CandidateOutcome | None: + """Return the outcome of this session's pending cleanup request on ``pull``, or None. + + Raises VerificationError if the PR head moved since or the outcome claims validated. + """ marker = self.marker.replace("klaud-outcome:", "klaud-cleanup:") - comments = github.items(self.repository, f"issues/{pull['number']}/comments?per_page=100") + comments = self.listing(f"issues/{pull['number']}/comments?per_page=100") matches = [ c for c in comments if c["user"]["login"] == BOT and c["body"].startswith(marker) ] @@ -284,7 +312,8 @@ def pending(self, pull: dict) -> CandidateOutcome | None: return outcome def branch_exists(self) -> bool: - refs = github.items(self.repository, "git/matching-refs/heads/" + self.branch) + """Return True if the candidate branch exists.""" + refs = self.listing("git/matching-refs/heads/" + self.branch) return any(ref["ref"] == "refs/heads/" + self.branch for ref in refs) def delete_branch(self, pull: dict) -> None: @@ -305,6 +334,11 @@ def verify( require_ready: bool = True, require_branch_deleted: bool = True, ) -> None: + """Raise unless GitHub state matches ``outcome`` for this session, else return None. + + Raises PendingCleanup while owned jobs run and VerificationError on any mismatch. The + flags relax the report, ready-for-review and deleted-branch checks. + """ pulls = self.pulls() pull = pulls[0] if pulls else None if pull and pull["merged_at"]: @@ -319,7 +353,6 @@ def verify( if outcome.outcome == "validated": if not pull or pull["state"] != "open" or (require_ready and pull["draft"]): raise VerificationError("Validated PR must remain ready for review") - # Klaud runs every final sweep fail-fast; any other sweep label is invalid. sweep_labels = {label["name"] for label in pull["labels"]} & SWEEP_LABELS if sweep_labels != {"full-sweep-fail-fast"}: raise VerificationError("Validated PR has incompatible sweep labels") @@ -368,6 +401,11 @@ def verify( raise VerificationError("Missing verified completion report") def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: + """Apply ``outcome`` to this session and return it with the real PR and run IDs. + + Validated PRs are published and marked ready. Others have runs cancelled, the PR closed + and the branch deleted. Then the report is posted and the family claim released. + """ pulls = self.pulls() pull = pulls[0] if pulls else None runs = self.runs() @@ -426,7 +464,11 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: self.check_parent() if pull: self.refresh(pull) - github.write(self.repository, f"actions/runs/{run['id']}/cancel", "POST") + try: + github.write(self.repository, f"actions/runs/{run['id']}/cancel", "POST") + except (OSError, subprocess.SubprocessError): + if not terminal(github.read(self.repository, f"actions/runs/{run['id']}")): + raise if any(not terminal(run) for run in self.runs()): raise PendingCleanup( "Cancellation requested; wait for terminal jobs then finish again" @@ -488,6 +530,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: or "—" ) repairs = outcome.repairs_used if outcome.repairs_used is not None else "unknown" + repairs_zh = outcome.repairs_used if outcome.repairs_used is not None else "未知" reason_en = f" · Reason: {outcome.reason_code}" if outcome.reason_code else "" reason_zh = f" · 原因:{outcome.reason_code}" if outcome.reason_code else "" why = ( @@ -510,7 +553,7 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: if proof else "PR closed; branch deleted for retry." ), - f"**{outcome.outcome}**{reason_zh} · 修复次数:{repairs} · 运行:{links} \n{why_zh}所有自有运行均已结束。" + f"**{outcome.outcome}**{reason_zh} · 修复次数:{repairs_zh} · 运行:{links} \n{why_zh}所有自有运行均已结束。" + ( "完整 sweep 已验证;已就绪,等待审查。" if proof @@ -529,6 +572,8 @@ def finish(self, outcome: CandidateOutcome) -> CandidateOutcome: def subprocess_ready(repository: str, number: int, *, undo: bool = True) -> None: + """Mark PR ``number`` ready for review, or back to draft if ``undo``, counting a write.""" + github.writes += 1 subprocess.run( ["gh", "pr", "ready", str(number), "--repo", repository] + (["--undo"] if undo else []), check=True, @@ -538,17 +583,21 @@ def subprocess_ready(repository: str, number: int, *, undo: bool = True) -> None def current_session() -> Session: + """Return the Session for the candidate in ``$KLAUD_EVIDENCE/candidate.json``. + + A resume run passes ``KLAUD_PARENT_RUN``, the planner run that owns the candidate. + """ context = json.loads((Path(os.environ["KLAUD_EVIDENCE"]) / "candidate.json").read_text()) candidate = OwnedCandidate.model_validate( {key: context[key] for key in ("id", "family", "base")} ) repository = os.environ["GITHUB_REPOSITORY"] - # A resume run continues the session of the planner run that owns the candidate. parent = os.environ.get("KLAUD_PARENT_RUN") or os.environ["GITHUB_RUN_ID"] return Session(repository, github.read(repository, f"actions/runs/{int(parent)}"), candidate) def hit_timeout(repository: str, job: dict) -> bool: + """Return True if ``job`` ran and GitHub annotated it as stopped at its timeout-minutes.""" return bool(job.get("runner_name")) and any( TIMEOUT_ANNOTATION in (annotation.get("message") or "") for annotation in github.items( @@ -558,20 +607,28 @@ def hit_timeout(repository: str, job: dict) -> bool: def preempted(repository: str, run: dict) -> bool: - """Cancelled by someone else: no job of its current attempt failed or hit its timeout.""" - return run["conclusion"] == "cancelled" and not any( - job["conclusion"] in ("failure", "timed_out") - or (job["conclusion"] == "cancelled" and hit_timeout(repository, job)) - for job in github.items( - repository, - f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100", - "jobs", - ) + """Cancelled by someone else: no job of its current attempt failed or hit its timeout. + + Failures are ruled out first because a fail-fast sweep cancels every other job. + """ + if run["conclusion"] != "cancelled": + return False + jobs = github.items( + repository, + f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100", + "jobs", + ) + return not any(job["conclusion"] in ("failure", "timed_out") for job in jobs) and not any( + job["conclusion"] == "cancelled" and hit_timeout(repository, job) for job in jobs ) def await_rerun(session: Session, pull: dict, run: dict, now: datetime) -> None: - """Keep a preempted final sweep's candidate alive until the sweep runs again.""" + """Raise AwaitingRerun to keep ``session`` while preempted final sweep ``run`` can rerun. + + Waits RERUN_GRACE_SECONDS for the scheduler's own rerun, then requests one. Returns if + ``run`` was not preempted or is out of attempts. + """ try: if not preempted(session.repository, run): return @@ -624,9 +681,7 @@ def candidate_jobs(session: Session) -> list[dict]: ( job for run in runs - for job in github.items( - session.repository, f"actions/runs/{run}/jobs?filter=all&per_page=100", "jobs" - ) + for job in session.listing(f"actions/runs/{run}/jobs?filter=all&per_page=100", "jobs") if job["name"].startswith(prefix) and job["started_at"] and utc(job["started_at"]) >= utc(job["created_at"]) @@ -651,30 +706,34 @@ def sessions_started(session: Session) -> int: def resumes(session: Session, pull: dict | None, now: datetime) -> bool: - """Whether a new resume run would continue this unfinished session.""" + """Return True if a new resume run would continue this unfinished session at ``now``. + + The last started job must have ended, run the gate step, and failed, timed out or been + cancelled within RESUME_GIVE_UP_SECONDS. Only klaud-abort stops a candidate. + """ if session.last_session_end() is None or not resumable(session, pull): return False started = [job for job in candidate_jobs(session) if started_session(job)] return ( 0 < len(started) < MAX_SESSIONS - # Runs that failed before their agent started never extend this. and bool(started[-1].get("completed_at")) and now - utc(started[-1]["completed_at"]) < timedelta(seconds=RESUME_GIVE_UP_SECONDS) - # A planner run from before resumption has no gate. and any(step["name"] == GATE_STEP for step in started[-1]["steps"]) - # A session that runs out fails its job, and a cancelled session was preempted like any - # other Klaud work; only klaud-abort stops a candidate. and started[-1]["conclusion"] in ("failure", "timed_out", "cancelled") ) def resume(session: Session, pull: dict | None, now: datetime) -> None: - """Approve resuming an unfinished session; recovery grants approved ones after its pass.""" + """Raise ResumeSession with ``pull`` when recovery may resume this session, else return. + + Recovery grants the approved sessions after its pass. + """ if session.recovering and resumes(session, pull, now): raise ResumeSession(pull) def resume_marker(parent: int, candidate: str, number: int) -> str: + """Return the comment marker that records the grant of session ``number``.""" return f"" @@ -690,7 +749,10 @@ def granted(session: Session, pull: dict, number: int) -> bool: def grant(session: Session, pull: dict) -> int: - """Grant an unfinished session its next session, then dispatch the resume run that runs it.""" + """Grant an unfinished session its next session, dispatch its resume run, return its number. + + If the dispatch fails the grant comment is deleted, so no uncounted session can start. + """ repository, parent = session.repository, session.parent["id"] session.refresh(pull) number = sessions_started(session) + 1 @@ -727,7 +789,6 @@ def grant(session: Session, pull: dict) -> int: }, ) except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): - # A grant without its resume run would let a later run start an uncounted session. if posted is not None: with contextlib.suppress( OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError @@ -754,6 +815,7 @@ def retained(session: Session, now: datetime) -> bool: def summarize(repository: str, candidate: OwnedCandidate, parent: int, status: str) -> None: + """Append ``status`` for ``candidate`` of run ``parent`` to the job summary, if one exists.""" if summary := os.environ.get("GITHUB_STEP_SUMMARY"): with open(summary, "a") as output: output.write( @@ -763,6 +825,11 @@ def summarize(repository: str, candidate: OwnedCandidate, parent: int, status: s def reconcile(session: Session, now: datetime) -> bool: + """Settle interrupted ``session`` at ``now`` and return True if anything changed. + + Raises PendingCleanup subclasses while work runs, a recent preempted sweep may rerun, + or the session may resume. Returns False for a merged PR or no owned work. + """ pulls = session.pulls() pull = pulls[0] if pulls else None if pull and pull["merged_at"]: @@ -842,6 +909,11 @@ def reconcile(session: Session, now: datetime) -> bool: def recover(max_resumes: int) -> None: + """Reconcile every owned candidate, then resume up to ``max_resumes`` approved sessions. + + Writes still-blocked families to ``$RUNNER_TEMP/klaud-recovery.json`` and the resume count + to ``$GITHUB_OUTPUT``. Unreadable state counts as live work, and oldest planners go first. + """ repository = os.environ["GITHUB_REPOSITORY"] now = datetime.now(UTC) inventory = github.items( @@ -851,22 +923,44 @@ def recover(max_resumes: int) -> None: for artifact in inventory: if artifact["expired"]: continue - parent = github.read(repository, f"actions/runs/{artifact['workflow_run']['id']}") - if ( - parent["head_branch"] != "main" - or parent["path"] != ".github/workflows/klaud-plan.yml" - or parent["event"] not in ("schedule", "workflow_dispatch") - ): + try: + parent = github.read(repository, f"actions/runs/{artifact['workflow_run']['id']}") + if ( + parent["head_branch"] != "main" + or parent["path"] != ".github/workflows/klaud-plan.yml" + or parent["event"] not in ("schedule", "workflow_dispatch") + ): + continue + with tempfile.TemporaryDirectory(prefix="klaud-ownership-") as temp: + github.download_json(repository, artifact, Path(temp)) + owner = Ownership.model_validate_json((Path(temp) / "ownership.json").read_text()) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + print(f"::warning::Klaud: ownership artifact {artifact['id']} is unreadable; skipped.") continue - with tempfile.TemporaryDirectory(prefix="klaud-ownership-") as temp: - github.download_json(repository, artifact, Path(temp)) - owner = Ownership.model_validate_json((Path(temp) / "ownership.json").read_text()) if owner.run_id != parent["id"]: raise VerificationError("Ownership parent mismatch") owners.append((owner, artifact)) - parents = { - owner.run_id: github.read(repository, f"actions/runs/{owner.run_id}") for owner, _ in owners - } + results = {} + blocked = set() + live = set() + parents = {} + for owner, _ in owners: + if owner.run_id in parents: + continue + try: + parents[owner.run_id] = github.read(repository, f"actions/runs/{owner.run_id}") + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + for candidate in owner.candidates: + results[(owner.run_id, candidate.id)] = False + blocked.add(candidate.family) + live.add(candidate.id) + summarize( + repository, + candidate, + owner.run_id, + "needs inspection / 需要检查 (state unavailable or invalid)", + ) + owners = [pair for pair in owners if pair[0].run_id in parents] if any( parent["head_branch"] != "main" or parent["path"] != ".github/workflows/klaud-plan.yml" @@ -874,9 +968,6 @@ def recover(max_resumes: int) -> None: for parent in parents.values() ): raise VerificationError("Untrusted ownership parent") - results = {} - blocked = set() - live = set() approved: list[tuple[Session, dict]] = [] for owner, _ in sorted(owners, key=lambda pair: pair[0].run_id, reverse=True): parent = parents[owner.run_id] @@ -931,7 +1022,6 @@ def recover(max_resumes: int) -> None: if status: summarize(repository, candidate, parent["id"], status) resumed = 0 - # Oldest planner runs first; resumed sessions share this wave's candidate cap. approved.sort(key=lambda pair: (pair[0].parent["id"], pair[0].candidate.id)) for index, (session, pull) in enumerate(approved): run_id = session.parent["id"] @@ -978,6 +1068,11 @@ def recover(max_resumes: int) -> None: def owned_activity(repository: str) -> tuple[set[str], dict[str, list[int]], set[int]]: + """Return active Klaud work in ``repository``. + + That is live candidate IDs, active targeted run IDs per candidate, and planner runs whose + reconcile job is unfinished. + """ live: set[str] = set() targeted: dict[str, list[int]] = {} planners: set[int] = set() @@ -1008,14 +1103,31 @@ def owned_activity(repository: str) -> tuple[set[str], dict[str, list[int]], set def bot_committed(commits: list[dict]) -> bool: + """Return True if ``commits`` is non-empty and the Actions bot committed all of them.""" return bool(commits) and all( (commit.get("committer") or {}).get("login") == COMMITTER for commit in commits ) def sweep_abandoned(repository: str, live: set[str]) -> set[str]: + """Clean up Klaud work no live session owns and return the widened ``live`` candidate IDs. + + Closes abandoned draft PRs and cancels orphan targeted runs. Deletes stale claims, leases + and branches. A claim whose planner run cannot be read counts as live. + """ sessions, targeted, planners = owned_activity(repository) live = live | sessions + claimed = [] + for owner in claims.family_owners(repository): + candidate = owner.candidates[0] + if candidate.id in live: + continue + try: + parent = github.read(repository, f"actions/runs/{owner.run_id}") + except (OSError, ValueError, subprocess.SubprocessError): + live.add(candidate.id) + continue + claimed.append((owner, candidate, parent)) pulls = github.items(repository, "pulls?state=open&per_page=100") abandoned = [] for pull in pulls: @@ -1037,11 +1149,8 @@ def sweep_abandoned(repository: str, live: set[str]) -> set[str]: if candidate not in live: for run in runs: github.write(repository, f"actions/runs/{run}/cancel", "POST") - for owner in claims.family_owners(repository): - candidate = owner.candidates[0] - if candidate.id not in live and terminal( - github.read(repository, f"actions/runs/{owner.run_id}") - ): + for owner, candidate, parent in claimed: + if candidate.id not in live and terminal(parent): claims.release_family(repository, candidate, owner.run_id) if planners <= {int(os.environ.get("GITHUB_RUN_ID", "0"))}: finished: dict[int, bool] = {} diff --git a/inferencex-e2e/infx/klaud/models.py b/inferencex-e2e/infx/klaud/models.py index 05b0af3a74..c952defae8 100644 --- a/inferencex-e2e/infx/klaud/models.py +++ b/inferencex-e2e/infx/klaud/models.py @@ -1,4 +1,7 @@ -"""Versioned local contracts, deliberately separate from external API schemas.""" +"""Versioned local contracts, deliberately separate from external API schemas. + +``ABORT_LABEL`` is the maintainer label that stops a candidate now and blocks retries while set. +""" from __future__ import annotations @@ -9,11 +12,11 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -# A maintainer label that stops a candidate now and keeps the planner from retrying it while set. ABORT_LABEL = "klaud-abort" def utc(value: str | datetime) -> datetime: + """Return an ISO timestamp or aware datetime as a UTC datetime, raising ValueError if naive.""" result = ( datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value ) @@ -23,17 +26,22 @@ def utc(value: str | datetime) -> datetime: def stamp(value: datetime) -> str: + """Return an aware datetime as an ISO UTC string ending in ``Z``.""" return utc(value).isoformat().replace("+00:00", "Z") def identity(value: Any) -> str: + """Return the SHA-256 hex digest of ``value`` as canonical JSON.""" return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() ).hexdigest() def normalized_image(value: str) -> str: - """Normalize registry paths without changing tags, digests or repositories.""" + """Return image ``value`` with a ``registry#path`` separator turned into ``/``. + + Tags, digests and repositories are left unchanged. + """ registry, separator, remainder = value.partition("#") if separator and remainder and ("." in registry or ":" in registry): return registry + "/" + remainder @@ -66,6 +74,7 @@ class CandidateReview(Contract): @model_validator(mode="after") def consistent_decision(self) -> CandidateReview: + """Return the review, raising ValueError if the decision contradicts its fields.""" if self.decision == "proceed" and ( not self.family or self.pull_requests or not self.baseline_model ): @@ -125,6 +134,7 @@ class CandidateOutcome(Contract): @model_validator(mode="after") def consistent_outcome(self) -> CandidateOutcome: + """Return the outcome, raising ValueError if phase, PR, runs or reason code disagree.""" if self.outcome != "unexpected-error" and self.phase == "unknown": raise ValueError("Completed outcomes require a known phase") if self.outcome in ("validated", "aborted") and self.pull_request is None: @@ -154,6 +164,7 @@ class Feed(Contract): @field_validator("retrieved_at") @classmethod def valid_timestamp(cls, value: str) -> str: + """Return ``value`` unchanged, raising ValueError unless it is an aware ISO timestamp.""" utc(value) return value @@ -175,6 +186,7 @@ class PublicRow(BaseModel): @field_validator("date") @classmethod def valid_date(cls, value: str) -> str: + """Return ``value`` unchanged, raising ValueError unless it is a YYYY-MM-DD date.""" from datetime import date if date.fromisoformat(value).isoformat() != value: diff --git a/inferencex-e2e/infx/klaud/reporting.py b/inferencex-e2e/infx/klaud/reporting.py index dc6bd20296..b7c6d04ec4 100644 --- a/inferencex-e2e/infx/klaud/reporting.py +++ b/inferencex-e2e/infx/klaud/reporting.py @@ -44,6 +44,10 @@ def public_prose(value: str) -> str: + """Return ``value`` unchanged, raising ValueError on mentions, HTML, newlines or pipes. + + Backquoted image digests are exempt from the check. + """ prose = re.sub(r"`[A-Za-z0-9_./:+-]+@sha256:[0-9a-f]{64}`", "", value) if any(char in prose for char in ("@", "<", ">", "\n", "|")): raise ValueError("Use one sentence of public prose without mentions, HTML or tables") @@ -107,6 +111,7 @@ class Point(Contract): @field_validator("dataset", mode="before") @classmethod def canonical_dataset(cls, value: Any) -> Any: + """Return a string ``value`` as its dashboard dataset slug, anything else unchanged.""" return dataset_slug(value) if isinstance(value, str) else value @@ -159,6 +164,7 @@ class Baseline(Contract): @field_validator("sources") @classmethod def public_sources(cls, urls: list[str]) -> list[str]: + """Return ``urls``, raising ValueError unless each is a public dashboard API URL.""" from urllib.parse import urlsplit for url in urls: @@ -174,6 +180,7 @@ def public_sources(cls, urls: list[str]) -> list[str]: @model_validator(mode="after") def distinct(self) -> Self: + """Return the baseline, raising ValueError on duplicates or invalid retirements.""" unique_points(self.points) unique_evals(self.evals) retired = [key for retirement in self.retirements for key in retirement.points] @@ -189,6 +196,7 @@ def distinct(self) -> Self: @model_serializer(mode="wrap") def compatible(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + """Return the serialized baseline, omitting ``retirements`` when it is empty.""" data = handler(self) if not self.retirements: data.pop("retirements", None) @@ -227,6 +235,7 @@ class Attempt(Contract): @model_validator(mode="after") def consistent(self) -> Self: + """Return the attempt, raising ValueError if coverage, numbering or provenance disagree.""" unique_points(self.points) unique_evals(self.evals) if ( @@ -251,11 +260,13 @@ def consistent(self) -> Self: def unique_points(points: list[Point]) -> None: + """Raise ValueError if two ``points`` share a key.""" if len({point.key for point in points}) != len(points): raise ValueError("Duplicate comparison point") def unique_evals(rows: list[Evaluation]) -> None: + """Raise ValueError if two eval ``rows`` share key, suite and metric.""" if len({(row.key, row.suite, row.metric) for row in rows}) != len(rows): raise ValueError("Duplicate comparison eval") @@ -268,6 +279,7 @@ def day(value: str) -> date: def point_key(entry: dict) -> str: + """Return the identity of matrix ``entry`` without image, name, fingerprint or queue fields.""" return identity( { key: value @@ -285,6 +297,7 @@ def point_key(entry: dict) -> str: def point_label(entry: dict) -> str: + """Return a label for matrix ``entry`` with its workload, concurrency, shape and key prefix.""" if entry.get("prefill") is not None: shape = "/".join( f"{role[0].upper()}TP{entry[role]['tp']}x{entry[role]['num-worker']}" @@ -303,7 +316,10 @@ def label_workload(label: str) -> tuple[int, int] | None: def values(row: dict) -> Values: - """Collector/API metrics are seconds; only this presentation layer converts ms.""" + """Return the throughput, latency in ms and error count of result ``row``. + + Collector and API metrics are seconds, and only this presentation layer converts to ms. + """ metrics = row.get("metrics", row) if "request_metrics" in metrics: request = metrics["request_metrics"] @@ -322,26 +338,32 @@ def values(row: dict) -> Values: output_tps_gpu=metrics.get("output_tput_per_gpu"), ttft_ms=metrics["mean_ttft"] * 1000 if metrics.get("mean_ttft") is not None else None, tpot_ms=metrics["mean_tpot"] * 1000 if metrics.get("mean_tpot") is not None else None, - request_errors=metrics.get("errors"), + request_errors=row.get("benchmark_outcome", {}).get("failed"), ) def number(value: float | None) -> str: + """Return ``value`` with two decimals and separators, trailing zeros trimmed, or N/A.""" return "N/A" if value is None else f"{value:,.2f}".rstrip("0").rstrip(".") def delta(old: float | None, new: float | None, comparable: bool = True) -> str: + """Return the signed percent change from ``old`` to ``new``, or N/A if not comparable.""" if not comparable or old is None or new is None or old == 0: return "N/A" return f"{(new / old - 1) * 100:+.1f}%".replace(".0%", "%") def translated(english: str, chinese: str) -> str: + """Return ``english`` followed by ``chinese`` in a collapsed details block.""" return english + "\n\n
\n中文\n\n" + chinese + "\n\n
" def short_label(label: str) -> str: + """Return ``label`` with its first ISL/OSL pair written in k where it divides by 1024.""" + def shape(match: re.Match) -> str: + """Return the matched ISL/OSL pair with each multiple of 1024 written in k.""" return "/".join( f"{int(n) // 1024}k" if int(n) > 0 and int(n) % 1024 == 0 else n for n in match.groups() ) @@ -350,7 +372,10 @@ def shape(match: re.Match) -> str: def point_layout(points: list[Point]) -> tuple[str, list[str], str]: - """Factor out only shared settings; retain full labels for ambiguous concurrencies.""" + """Return the first column title, row labels and shared caption for ``points``. + + Only settings shared by every point are factored out, so ambiguous concurrencies keep labels. + """ labels = [short_label(p.label) for p in points] shapes = [ re.fullmatch(r"(.+) c\d+ (.+)", re.sub(r" [0-9a-f]{6}$", "", label)) for label in labels @@ -380,6 +405,7 @@ def point_layout(points: list[Point]) -> tuple[str, list[str], str]: def table(header: list[str], rows: list[list[str]]) -> str: + """Return a Markdown table of ``header`` and ``rows``, or an empty string without rows.""" if not rows: return "" alignment = [ @@ -396,6 +422,7 @@ def table(header: list[str], rows: list[list[str]]) -> str: def note_lines(issues: dict[str, list[str]], total: int) -> str: + """Return one Markdown note per reason in ``issues``, naming its labels or all rows.""" return " \n".join( f"**Note:** {'All rows' if len(labels) == total else ', '.join(labels)}: {reason}." for reason, labels in issues.items() @@ -403,6 +430,7 @@ def note_lines(issues: dict[str, list[str]], total: int) -> str: def measured(new: float | None, old: float | None, comparable: bool) -> str: + """Return ``new`` with its change from ``old``, or N/A if ``new`` is missing.""" return "N/A" if new is None else f"{number(new)} ({delta(old, new, comparable)})" @@ -429,6 +457,10 @@ def mismatch(previous: Point | None, point: Point) -> str | None: def point_table( points: list[Point], baseline: Baseline | None = None, *, context: bool = True ) -> str: + """Return the Markdown comparison of ``points`` against ``baseline`` with its notes. + + ``context`` adds the shared settings caption. + """ old = {point.key: point for point in baseline.points} if baseline else {} heading, labels, settings = point_layout(points) rows, issues = [], {} @@ -479,6 +511,7 @@ def point_table( def eval_table(rows: list[Evaluation], baseline: Baseline | None, *, compare: bool = True) -> str: + """Return the Markdown table of eval ``rows`` with notes, versus ``baseline`` if ``compare``.""" previous = {(row.key, row.suite, row.metric): row for row in baseline.evals} if baseline else {} values, issues = [], {} for row in rows: @@ -539,6 +572,10 @@ def retirement_reasons(baseline: Baseline) -> dict[str, str]: def baseline_table(baseline: Baseline, *, context: bool = True) -> str: + """Return the Markdown table of the frozen ``baseline`` points with notes. + + ``context`` adds the shared settings caption. + """ points = baseline.points retired = retirement_reasons(baseline) heading, labels, settings = point_layout(points) @@ -563,7 +600,7 @@ def baseline_table(baseline: Baseline, *, context: bool = True) -> str: reason = f"{number(point.values.request_errors)} request error" + ( "s" if point.values.request_errors != 1 else "" ) - issues.setdefault(reason, []).append(label) + issues.setdefault(reason, []).append(shown) return "\n\n".join( part for part in ( @@ -585,6 +622,7 @@ def baseline_table(baseline: Baseline, *, context: bool = True) -> str: def render_body(baseline: Baseline) -> str: + """Return the bilingual PR body section that presents ``baseline`` and the abort label.""" _, _, settings = point_layout(baseline.points) sources = ", ".join(f"[API {i + 1}]({url})" for i, url in enumerate(baseline.sources)) or "N/A" meta = " · ".join(part for part in (settings, f"Sources: {sources}") if part) @@ -610,6 +648,7 @@ def render_body(baseline: Baseline) -> str: def render_attempt(record: Attempt, baseline: Baseline | None, repository: str) -> str: + """Return the bilingual comment for attempt ``record`` in ``repository`` versus ``baseline``.""" titles = { "initial": ("Initial attempt", "初次尝试"), "repair": (f"Repair {record.number}/5", f"修复 {record.number}/5"), @@ -656,16 +695,23 @@ def render_attempt(record: Attempt, baseline: Baseline | None, repository: str) chinese = ( f"**{title_zh} · {status_zh}** · {link} · {timestamp} \n" + meta.replace("Mean latency", "平均延迟").replace("Median latency", "中位延迟") - + f" \n**变更:**{record.change.zh}实测数值及异常说明见上表。 \n**下一步:**{record.next.zh}" + + f" \n**变更:**{record.change.zh}" + + ("实测数值及异常说明见上表。" if results else "") + + f" \n**下一步:**{record.next.zh}" ) return translated(english, chinese) def marker(session: Session, name: str) -> str: + """Return the comment marker of report ``name`` in ``session``.""" return f"", 1)[0]) if "record" in value: record = value["record"] @@ -697,6 +747,7 @@ def decode[Record: Contract]( def latest_next(session: Session, pull: dict) -> Prose | None: + """Return the next step of the session's latest attempt report on ``pull``, or None.""" attempts = [ c for c in github.items(session.repository, f"issues/{pull['number']}/comments?per_page=100") @@ -715,6 +766,7 @@ def latest_next(session: Session, pull: dict) -> Prose | None: def baseline_for(session: Session, pull: dict) -> Baseline | None: + """Return the baseline frozen in ``pull``'s body or baseline comment, or None.""" current = session.refresh(pull) embedded = baseline_from_body(current.get("body") or "") if embedded: @@ -724,12 +776,17 @@ def baseline_for(session: Session, pull: dict) -> Baseline | None: def baseline_marker(record: Baseline) -> str: + """Return a hidden HTML comment embedding ``record`` as compressed base64 JSON.""" packed = zlib.compress(record.model_dump_json(by_alias=True).encode(), level=9) encoded = base64.urlsafe_b64encode(packed).decode() return f"" def baseline_from_body(body: str) -> Baseline | None: + """Return the baseline embedded in PR ``body``, or None. + + Raises VerificationError if the record is ambiguous or invalid. + """ prefix = "" @@ -770,7 +832,10 @@ def initialize_body(session: Session, pull: dict, record: Baseline) -> None: def check_reported_datasets(record: Attempt) -> None: - """A hand-written attempt must name each passed AgentX point's corpus, or deltas stay N/A.""" + """Raise VerificationError if a passed AgentX point of ``record`` lacks its dataset. + + Without the corpus name its deltas would stay N/A. + """ if any( point.scenario == "agentic-coding" and point.result == "passed" and not point.dataset for point in record.points @@ -781,7 +846,15 @@ def check_reported_datasets(record: Attempt) -> None: def publish(session: Session, record: Baseline | Attempt) -> None: - pull = session.pulls()[0] + """Publish ``record`` to the candidate PR, raising VerificationError if it mismatches. + + A baseline is frozen in the PR body. An attempt is upserted as a comment, split into + numbered chunk comments past 20 points or evals. + """ + pulls = session.pulls() + if not pulls: + raise VerificationError("Candidate PR not found") + pull = pulls[0] session.refresh(pull) if isinstance(record, Baseline): if record.family != session.candidate.family: @@ -806,7 +879,8 @@ def publish(session: Session, record: Baseline | Attempt) -> None: ) name = f"run-{record.run_id}-{record.run_attempt}" previous = stored(session, pull, name) - text = render_attempt(record, baseline_for(session, pull), session.repository) + baseline = baseline_for(session, pull) + text = render_attempt(record, baseline, session.repository) packed = record.model_dump(by_alias=True) chunks = [] if len(record.points) > 20 or len(record.evals) > 20: @@ -816,26 +890,23 @@ def publish(session: Session, record: Baseline | Attempt) -> None: "evals": packed["evals"][offset : offset + 20], } part_name = f"{name}-part-{offset // 20 + 1}-{identity(part)[:16]}" - part_text = point_table( - record.points[offset : offset + 20], baseline_for(session, pull) - ) + part_text = point_table(record.points[offset : offset + 20], baseline) if part["evals"]: - part_text += "\n\n" + eval_table( - record.evals[offset : offset + 20], - baseline_for(session, pull), - ) + part_text += "\n\n" + eval_table(record.evals[offset : offset + 20], baseline) upsert(session, pull, part_name, json.dumps(part), part_text) chunks.append(part_name) packed.update(points=[], evals=[]) packed = {"record": packed, "chunks": chunks} text = render_attempt( record.model_copy(update={"points": [], "evals": []}), - baseline_for(session, pull), + baseline, session.repository, ) note = "\n\nResults continue in numbered report comments." text = ( - text.replace("\n\n
", note + "\n\n
", 1) + text.replace("\n\n
", note + "\n\n
", 1).replace( + "\n\n
", " \n结果见后续编号的报告评论。\n\n
", 1 + ) if "
" in text else text + note ) @@ -843,6 +914,10 @@ def publish(session: Session, record: Baseline | Attempt) -> None: def upsert(session: Session, pull: dict, name: str, data: str, text: str) -> None: + """Create or update report comment ``name`` on ``pull`` with hidden ``data`` and ``text``. + + Raises VerificationError if the body exceeds GitHub's comment size. + """ previous = stored(session, pull, name) body = marker(session, name) + data + "\n-->\n" + text if len(body) > 60000: @@ -857,7 +932,11 @@ def upsert(session: Session, pull: dict, name: str, data: str, text: str) -> Non def publish_final( session: Session, run: dict, evidence: tuple[dict, list[dict], list[dict]] ) -> None: - """Normal finish and recovery publish the same artifact-derived final report.""" + """Publish the final report of passed sweep ``run`` built from its verified ``evidence``. + + Normal finish and recovery publish the same artifact-derived report. Mixed images raise + VerificationError. + """ from infx.results.eval_artifacts import eval_key from .validation import benchmark_entries, expected_evals @@ -932,6 +1011,7 @@ def publish_final( def public_point(entry: dict) -> dict: """Project generated settings onto the public BenchmarkRow identity (not metrics).""" from infx.matrix.generate import _hardware_family + from infx.results.identity import public_framework from .models import normalized_image @@ -940,7 +1020,7 @@ def public_point(entry: dict) -> dict: point = { "model": entry["model-prefix"], "hardware": _hardware_family(entry["runner"]), - "framework": entry["framework"], + "framework": public_framework(entry["framework"]), "precision": entry["precision"], "spec_method": entry["spec-decoding"], "disagg": entry.get("disagg", False), @@ -987,6 +1067,7 @@ def _matches_public_point(row: dict, point: dict) -> bool: def matrix_points(matrix: dict) -> list[dict]: + """Return one copy of each benchmark entry in ``matrix`` per concurrency, as ``conc``.""" from .validation import benchmark_entries return [ @@ -1006,7 +1087,9 @@ def missing_baseline_points(matrix: dict, baseline: Baseline) -> list[Point]: def check_baseline_coverage( repository: str, candidate: OwnedCandidate, matrix: dict, baseline: Baseline | None ) -> None: - """Current-family completeness cannot replace the frozen original point roster. + """Raise VerificationError unless ``matrix`` keeps the frozen ``baseline`` point roster. + + Current-family completeness cannot replace that roster. The agent can rewrite the recorded baseline, so its retirements exempt points only when the planner's rule, re-run on the family and MODELS.md at the candidate base (never the PR @@ -1122,6 +1205,7 @@ def markdown_table(markdown: str, header: tuple[str, ...]) -> list[dict[str, str """ def cells(line: str) -> list[str] | None: + """Return the trimmed cells of table row ``line``, or None if it is not one.""" line = line.strip() if len(line) < 2 or line[0] != "|" or line[-1] != "|": return None @@ -1142,6 +1226,7 @@ def cells(line: str) -> list[str] | None: def calendar_date(value: str) -> bool: + """Return True if ``value`` parses as an ISO date.""" try: datetime.fromisoformat(value) except ValueError: @@ -1169,6 +1254,7 @@ def deprecated_workloads( return {} def lists(cell: str, scenario: str) -> bool: + """Return True if ``cell`` names ``scenario`` as a whole word.""" return re.search(rf"(? dict: - """The public eval-row fields that identify a default eval of ``entry`` at ``conc``.""" + """The public eval-row fields that identify a default eval of ``entry`` at ``conc``. + + Eval rows carry no workload, offload or image, and eval ingest encodes an aggregate + multi-node deployment as disaggregated without decode workers. + """ point = public_point({**entry, "conc": conc}) omitted = ( EVAL_IDENTITY_OMITS @@ -1339,20 +1427,22 @@ def resolve_baseline( model: str, goal: Prose, producers: Mapping[str, dict], + matrix: dict, ) -> Baseline: """Freeze source-date rows against their own producer's complete family. - ``producers`` holds the families producer revisions regenerated themselves, so nothing - here runs a revision's code. Legacy fingerprints may be absent, but exact producer - provenance and a unique workload/topology/concurrency match are required. Raw API data - stays private. + ``matrix`` is the candidate family's canonical matrix at the candidate base. ``producers`` + holds the families producer revisions regenerated themselves, so nothing here runs a + revision's code. Legacy fingerprints may be absent, but exact producer provenance and a + unique workload/topology/concurrency match are required. Raw API data stays private. """ from fnmatch import fnmatchcase + from infx.results.identity import public_config_key + from .api import fetch - from .validation import PRODUCER_UNAVAILABLE, canonical_matrix + from .validation import PRODUCER_UNAVAILABLE - matrix = canonical_matrix(repository, candidate.base, candidate.family) public = publication(repository, context, model) old_image = context["source"]["image"] entries = {point_key(entry): entry for entry in matrix_points(matrix)} @@ -1364,6 +1454,10 @@ def resolve_baseline( current = {key: public_point(entry) for key, entry in entries.items()} def current_key(entry: dict) -> str | None: + """Return the current-family key of producer ``entry``, or None if it has none. + + Raises VerificationError if several current points match. + """ key = point_key(entry) if key in current: return key @@ -1373,7 +1467,8 @@ def current_key(entry: dict) -> str | None: raise VerificationError("Public baseline recipe identity is ambiguous or mismatched") return keys[0] if keys else None - historical: dict[str, list[dict]] = {} + historical: dict[str, list[tuple[dict, dict]]] = {} + additions: dict[str, list[tuple[str, dict]]] = {} published: dict[str, Point] = {} unverified: list[dict] = [] sources = list(public.sources) @@ -1395,10 +1490,11 @@ def dataset(row: dict, run_id: int, run_attempt: int | None) -> str | None: sources.append(feed.url) return datasets[run_id, run_attempt] + published_family = public_config_key(candidate.family.split(":", 1)[1]) family_runs = { int(change["workflow_run_id"]) for change in public.changelogs - if any(fnmatchcase(candidate.family.split(":", 1)[1], key) for key in change["config_keys"]) + if any(fnmatchcase(published_family, key) for key in change["config_keys"]) } for row, run_id, run_attempt, head in public.rows: if head is None: @@ -1407,10 +1503,10 @@ def dataset(row: dict, run_id: int, run_attempt: int | None) -> str | None: if head not in historical: if head not in producers: raise VerificationError(PRODUCER_UNAVAILABLE) - historical[head] = matrix_points(producers[head]) - matches = [ - entry for entry in historical[head] if _matches_public_point(row, public_point(entry)) - ] + historical[head] = [ + (entry, public_point(entry)) for entry in matrix_points(producers[head]) + ] + matches = [entry for entry, point in historical[head] if _matches_public_point(row, point)] if not matches: continue if row.get("recipe_fingerprint"): @@ -1426,12 +1522,14 @@ def dataset(row: dict, run_id: int, run_attempt: int | None) -> str | None: key = current_key(matches[0]) or point_key(matches[0]) if key in published: raise VerificationError("Duplicate public baseline point") - entries.update( - (point_key(point), point) - for point in historical[head] - if normalized_image(point["image"]) == normalized_image(old_image) - and current_key(point) is None - ) + if head not in additions: + additions[head] = [ + (point_key(point), point) + for point, _ in historical[head] + if normalized_image(point["image"]) == normalized_image(old_image) + and current_key(point) is None + ] + entries.update(additions[head]) entry = entries[key] published[key] = Point( key=key, diff --git a/inferencex-e2e/infx/klaud/validation.py b/inferencex-e2e/infx/klaud/validation.py index 99fd284577..b5deafd5d5 100644 --- a/inferencex-e2e/infx/klaud/validation.py +++ b/inferencex-e2e/infx/klaud/validation.py @@ -19,15 +19,19 @@ if TYPE_CHECKING: from infx.matrix.revision import Revision -# Klaud autosweeps cover only AgentX; fixed-seq-len points are out of scope. SCENARIO = "agentic-coding" def in_scope(entry: dict) -> bool: + """Return True if matrix ``entry`` is an AgentX point. + + Klaud autosweeps cover only AgentX, so fixed-seq-len points are out of scope. + """ return entry.get("scenario-type") == SCENARIO def benchmark_entries(matrix: dict) -> list[dict]: + """Return every single-node and multi-node benchmark entry of ``matrix``.""" return [ entry for group in ("single_node", "multi_node") @@ -37,6 +41,7 @@ def benchmark_entries(matrix: dict) -> list[dict]: def benchmark_points(entries: list[dict]) -> set[tuple]: + """Return the (recipe fingerprint, concurrency, image) point of each entry concurrency.""" return { (entry["recipe-fingerprint"], int(conc), entry["image"]) for entry in entries @@ -51,7 +56,7 @@ def project_prefix(repository: str, head: str) -> str: def canonical_matrix(repository: str, head: str, family: str) -> dict: - """Generate the family's AgentX points from exact-head YAML using trusted local code. + """Return the family's AgentX matrix generated from exact-head YAML by trusted local code. Candidate source is data only. A future generator-policy change can require manual inspection of an older run; it cannot silently waive points or required evals. @@ -116,6 +121,7 @@ class ProducerRegenerationError(VerificationError): """ def __init__(self, stderr: str) -> None: + """Store the tool's ``stderr`` and carry the fixed public reason as the message.""" super().__init__(PRODUCER_UNAVAILABLE) self.stderr = stderr @@ -163,6 +169,10 @@ def producer_matrix(repository: str, head: str, family: str) -> dict: def throughput_rows(producer: Revision, key: str) -> list[dict]: + """Return the producer's throughput rows for config ``key`` without eval-only rows. + + Producers too old for ``--no-evals`` have their eval rows stripped here instead. + """ try: return producer.generate([key], ["--no-evals"]) except subprocess.CalledProcessError as error: @@ -179,6 +189,10 @@ def throughput_rows(producer: Revision, key: str) -> list[dict]: def check_matrix(matrix: dict, canonical: dict, head: str, family: str) -> None: + """Raise VerificationError unless ``matrix`` matches ``canonical`` for ``head`` and ``family``. + + Checks the selected family, per-point settings, duplicate points, coverage and default evals. + """ from pydantic import TypeAdapter from infx.matrix.validation import ( @@ -227,6 +241,7 @@ def check_matrix(matrix: dict, canonical: dict, head: str, family: str) -> None: def expected_evals(matrix: dict) -> set[tuple]: + """Return the artifact identities of every default eval in ``matrix``.""" return {key for key, _, _ in eval_identities(matrix)} @@ -271,6 +286,10 @@ def eval_identities(matrix: dict) -> list[tuple[tuple, dict, Any]]: def check_coverage( directory: Path, manifest: dict, run: dict, family: str, canonical: dict ) -> None: + """Raise VerificationError unless the artifacts in ``directory`` cover the sweep. + + ``manifest`` must match ``run``, and its matrix must match ``canonical`` for ``family``. + """ if ( manifest["head"] != run["head_sha"] or manifest["run-id"] != run["id"] @@ -302,11 +321,10 @@ def check_coverage( def select_artifacts(inventory: list[dict], run: dict, attempt: dict) -> list[dict]: - """Newest same-name artifact within this run; never overlay two archives. + """Return the newest ``inventory`` artifact of each wanted name in ``run``, never overlaid. - Retained point artifacts from successful jobs in an earlier attempt are allowed. - The final manifest must be current. An earlier aggregate is retained only when - none of its point producers reran. Coverage/raw agreement still must pass. + Points from before ``attempt`` may be retained, but the manifest must be current and an + older aggregate only stands if none of its producers reran. Raises VerificationError. """ selected: dict[str, dict] = {} for artifact in inventory: @@ -355,6 +373,10 @@ def select_artifacts(inventory: list[dict], run: dict, attempt: dict) -> list[di def verify_sweep(repository: str, run: dict, family: str) -> tuple[dict, list[dict], list[dict]]: + """Verify the passed final sweep ``run`` for ``family`` and return its data. + + Return the canonical matrix, benchmark rows and eval rows, or raise VerificationError. + """ if run["status"] != "completed" or run["conclusion"] != "success": raise VerificationError("Final sweep has not passed") attempt = github.read(repository, f"actions/runs/{run['id']}/attempts/{run['run_attempt']}") diff --git a/inferencex-e2e/infx/launch/__main__.py b/inferencex-e2e/infx/launch/__main__.py index 97ba4f4ca7..7cd2bffb14 100644 --- a/inferencex-e2e/infx/launch/__main__.py +++ b/inferencex-e2e/infx/launch/__main__.py @@ -14,6 +14,7 @@ import yaml +from infx.bench.proc import status from infx.clusters import Cluster, resolve_cluster from infx.launch import drivers from infx.launch.backends import BACKENDS, backend_class @@ -28,7 +29,7 @@ def launch(cluster: Cluster, request: LaunchRequest, event: JobEventBuilder | No """Run ``request`` on ``cluster``; a launch error is one ``ERROR:`` line, not a traceback.""" with Lifecycle(event) as life: try: - rc = drivers.run(cluster, request, life) + rc = status(drivers.run(cluster, request, life)) except (LaunchError, BackendError, RequestError) as error: life.event.error(error) rc = 1 @@ -38,11 +39,16 @@ def launch(cluster: Cluster, request: LaunchRequest, event: JobEventBuilder | No def _run(runner_config: Path | None) -> int: + """Launch the point the environment describes, emit its job record and return its exit code.""" event = JobEventBuilder.begin(os.environ) rc = 1 try: request = LaunchRequest.from_env() - cluster = resolve_cluster(request.runner_name, runner_config) + try: + cluster = resolve_cluster(request.runner_name, runner_config) + except (OSError, ValueError, yaml.YAMLError) as error: + event.error(error) + return rc event.set(cluster=cluster.id) rc = launch(cluster, request, event) except RequestError as error: @@ -59,6 +65,7 @@ def _run(runner_config: Path | None) -> int: def _cleanup(runner_config: Path | None) -> int: + """Remove leftovers of earlier launches on ``RUNNER_NAME`` and return 0, or 1 if it is unset.""" runner = os.environ.get("RUNNER_NAME", "") if not runner: print("ERROR: RUNNER_NAME is required", file=sys.stderr) @@ -75,6 +82,7 @@ def _cleanup(runner_config: Path | None) -> int: def main(argv: list[str] | None = None) -> int: + """Parse ``argv``, run the ``run`` or ``cleanup`` command and return its exit code.""" parser = argparse.ArgumentParser(prog="python -m infx.launch", description=__doc__) parser.add_argument( "--runner-config", diff --git a/inferencex-e2e/infx/launch/artifacts.py b/inferencex-e2e/infx/launch/artifacts.py index 555ef2dd9f..9579d005e9 100644 --- a/inferencex-e2e/infx/launch/artifacts.py +++ b/inferencex-e2e/infx/launch/artifacts.py @@ -30,10 +30,12 @@ class ArtifactError(RuntimeError): def _say(message: str) -> None: + """Print ``message`` to stdout and flush.""" print(message, flush=True) def _warn(message: str) -> None: + """Print ``message`` to stderr and flush.""" print(message, file=sys.stderr, flush=True) @@ -61,6 +63,7 @@ def copy_to_workspace(source: Path, destination: Path) -> None: def _point_fields(filename: str) -> list[str]: + """Return the concurrency, GPU, ctx and gen fields of a result filename, blank if absent.""" return [match.group(1) if (match := p.search(filename)) else "" for p in _POINT_FIELDS] @@ -77,7 +80,10 @@ def _result_subdirs(logs_dir: Path) -> list[Path]: def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename: str) -> None: - """Copy srt-slurm's ``results_concurrency_*.json`` points under bounded workspace names.""" + """Copy srt-slurm's ``results_concurrency_*.json`` points under bounded workspace names. + + Skips benchmark_serving's ``.pytorch.json`` sidecars, which share a point's name prefix. + """ subdirs = _result_subdirs(logs_dir) if not subdirs: _warn(f"WARNING: no result subdirectories found in {logs_dir}") @@ -85,7 +91,7 @@ def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename _say(f"Processing result subdirectory: {subdir}") config_name = subdir.name for result_file in sorted(subdir.rglob("results_concurrency_*.json")): - if not result_file.is_file(): + if not result_file.is_file() or result_file.name.endswith(".pytorch.json"): continue concurrency, gpus, ctx, gen = _point_fields(result_file.name) _say( @@ -103,6 +109,7 @@ def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename def _top_level_files(directory: Path, pattern: str = "*") -> list[Path]: + """Return regular, non-symlink files directly in ``directory`` matching ``pattern``, sorted.""" return sorted( entry for entry in directory.iterdir() diff --git a/inferencex-e2e/infx/launch/backends/__init__.py b/inferencex-e2e/infx/launch/backends/__init__.py index 54971195fc..69cf8bd021 100644 --- a/inferencex-e2e/infx/launch/backends/__init__.py +++ b/inferencex-e2e/infx/launch/backends/__init__.py @@ -15,5 +15,6 @@ def backend_class(scheduler: str) -> type[Backend]: + """Import and return the backend class registered for ``scheduler``, or raise KeyError.""" module, _, name = BACKENDS[scheduler].partition(":") return getattr(importlib.import_module(module), name) diff --git a/inferencex-e2e/infx/launch/backends/base.py b/inferencex-e2e/infx/launch/backends/base.py index e13e6c00df..18a10888d4 100644 --- a/inferencex-e2e/infx/launch/backends/base.py +++ b/inferencex-e2e/infx/launch/backends/base.py @@ -57,6 +57,7 @@ class JobStatus: @property def succeeded(self) -> bool: + """Return whether the job ended in the succeeded state.""" return self.state is JobState.SUCCEEDED @@ -115,6 +116,7 @@ class Backend(ABC): """Runs containers and follows jobs on one cluster's scheduler for one launch.""" def __init__(self, cluster: Cluster, request: LaunchRequest, life: Lifecycle) -> None: + """Bind the backend to ``cluster``, the launch ``request`` and its ``life`` cycle.""" self.cluster = cluster self.request = request self.life = life diff --git a/inferencex-e2e/infx/launch/backends/slurm/__init__.py b/inferencex-e2e/infx/launch/backends/slurm/__init__.py index ef5b9c169d..4b1c7eae21 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/__init__.py +++ b/inferencex-e2e/infx/launch/backends/slurm/__init__.py @@ -48,7 +48,6 @@ from infx.launch.request import LaunchRequest CANCEL_POLL_S = 10.0 -# End states that lose the job to the cluster rather than fail its workload. _LOST_STATES = frozenset({"BOOT_FAIL", "CANCELLED", "NODE_FAIL", "PREEMPTED"}) @@ -144,7 +143,7 @@ def run_container(self, container: Container) -> SlurmJob: rendered = spec.srun_args() extra = ["--mpi=none", *(arg for arg in self.settings.srun_args if arg not in rendered)] argv = cli.srun_argv(allocation, container.command, container=spec, extra=extra) - echo(argv) + echo(argv, spec.env) step = subprocess.Popen(argv) self.life.callback(_stop, step) return SlurmJob(allocation.id, outputs=container.workspace, step=step) @@ -172,22 +171,26 @@ def state(self, job: Job) -> JobStatus: accounting times. """ job = _slurm_job(job) + allocation = "" if job.step is not None: rc = job.step.poll() if rc is None: return JobStatus(JobState.RUNNING, "srun step running") state = JobState.SUCCEEDED if rc == 0 else JobState.FAILED code = rc if rc >= 0 else 128 - rc - status = replace(cli.observe(job), state=state, raw=f"srun exit {rc}", exit_code=code) + observed = cli.observe(job) + allocation = observed.raw + status = replace(observed, state=state, raw=f"srun exit {rc}", exit_code=code) elif (queued := cli.queue_state(job)) is not None: pending = queued in {"PENDING", "CONFIGURING"} return JobStatus(JobState.PENDING if pending else JobState.RUNNING, queued) else: status = cli.final_status(job) - self._record_end(job, status) + self._record_end(job, status, allocation) return status - def _record_end(self, job: SlurmJob, status: JobStatus) -> None: + def _record_end(self, job: SlurmJob, status: JobStatus, allocation: str) -> None: + """Record ``status``; ``allocation`` is a container step's allocation accounting.""" event = self.life.event known = { "nodes": status.nodes, @@ -202,7 +205,13 @@ def _record_end(self, job: SlurmJob, status: JobStatus) -> None: return if job.step is not None: message = f"container step in Slurm job {job.id} exited {status.exit_code}" - event.fail("JobFailed", message, stage="run") + lost = allocation.partition("|")[0].split(" ", 1)[0] + if lost in _LOST_STATES: + kind = "Job" + lost.title().replace("_", "") + message += f" after its allocation ended {lost}" + event.fail(kind, message, stage="run", retriable=True) + else: + event.fail("JobFailed", message, stage="run") elif status.state in {JobState.FAILED, JobState.CANCELLED}: state, _, exit_code = status.raw.partition("|") ended = state.split(" ", 1)[0] @@ -377,11 +386,28 @@ def _resources(self, gpus: int, time_min: int) -> cli.Resources: ) def _allocate(self, gpus: int, time_min: int) -> Job: - """``salloc`` one node and cancel it when the launch ends.""" + """``salloc`` one node and cancel it when the launch ends. + + A signal kills a queued ``salloc`` before it names its job, so that request is + cancelled by the runner's job name instead. + """ settings = self.settings extra = ["--exclusive"] if settings.exclusive else [] extra += [f"--{name}={value}" for name, value in settings.cpu_directives().items()] - job = cli.salloc(self._resources(gpus, time_min), extra=[*extra, *settings.salloc_args]) + returned = False + + def cancel_unreturned() -> None: + """Cancel the runner's queued request by name if ``salloc`` never returned a job.""" + if not returned: + cli.cancel_named([self.request.runner_name], wait=False) + + self.life.callback(cancel_unreturned) + try: + job = cli.salloc(self._resources(gpus, time_min), extra=[*extra, *settings.salloc_args]) + except Exception: + returned = True + raise + returned = True self.life.callback(cli.cancel, job) return job diff --git a/inferencex-e2e/infx/launch/backends/slurm/cli.py b/inferencex-e2e/infx/launch/backends/slurm/cli.py index 3df075e207..4e3f5dd0a4 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/cli.py +++ b/inferencex-e2e/infx/launch/backends/slurm/cli.py @@ -19,6 +19,8 @@ from infx.launch.backends.base import BackendError, Job, JobState, JobStatus _GRANTED = re.compile(r"Granted job allocation ([1-9][0-9]*)") +SQUEUE_RETRY_S = 5.0 +CANCEL_NAMED_WAIT_S = 300.0 class SlurmError(BackendError): @@ -35,6 +37,7 @@ class ContainerSpec: env: dict[str, str] = field(default_factory=dict) def srun_args(self) -> list[str]: + """Return the ``srun`` container flags for this spec.""" args = [f"--container-image={self.image}"] if self.mounts: args.append( @@ -59,6 +62,7 @@ class Resources: exclude: Sequence[str] = () def args(self) -> list[str]: + """Return the ``salloc``/``sbatch`` flags requesting these resources on one node.""" args = [f"--partition={self.partition}"] if self.account: args.append(f"--account={self.account}") @@ -125,15 +129,26 @@ def sbatch( return Job(job_id) -def queue_state(job: Job, *, echo_command: bool = True) -> str | None: - """The job's squeue state, or None once squeue no longer lists it (or cannot run).""" +def _listed_state(job: Job, *, echo_command: bool) -> str | None: + """The job's squeue state, or None if squeue answers without it or is not installed. + + A failing squeue (a busy controller times out) is retried, then raises ``SlurmError``; + an id the controller has already purged is a definite answer, not a failure. + """ argv = ["squeue", "-j", job.id, "--noheader", "--format=%i|%T"] - try: - result = proc.run(argv, capture=True, echo_command=echo_command) - except FileNotFoundError: - return None - if result.returncode != 0: - return None + for attempt in range(3): + if attempt: + time.sleep(SQUEUE_RETRY_S) + try: + result = proc.run(argv, capture=True, echo_command=echo_command) + except FileNotFoundError: + return None + if result.returncode == 0: + break + if "Invalid job id specified" in result.stderr: + return None + else: + raise SlurmError(f"squeue failed for job {job.id} (exit {result.returncode})") for line in result.stdout.splitlines(): job_id, _, state = line.strip().partition("|") if job_id == job.id: @@ -141,8 +156,20 @@ def queue_state(job: Job, *, echo_command: bool = True) -> str | None: return None +def queue_state(job: Job, *, echo_command: bool = True) -> str | None: + """The job's squeue state, or None once squeue no longer lists it (or cannot run).""" + try: + return _listed_state(job, echo_command=echo_command) + except SlurmError: + return None + + def is_active(job: Job) -> bool: - return queue_state(job) is not None + """Whether squeue lists the job; a squeue that keeps failing cannot rule it out.""" + try: + return _listed_state(job, echo_command=True) is not None + except SlurmError: + return True def cancel(job: Job) -> None: @@ -151,24 +178,36 @@ def cancel(job: Job) -> None: proc.run(["scancel", job.id]) -def cancel_named(names: Sequence[str]) -> None: - """``scancel`` this user's jobs named any of ``names`` and wait until squeue drops them.""" +def cancel_named(names: Sequence[str], *, wait: bool = True) -> None: + """``scancel`` this user's jobs named any of ``names``; with ``wait``, until squeue drops them. + + A job stuck in the queue (e.g. ``COMPLETING`` on a hung node) is given up on after + ``CANCEL_NAMED_WAIT_S`` with a warning. + """ user = os.environ.get("USER") or getpass.getuser() for name in names: proc.run(["scancel", f"--user={user}", f"--name={name}"]) + if not wait: + return query = ["squeue", f"--user={user}", "--name=" + ",".join(names), "--noheader", "--format=%i"] + deadline = time.monotonic() + CANCEL_NAMED_WAIT_S while True: result = subprocess.run(query, capture_output=True, text=True, check=False) if result.returncode != 0 or not result.stdout.strip(): return - print( - f"Waiting for jobs {' '.join(result.stdout.split())} to leave the queue", - file=sys.stderr, - ) + jobs = " ".join(result.stdout.split()) + if time.monotonic() >= deadline: + print( + f"WARNING: jobs {jobs} still in the queue after {CANCEL_NAMED_WAIT_S:.0f}s", + file=sys.stderr, + ) + return + print(f"Waiting for jobs {jobs} to leave the queue", file=sys.stderr) time.sleep(5) def _query(argv: list[str], env: Mapping[str, str] | None = None) -> str: + """Run a Slurm query and return its stdout, or an empty string if it fails or is absent.""" try: result = proc.run(argv, env=env, capture=True) except FileNotFoundError: @@ -197,7 +236,8 @@ def _time(text: str) -> datetime | None: def observe(job: Job) -> JobStatus: """One reading of the allocation: accounting (``sacct -X``), else the controller's record. - Never raises; a job neither knows reads as ``UNKNOWN``. + Never raises; a job neither knows reads as ``UNKNOWN``. Start and end times are dropped while + the job is not yet started or finished, since scontrol projects them from the time limit. """ env = {**os.environ, "SLURM_TIME_FORMAT": "%s"} argv = ["sacct", "-X", "-n", "-P", "-j", job.id] @@ -216,9 +256,11 @@ def observe(job: Job) -> JobStatus: fields.get(name, "") for name in ("JobState", "ExitCode", "NodeList", "SubmitTime", "StartTime", "EndTime") ) - nodes = nodes.strip() + nodes, state = nodes.strip(), state.strip() + started = "" if state == "PENDING" else started + ended = "" if state in {"PENDING", "CONFIGURING", "RUNNING"} else ended return replace( - job_status(state.strip(), exit_code.strip()), + job_status(state, exit_code.strip()), nodes=None if nodes in {"", "None assigned", "(null)"} else nodes, submitted_at=_time(submitted), started_at=_time(started), @@ -227,10 +269,17 @@ def observe(job: Job) -> JobStatus: def job_status(state: str, exit_code: str) -> JobStatus: - """Only ``COMPLETED`` with ``0:0`` succeeds; any other state after the job ran failed.""" - code, _, _ = exit_code.partition(":") + """Only ``COMPLETED`` with ``0:0`` succeeds; any other state after the job ran failed. + + Slurm's ``ExitCode`` is ``status:signal``, and a death by signal N reads as 128 + N. + Only signals 1 to 64 count, so OUT_OF_MEMORY's ``0:125`` marker is not read as a signal. + """ + code, _, signal = exit_code.partition(":") raw = f"{state}|{exit_code}" - exit_status = int(code) if code.isdigit() else None + if signal.isdigit() and 0 < int(signal) <= 64: + exit_status: int | None = 128 + int(signal) + else: + exit_status = int(code) if code.isdigit() else None if state == "COMPLETED" and exit_code == "0:0": return JobStatus(JobState.SUCCEEDED, raw, 0) if not state: @@ -268,7 +317,7 @@ def wait_for_log(job: Job, path: Path, *, wait_s: float = 5.0) -> None: Raises ``SlurmError``, after printing ``scontrol show job``, if the job ends first. """ while not path.exists(): - if queue_state(job, echo_command=False) is None: + if queue_state(job, echo_command=False) is None and not path.exists(): print(f"ERROR: job {job.id} failed before creating {path}", file=sys.stderr) with contextlib.suppress(FileNotFoundError): proc.run(["scontrol", "show", "job", job.id]) diff --git a/inferencex-e2e/infx/launch/backends/slurm/squash.py b/inferencex-e2e/infx/launch/backends/slurm/squash.py index 5c2126db9b..3989ae1586 100644 --- a/inferencex-e2e/infx/launch/backends/slurm/squash.py +++ b/inferencex-e2e/infx/launch/backends/slurm/squash.py @@ -11,6 +11,7 @@ import glob import os import re +import shutil import socket import stat import sys @@ -59,6 +60,7 @@ def squash_path(image: str, policy: SquashPolicy) -> Path: def lock_path(image: str, policy: SquashPolicy) -> Path: + """Return the lock file guarding the squash file of ``image`` under the policy's lock layout.""" path = squash_path(image, policy) if policy.lock_file == "locks-dir": return path.parent / ".locks" / f"{squash_key(image)}.lock" @@ -99,6 +101,7 @@ def registry_reference(image: str) -> str: def is_valid_squash(path: Path) -> bool: + """Return whether ``path`` is readable and ``unsquashfs`` can list it.""" if not os.access(path, os.R_OK): return False try: @@ -157,6 +160,8 @@ def _locked(path: Path, timeout_s: int) -> Iterator[None]: def _import_here(image: str, policy: SquashPolicy) -> None: """Import on this host into a temp file, validate, ``chmod a+r``, rename into place.""" path = squash_path(image, policy) + if shutil.which("unsquashfs") is None: + raise _PermanentImageError(f"unsquashfs is not installed here; cannot validate {path}") with _locked(lock_path(image, policy), policy.lock_timeout_s): if is_valid_squash(path): print( @@ -305,7 +310,9 @@ def ensure_image( return str(path) node_local = policy.visibility == "node-local" if node_local and job is None: - raise ImageError(f"node-local image storage for {image} requires a job allocation") + raise _PermanentImageError( + f"node-local image storage for {image} requires a job allocation" + ) if mode == "pre-staged": if node_local: _retrying( @@ -313,7 +320,7 @@ def ensure_image( lambda: _run_node_script(image, policy, "validate", job=job, alloc_args=()), ) elif not is_valid_squash(path): - raise ImageError( + raise _PermanentImageError( f"pre-staged image {image} is missing or invalid at {path}; stage it there before running" ) return str(path) @@ -324,7 +331,9 @@ def ensure_image( print(f"Squash file already exists and is valid, skipping import: {path}", file=sys.stderr) return str(path) if job is None and (mode == "all-nodes" or not alloc_args): - raise ImageError(f"{mode} import of {image} needs a job allocation or allocation arguments") + raise _PermanentImageError( + f"{mode} import of {image} needs a job allocation or allocation arguments" + ) _retrying( f"import of {image}", lambda: _run_node_script(image, policy, "import", job=job, alloc_args=alloc_args), diff --git a/inferencex-e2e/infx/launch/drivers/script.py b/inferencex-e2e/infx/launch/drivers/script.py index adec32bd58..995a35fd29 100644 --- a/inferencex-e2e/infx/launch/drivers/script.py +++ b/inferencex-e2e/infx/launch/drivers/script.py @@ -7,6 +7,7 @@ from __future__ import annotations import fnmatch +import posixpath from pathlib import PurePosixPath from typing import TYPE_CHECKING @@ -32,6 +33,7 @@ def container_workspace(image: str) -> PurePosixPath: + """Return ``/ix`` for images with an editable install there, else ``/workspace``.""" if any(fnmatch.fnmatchcase(image, glob) for glob in EDITABLE_INSTALL_IMAGE_GLOBS): return PurePosixPath("/ix") return PurePosixPath("/workspace") @@ -58,13 +60,14 @@ def script_outputs(request: ScriptRequest, workdir: PurePosixPath) -> tuple[Pure out_yaml = request.env.get("OUT_YAML") if not out_yaml: return (RESULTS_DIR,) - path = workdir / out_yaml + path = PurePosixPath(posixpath.normpath(workdir / out_yaml)) if not path.is_relative_to(workdir): raise LaunchError(f"OUT_YAML {out_yaml} lies outside the container workspace {workdir}") return (path.relative_to(workdir), RESULTS_DIR) def run(launch: Launch) -> int: + """Run the request's benchmark script in one container and return its exit code.""" request = ScriptRequest.from_env(launch.request.env) cluster, backend = launch.cluster, launch.backend time_limit = policy.salloc_time_limit(cluster.id, request) diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index 48e50d567d..1b0a3b0a96 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -54,6 +54,8 @@ def run_single_node(launch: Launch) -> int: event = launch.life.event with event.stage("prepare"): request = SingleNodeRequest.from_env(launch.request.env) + if request.is_agentic and not request.eval_only: + require(request, "INFERENCEX_RESULTS_PYTHON") model_path = models.single_node_model_path(launch.cluster, request) staged = {"MODEL_PATH": model_path} if model_path.startswith("/") else {} run = SrtRun.create(launch, request, staged) @@ -149,9 +151,9 @@ def run_batch(launch: Launch) -> int: backend.stream_logs(job) except BackendError: return 1 - succeeded = backend.state(job).succeeded + status = backend.state(job) event.adopt_reentry() - return 0 if succeeded else 1 + return 0 if status.succeeded else status.exit_code or 1 def run_multinode(launch: Launch) -> int: @@ -168,6 +170,8 @@ def run_multinode(launch: Launch) -> int: raise LaunchError(f"{srt_recipe} is not in the recipe mirror") decision = power.decide_power(launch.cluster.id, launch.path, request) event.set(power=asdict(decision)) + if not request.eval_only and (decision.agentx or decision.adapter): + require(request, "INFERENCEX_RESULTS_PYTHON") model = models.checkpoint(launch.cluster, request) served = models.served_path(launch.cluster, request, model) model_paths = models.model_paths(launch.cluster, request, served) @@ -204,6 +208,7 @@ def run_multinode(launch: Launch) -> int: def _srt_settings(cluster: Cluster | None) -> SrtSlurmSettings | None: + """Return the cluster's srt-slurm settings, or None if it is absent or not a Slurm cluster.""" settings = cluster.scheduler_settings if cluster is not None else None return settings.srt_slurm if isinstance(settings, SlurmSettings) else None @@ -213,15 +218,18 @@ def table_problems(clusters: Mapping[str, Cluster], only: str | None = None) -> problems: list[str] = [] def scoped(cluster_id: str) -> bool: + """Return whether ``cluster_id`` is within the checked scope.""" return only in (None, cluster_id) def profile(table: str, cluster_id: str) -> SrtSlurmSettings | None: + """Return the srt-slurm settings of ``cluster_id``, recording a problem if it has none.""" srt = _srt_settings(clusters.get(cluster_id)) if srt is None: problems.append(f"{table}[{cluster_id!r}]: no srt-slurm cluster {cluster_id!r}") return srt def volume(where: str, cluster_id: str, name: str) -> None: + """Record a problem at ``where`` if ``cluster_id`` has no volume ``name``.""" if name not in clusters[cluster_id].scheduler_settings.volumes: problems.append(f"{where}: no volume {name!r}") diff --git a/inferencex-e2e/infx/launch/drivers/srt/checkout.py b/inferencex-e2e/infx/launch/drivers/srt/checkout.py index 3653ba2dbc..49e0a5e537 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/checkout.py +++ b/inferencex-e2e/infx/launch/drivers/srt/checkout.py @@ -36,10 +36,12 @@ class Checkout: @property def venv(self) -> Path: + """Return the checkout's virtualenv directory.""" return self.root / ".venv" def _git(*args: str | Path, capture: bool = False) -> str: + """Run ``git`` and return stripped stdout if ``capture``, or raise LaunchError on failure.""" try: result = proc.run(["git", *args], check=True, capture=capture) except (OSError, subprocess.CalledProcessError) as error: @@ -48,6 +50,7 @@ def _git(*args: str | Path, capture: bool = False) -> str: def _checked(result: subprocess.CompletedProcess[str], action: str) -> None: + """Raise LaunchError naming ``action`` if ``result`` exited non-zero.""" if result.returncode: raise LaunchError(f"{action} failed (exit {result.returncode})") @@ -113,7 +116,8 @@ def _uv(run: SrtRun) -> str: install_dir = root / "bin" if root is not None else Path.home() / ".local/bin" install_dir.mkdir(parents=True, exist_ok=True) env = {**run.env, "UV_INSTALL_DIR": str(install_dir), "UV_NO_MODIFY_PATH": "1"} - _checked(proc.run(["sh", "-c", f"curl -LsSf {UV_INSTALLER} | sh"], env=env), "installing uv") + installer = f"curl -LsSf {UV_INSTALLER} | sh" + _checked(proc.run(["bash", "-o", "pipefail", "-c", installer], env=env), "installing uv") run.prepend_path(install_dir) return str(install_dir / "uv") @@ -163,6 +167,7 @@ def _installed_version(venv: Path, name: str) -> str | None: def _run_logged(argv: list[str], log: Path, *, env: dict[str, str], cwd: Path) -> int: + """Echo and run ``argv`` in ``cwd``, append its output to ``log`` and return its exit code.""" echo(argv, env) with log.open("a") as handle: return subprocess.run( diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index 2cd09331f7..228021b558 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -164,6 +164,7 @@ def collect( logs_staged = False def stage_logs_on_exit() -> None: + """Fetch and stage the job's logs when the launch ends before collect staged them.""" if not logs_staged: _stage_logs(run, backend.fetch_outputs(job, fetched) / "logs", power) @@ -258,6 +259,9 @@ def cleanup_outputs(root: Path, *, sleep: Callable[[float], None] = time.sleep) except FileNotFoundError: break except OSError: + if attempt == attempts: + print(f"outputs still busy after {attempts} attempts; leaving it", flush=True) + break print(f"Retry {attempt}/{attempts}: Waiting for NFS locks to release...", flush=True) sleep(10) for directory, dirnames, filenames in os.walk(root, topdown=False): diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 4ffbeca23b..f33ae62222 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -107,6 +107,7 @@ class SrtLane: def srt_lane(cluster_id: str, path: LaunchPath) -> SrtLane: + """Return the srt-slurm lane for a cluster and launch path, or raise LaunchError if none.""" try: return SRT_LANES[(cluster_id, path)] except KeyError: diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 5f0e9b33cb..f52d8b62c5 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -117,6 +117,7 @@ def checkpoint(cluster: Cluster, request: LaunchRequest) -> Checkpoint | None: entries = cluster.models.entries def node_local(key: str) -> bool: + """Return whether model entry ``key`` lives on a node-local volume.""" return volumes[entries[key].root].visibility == "node-local" key = _override(cluster, request, "entry") diff --git a/inferencex-e2e/infx/launch/drivers/srt/recipe.py b/inferencex-e2e/infx/launch/drivers/srt/recipe.py index 074e4b0fed..57e63c9fe6 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/recipe.py +++ b/inferencex-e2e/infx/launch/drivers/srt/recipe.py @@ -12,6 +12,7 @@ import yaml +from infx import yamlio from infx.launch.context import LaunchError if TYPE_CHECKING: @@ -40,7 +41,7 @@ def prepare_recipe(recipe: Path, job_name: str, dist_timeout_s: int | None) -> N """Edit the bound recipe, the variant srtctl submits, for this job: ``job_name`` becomes srtctl's run name, the health check waits at least HEALTH_ATTEMPTS attempts, and each role with a ``watchdog-timeout`` gets the cluster's ``dist-timeout``.""" - bound = yaml.safe_load(recipe.read_text()) + bound = yamlio.safe_load(recipe.read_text()) if "name" in bound: bound["name"] = job_name if "max_attempts" in (health := bound.get("health_check") or {}): diff --git a/inferencex-e2e/infx/launch/drivers/srt/run.py b/inferencex-e2e/infx/launch/drivers/srt/run.py index 80effcf96e..d11983b760 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/run.py +++ b/inferencex-e2e/infx/launch/drivers/srt/run.py @@ -52,9 +52,11 @@ def create( @property def workspace(self) -> Path: + """Return the request's runner workspace directory.""" return self.request.workspace def prepend_path(self, directory: Path) -> None: + """Put ``directory`` first on the children's ``PATH``.""" self.env["PATH"] = os.pathsep.join(filter(None, (str(directory), self.env.get("PATH")))) @@ -66,6 +68,7 @@ def require(request: LaunchRequest, *names: str) -> None: def slurm_backend(launch: Launch) -> SlurmBackend: + """Return the launch's Slurm backend, or raise TypeError if it uses another backend.""" if not isinstance(launch.backend, SlurmBackend): raise TypeError( f"{launch.path} needs the Slurm backend, got {type(launch.backend).__name__}" diff --git a/inferencex-e2e/infx/launch/drivers/srt/submit.py b/inferencex-e2e/infx/launch/drivers/srt/submit.py index b7d6735619..19787679aa 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/submit.py +++ b/inferencex-e2e/infx/launch/drivers/srt/submit.py @@ -205,6 +205,7 @@ def cancel(self, backend: SlurmBackend) -> None: backend.cancel(job) def adopted(self) -> SlurmJob: + """Return the submitted job, or raise LaunchError if srtctl reported none.""" if self.job is None: raise LaunchError("srtctl reported no submitted job") return self.job diff --git a/inferencex-e2e/infx/launch/event.py b/inferencex-e2e/infx/launch/event.py index b8ec37664a..cf8237b255 100644 --- a/inferencex-e2e/infx/launch/event.py +++ b/inferencex-e2e/infx/launch/event.py @@ -118,6 +118,7 @@ class Metrics(_Record): ) @classmethod def _rounded(cls, value: float | None) -> float | None: + """Round a metric to three decimals, keeping None.""" return None if value is None else round(value, 3) @@ -208,6 +209,7 @@ def __init__( *, annotate: bool = False, ) -> None: + """Start timing a launch that records into ``event`` and tracks ``workspace`` files.""" self.event = event if event is not None else JobEvent() self.workspace = workspace self._annotate = annotate @@ -223,7 +225,11 @@ def __init__( @classmethod def begin(cls, env: Mapping[str, str]) -> Self: - """Identify the job from the workflow environment; never raises.""" + """Identify the job from the workflow environment; never raises. + + AgentX replays traces, so agentic runs leave ISL and OSL unset. A batch-wrapped launch + never annotates on GitHub, because its wrapper is the job's exit boundary. + """ scenario = _text(env, "SCENARIO_TYPE") multinode = env.get("IS_MULTINODE") == "true" eval_only = env.get("EVAL_ONLY") == "true" @@ -245,7 +251,6 @@ def begin(cls, env: Mapping[str, str]) -> Self: for word in env.get("CONC_LIST", "").split() if word.isascii() and word.isdecimal() ], - # AgentX replays traces; the workflow's ISL/OSL of 0 only fill the fixed shape. isl=None if scenario == AGENTIC else _number(env, "ISL"), osl=None if scenario == AGENTIC else _number(env, "OSL"), topology=_topology(env, multinode=multinode), @@ -261,15 +266,16 @@ def begin(cls, env: Mapping[str, str]) -> Self: kv_offload_backend=_text(env, "KV_OFFLOAD_BACKEND"), ) workspace = Path(env["GITHUB_WORKSPACE"]) if env.get("GITHUB_WORKSPACE") else None - # A batch-wrapped launch is not the job's exit boundary; the wrapper annotates. annotate = env.get("GITHUB_ACTIONS") == "true" and env.get(BATCH_REENTRY_ENV) != "1" return cls(event, workspace, annotate=annotate) def set(self, **fields: object) -> None: + """Set the given fields on the event.""" for name, value in fields.items(): setattr(self.event, name, value) def set_versions(self, **versions: str | None) -> None: + """Set the given component versions on the event.""" for name, value in versions.items(): setattr(self.event.versions, name, value) @@ -290,6 +296,7 @@ def stage(self, name: str) -> Iterator[None]: @property def current_stage(self) -> str: + """Return the innermost running stage, else the last one, else ``prepare``.""" return self._active[-1] if self._active else self._last or "prepare" def evidence(self, log: Path) -> None: @@ -324,10 +331,12 @@ def error(self, error: BaseException, *, stage: str | None = None, report: bool self.fail(kind, str(error), stage=stage, retriable=retriable, report=report) def interrupted(self, signum: int) -> None: + """Record a retriable failure for the received signal ``signum``.""" message = f"received {signal.Signals(signum).name}" self.fail("Interrupted", message, retriable=True, report=False) def exited(self, returncode: int) -> None: + """Record a failure if ``returncode`` is non-zero.""" if returncode: self.fail("ExitStatus", f"exited {returncode}", report=False) @@ -344,13 +353,17 @@ def adopt_reentry(self) -> None: self._reentry = inner def finish(self, returncode: int) -> JobEvent: + """Return the completed record for a launch that exited ``returncode``. + + After a batch re-entry the wrapped launch's record is the base, since it knows the + benchmark job, and this launch adds its batch job, queue time and ``batch_`` stages. + """ self._lap() self.exited(returncode) record = self.event.model_copy(deep=True) stages = dict(self._stages) failure = self._failure if (inner := self._reentry) is not None: - # The wrapped launch knows the benchmark job; this one only the batch around it. outer = record record = inner.model_copy(deep=True) record.launch_path = outer.launch_path @@ -412,6 +425,7 @@ def emit(self, returncode: int) -> None: print(f"WARNING: could not write {path}: {error}", file=sys.stderr) def _lap(self) -> None: + """Charge the time since the last lap to the innermost active stage.""" now = time.monotonic() if self._active: name = self._active[-1] @@ -419,6 +433,7 @@ def _lap(self) -> None: self._since = now def _shown(self, path: Path | None) -> str | None: + """Return ``path`` relative to the workspace when inside it, else as given, or None.""" if path is None: return None if self.workspace is not None and path.is_relative_to(self.workspace): @@ -456,6 +471,7 @@ def annotation(error: JobError) -> str: def _escape_data(text: str) -> str: + """Escape ``%``, CR and LF in ``text`` for a GitHub Actions workflow command.""" return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") @@ -474,19 +490,23 @@ def _files(directory: Path | None) -> dict[str, tuple[int, int]]: def _text(env: Mapping[str, str], name: str) -> str | None: + """Return the variable ``name`` from ``env``, or None if unset or empty.""" return env.get(name) or None def _number(env: Mapping[str, str], name: str) -> int | None: + """Return the variable ``name`` as a non-negative integer, or None if it is not one.""" value = env.get(name, "") return int(value) if value.isascii() and value.isdecimal() else None def _flag(env: Mapping[str, str], name: str) -> bool | None: + """Return the variable ``name`` as a bool when it is ``true`` or ``false``, else None.""" return {"true": True, "false": False}.get(env.get(name, "")) def _topology(env: Mapping[str, str], *, multinode: bool) -> Topology: + """Return single-node parallelism, or per-role topology when ``multinode``.""" if not multinode: return Topology( tp=_number(env, "TP"), @@ -508,6 +528,7 @@ def _topology(env: Mapping[str, str], *, multinode: bool) -> Topology: def _versions(env: Mapping[str, str], *, agentx: bool) -> Versions: + """Return checkout, workflow and image versions, plus the AgentX harness if ``agentx``.""" head = None with contextlib.suppress(OSError): head = _git_head(repository_root()) @@ -527,6 +548,7 @@ def _versions(env: Mapping[str, str], *, agentx: bool) -> Versions: def _github(env: Mapping[str, str]) -> GitHub: + """Return the run and job URLs and the event, PR, test and ref details from ``env``.""" server, repository, run = ( _text(env, name) for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID") ) @@ -633,16 +655,19 @@ def _agentx_point(name: str, result: dict[str, Any]) -> Metrics: def _mapping(parent: dict[str, Any], key: str) -> dict[str, Any]: + """Return ``parent[key]`` if it is a dict, else an empty dict.""" value = parent.get(key) return value if isinstance(value, dict) else {} def _count(parent: dict[str, Any], key: str) -> int | None: + """Return ``parent[key]`` if it is a non-bool int, else None.""" value = parent.get(key) return value if isinstance(value, int) and not isinstance(value, bool) else None def _value(parent: dict[str, Any], key: str) -> float | None: + """Return ``parent[key]`` as a float if it is a finite non-bool number, else None.""" value = parent.get(key) if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value): return None @@ -650,4 +675,5 @@ def _value(parent: dict[str, Any], key: str) -> float | None: def _milliseconds(seconds: float | None) -> float | None: + """Convert ``seconds`` to milliseconds, keeping None.""" return None if seconds is None else seconds * 1000 diff --git a/inferencex-e2e/infx/launch/evidence.py b/inferencex-e2e/infx/launch/evidence.py index 9a3f62b455..9e935768a8 100644 --- a/inferencex-e2e/infx/launch/evidence.py +++ b/inferencex-e2e/infx/launch/evidence.py @@ -18,7 +18,6 @@ _ESCAPES = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])") _NOISE = re.compile(r"warn|^\+ ", re.IGNORECASE) -# Most specific first: a resource fault names the cause better than the exception it raises. _SIGNATURES = ( re.compile( r"out of memory|OutOfMemoryError|\bOOM\b|oom[-_ ]?kill|CUDA error|\bNCCL\w* ?error\b" @@ -60,7 +59,10 @@ def error_detail(log: Path) -> str | None: def _cause(handle: BinaryIO) -> str | None: - """The first line of the most specific signature rank in the scanned head of ``handle``.""" + """The first line of the most specific signature rank in the scanned head of ``handle``. + + Resource faults rank first, since they name the cause better than the exception they raise. + """ best: tuple[int, str] | None = None scanned = 0 while scanned < SCAN_BYTES and (raw := handle.readline(_LINE_BYTES)): diff --git a/inferencex-e2e/infx/launch/lifecycle.py b/inferencex-e2e/infx/launch/lifecycle.py index fae7f60700..7993ae1ef3 100644 --- a/inferencex-e2e/infx/launch/lifecycle.py +++ b/inferencex-e2e/infx/launch/lifecycle.py @@ -17,6 +17,7 @@ class _Interrupted(BaseException): def __init__(self, signum: int) -> None: + """Carry the received signal number ``signum``.""" super().__init__(signum) self.signum = signum @@ -30,6 +31,7 @@ class Lifecycle: """ def __init__(self, event: JobEventBuilder | None = None) -> None: + """Start with no cleanups and exit code 0, recording into ``event`` or a fresh one.""" self.event = event if event is not None else JobEventBuilder() self._stack = ExitStack() self._rc = 0 @@ -40,11 +42,13 @@ def __init__(self, event: JobEventBuilder | None = None) -> None: @property def returncode(self) -> int: + """Return the first recorded nonzero code, else 1 if a cleanup failed, else 0.""" if self._rc: return self._rc return 1 if self._cleanup_failed else 0 def record(self, rc: int) -> None: + """Keep ``rc`` as the exit code unless it is 0 or a nonzero code is already recorded.""" if rc and not self._rc: self._rc = rc @@ -52,6 +56,7 @@ def callback(self, fn: Callable[..., Any], /, *args: Any, **kwargs: Any) -> None """Register a cleanup; one that raises or returns a nonzero int fails the launch.""" def run() -> None: + """Call the cleanup and record a failure if it raises or returns a nonzero int.""" try: result = fn(*args, **kwargs) except Exception as error: # noqa: BLE001 @@ -68,6 +73,7 @@ def run() -> None: self._stack.callback(run) def _on_signal(self, signum: int, frame: FrameType | None) -> None: + """Defer ``signum`` while exiting, else ignore later signals and raise ``_Interrupted``.""" if self._exiting or (frame is not None and frame.f_code is _EXIT_CODE): self._signum = self._signum or signum return @@ -76,6 +82,7 @@ def _on_signal(self, signum: int, frame: FrameType | None) -> None: raise _Interrupted(signum) def __enter__(self) -> Self: + """Install the signal handlers and return this lifecycle.""" for sig in _SIGNALS: self._previous[sig] = signal.signal(sig, self._on_signal) return self @@ -86,6 +93,10 @@ def __exit__( exc: BaseException | None, tb: TracebackType | None, ) -> bool: + """Record how the body ended, run the cleanups and restore the signal handlers. + + Raises ``SystemExit(128 + signum)`` after a signal, else returns False to propagate errors. + """ self._exiting = True mask = signal.pthread_sigmask(signal.SIG_BLOCK, _SIGNALS) if isinstance(exc, _Interrupted): diff --git a/inferencex-e2e/infx/launch/policy.py b/inferencex-e2e/infx/launch/policy.py index d6bf0edba4..9a7b964e9f 100644 --- a/inferencex-e2e/infx/launch/policy.py +++ b/inferencex-e2e/infx/launch/policy.py @@ -6,7 +6,6 @@ from __future__ import annotations -import contextlib import fnmatch import json from collections.abc import Mapping @@ -14,12 +13,15 @@ from enum import StrEnum from typing import TYPE_CHECKING +from infx.launch.request import RequestError + if TYPE_CHECKING: from infx.clusters import Cluster from infx.launch.request import LaunchRequest def any_of(*values: str) -> frozenset[str]: + """Return ``values`` as a frozenset for a ``Match`` field.""" return frozenset(values) @@ -36,6 +38,7 @@ class Match: model_glob: str | None = None def __call__(self, request: LaunchRequest) -> bool: + """Return whether ``request`` satisfies every set field.""" return ( (self.prefixes is None or request.model_prefix in self.prefixes) and (self.precisions is None or request.precision in self.precisions) @@ -75,6 +78,7 @@ class LaunchPath(StrEnum): def launch_path(cluster_id: str, request: LaunchRequest) -> LaunchPath: + """Return the launch path that runs ``request`` on the cluster ``cluster_id``.""" if request.is_multinode: if any(lane(request) for lane in NATIVE_SRT_LANES.get(cluster_id, ())): return LaunchPath.SRT_NATIVE @@ -133,12 +137,23 @@ def runtime_env( def point_settings(request: LaunchRequest) -> frozenset[str]: - """The names the point's NAME=value additional-settings set; the workflow exports them.""" + """The names the point's NAME=value additional-settings set; the workflow exports them. + + A null value, which the workflow sends for a role without additional-settings, sets none. + Raises ``RequestError`` if a value is not a JSON list. + """ names: set[str] = set() for name in ("PREFILL_ADDITIONAL_SETTINGS", "DECODE_ADDITIONAL_SETTINGS"): - with contextlib.suppress(ValueError): + try: values = json.loads(request.env.get(name) or "[]") - names.update(v.partition("=")[0] for v in values or [] if isinstance(v, str)) + except ValueError: + continue + if values is None: + continue + if not isinstance(values, list): + kind = type(values).__name__ + raise RequestError(f"{name} must be a JSON list of NAME=value strings, got {kind}") + names.update(v.partition("=")[0] for v in values if isinstance(v, str)) return frozenset(names) @@ -146,6 +161,7 @@ def table_problems(clusters: Mapping[str, Cluster], only: str | None = None) -> """Rows of this module's tables that ``clusters`` contradicts; only ``only``'s rows if given.""" def keys(table: Mapping[str, object]) -> list[str]: + """Return the keys of ``table`` within the checked scope.""" return [key for key in table if only in (None, key)] tables: dict[str, Mapping[str, object]] = { diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index 2c463a062d..2c37dd67ad 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -21,14 +21,17 @@ class RequestError(ValueError): @classmethod def missing(cls, *names: str) -> RequestError: + """Return an error naming the unset environment variables ``names``.""" return cls(f"required environment variables are not set: {', '.join(names)}") def _equals(*truthy: str) -> BeforeValidator: + """Return a validator reading a string as True when it is one of ``truthy``.""" return BeforeValidator(lambda value: value if isinstance(value, bool) else value in truthy) def _int_list(value: Any) -> Any: + """Split a whitespace-separated string into a list, passing other values through.""" return value.split() if isinstance(value, str) else value @@ -39,6 +42,7 @@ def _int_list(value: Any) -> Any: def _describe(error: ValidationError) -> str: + """Render a validation error as one message listing missing then invalid inputs.""" missing: list[str] = [] invalid: list[str] = [] for detail in error.errors(include_url=False, include_input=False): @@ -113,6 +117,7 @@ def from_env(cls, env: Mapping[str, str] | None = None) -> Self: @property def workspace(self) -> Path: + """Return ``GITHUB_WORKSPACE``, or raise ``RequestError`` if it is unset.""" if self.github_workspace is None: raise RequestError.missing("GITHUB_WORKSPACE") return self.github_workspace diff --git a/inferencex-e2e/infx/matrix/changed.py b/inferencex-e2e/infx/matrix/changed.py index c129b953f3..f18b6fe7b5 100644 --- a/inferencex-e2e/infx/matrix/changed.py +++ b/inferencex-e2e/infx/matrix/changed.py @@ -4,8 +4,8 @@ import json import os import subprocess +import threading from collections.abc import Iterable -from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any @@ -13,28 +13,35 @@ from infx.config import MASTER_CONFIGS, RUNNER_CONFIG -from .fingerprint import recipe_fingerprint +from .fingerprint import recipe_fingerprint, row_fingerprint from .generate import generate_config_matrix from .revision import snapshot from .validation import config_root, load_config_files, load_runner_file -Matrix = dict[str, tuple[dict[tuple[str | None, int], dict[str, Any]], list[str]]] +Matrix = dict[str, tuple[dict[tuple[str, int], dict[str, Any]], list[str]]] def _points(rows: list[dict], root: Path) -> tuple[dict, list[str]]: + """Expand generated rows into one point per concurrency, keyed by (fingerprint, conc). + + Return the points and the messages of rows whose recipe could not be fingerprinted. + """ points, errors = {}, [] for row in rows: try: fingerprint = recipe_fingerprint(row, root) + identity = fingerprint except ValueError as error: fingerprint = None + identity = row_fingerprint(row) errors.append(str(error)) for conc in row["conc"] if isinstance(row["conc"], list) else [row["conc"]]: - points[fingerprint, conc] = {**row, "conc": conc, "recipe-fingerprint": fingerprint} + points[identity, conc] = {**row, "conc": conc, "recipe-fingerprint": fingerprint} return points, errors def head_matrix(config_files: list[str], runner_config: str) -> Matrix: + """Generate the working tree's matrix for each config key, as points and fingerprint errors.""" master = load_config_files(config_files) runners = load_runner_file(runner_config) root = config_root(config_files) @@ -47,6 +54,7 @@ def head_matrix(config_files: list[str], runner_config: str) -> Matrix: def base_matrix(ref: str, config_files: list[str]) -> Matrix: + """Generate the matrix of each config key at git ``ref``, as points and fingerprint errors.""" with snapshot(ref) as revision: configs = Path(revision.runner_config).parent files = [configs / Path(path).name for path in config_files] @@ -57,11 +65,10 @@ def base_matrix(ref: str, config_files: list[str]) -> Matrix: single.write_text(yaml.safe_dump({key: entry}, sort_keys=False)) singles.append(str(single)) - def generate(key: str, single: str) -> list[dict]: - return revision.generate([key], ["--no-evals"], [single]) - - with ThreadPoolExecutor(os.cpu_count()) as pool: - rows = list(pool.map(generate, master, singles)) + requests = [ + ([key], ["--no-evals"], [single]) for key, single in zip(master, singles, strict=True) + ] + rows = revision.generate_each(requests, os.cpu_count() or 1) return { key: _points(generated, revision.root) for key, generated in zip(master, rows, strict=True) @@ -69,19 +76,22 @@ def generate(key: str, single: str) -> list[dict]: def _sorted(points: Iterable[dict]) -> list[dict]: + """Return the points ordered by experiment name, then concurrency.""" return sorted(points, key=lambda point: (point["exp-name"], point["conc"])) def _keys(base: Matrix, head: Matrix) -> list[str]: + """Return head's config keys followed by keys that exist only in base.""" return [*head, *(key for key in base if key not in head)] def compare(base: Matrix, head: Matrix) -> list[dict]: + """List the config keys whose points differ, with counts and the added and removed points.""" changed = [] for key in _keys(base, head): before, after = base.get(key, ({}, []))[0], head.get(key, ({}, []))[0] - added = _sorted(after[point] for point in after.keys() - before.keys()) - removed = _sorted(before[point] for point in before.keys() - after.keys()) + added = _sorted(point for point_id, point in after.items() if point_id not in before) + removed = _sorted(point for point_id, point in before.items() if point_id not in after) if added or removed: changed.append( { @@ -96,6 +106,7 @@ def compare(base: Matrix, head: Matrix) -> list[dict]: def _label(point: dict) -> str: + """Describe a point in one line by experiment, shape, conc, and short fingerprint.""" if "prefill" in point: shape = str(point.get("srt-recipe", "")).rpartition("/")[2] else: @@ -106,6 +117,7 @@ def _label(point: dict) -> str: def main(argv: list[str] | None = None) -> None: + """Report config keys whose points or recipes changed since ``--base``, as text or JSON.""" parser = argparse.ArgumentParser( description="List config keys whose benchmark points or recipes differ from a base." ) @@ -114,11 +126,33 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--runner-config", default=RUNNER_CONFIG) parser.add_argument("--json", action="store_true", help="print the report as JSON") args = parser.parse_args(argv) - head = head_matrix(args.config_files, args.runner_config) + head_outcome: dict[str, Any] = {} + + def build_head() -> None: + """Build the head matrix into ``head_outcome``, storing any error instead of raising. + + Runs in a daemon thread during base generation so an interrupt exits at once. + """ + try: + head_outcome["matrix"] = head_matrix(args.config_files, args.runner_config) + except BaseException as error: # noqa: BLE001 + head_outcome["error"] = error + + head_thread = threading.Thread(target=build_head, daemon=True) + head_thread.start() try: base = base_matrix(args.base, args.config_files) - except subprocess.CalledProcessError as error: - parser.exit(1, f"{args.base} could not generate its matrix:\n{error.stderr}") + except Exception as error: + head_thread.join() + if "error" in head_outcome: + raise head_outcome["error"] from None + if isinstance(error, subprocess.CalledProcessError): + parser.exit(1, f"{args.base} could not generate its matrix:\n{error.stderr}") + raise + head_thread.join() + if "error" in head_outcome: + raise head_outcome["error"] + head = head_outcome["matrix"] changed = compare(base, head) keys = _keys(base, head) if args.json: diff --git a/inferencex-e2e/infx/matrix/fingerprint.py b/inferencex-e2e/infx/matrix/fingerprint.py index 41a60577da..432ddc3c13 100644 --- a/inferencex-e2e/infx/matrix/fingerprint.py +++ b/inferencex-e2e/infx/matrix/fingerprint.py @@ -24,19 +24,23 @@ def _digest(value: Any) -> str: + """Return the SHA-256 hex digest of ``value`` serialized as canonical JSON.""" canonical = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() def _row_identity(entry: Mapping[str, Any]) -> dict[str, Any]: + """Return the row without the keys that do not identify a benchmark point.""" return {key: value for key, value in entry.items() if key not in ROW_EXCLUDED} def row_fingerprint(entry: Mapping[str, Any]) -> str: + """Return the digest of a matrix row, ignoring conc, experiment name, and fingerprint.""" return _digest(_row_identity(entry)) def _without(block: dict[str, Any], path: tuple[str, ...]) -> dict[str, Any]: + """Return a copy of ``block`` without the nested key at ``path``, or ``block`` if absent.""" key, *rest = path if key not in block: return block @@ -48,6 +52,7 @@ def _without(block: dict[str, Any], path: tuple[str, ...]) -> dict[str, Any]: def _concrete_recipes(entry: Mapping[str, Any], root: Path) -> list[dict[str, Any]]: + """Render the row's bound srt recipe, minus the fields that vary per point or run.""" power_port = EXPORTER_PORT if entry.get("power") else None _, recipe = bound_variant( entry, point_environment(entry), root, expand=expand_variants, power_port=power_port @@ -58,6 +63,10 @@ def _concrete_recipes(entry: Mapping[str, Any], root: Path) -> list[dict[str, An def recipe_fingerprint(entry: Mapping[str, Any], root: Path) -> str: + """Return the digest of a row and its rendered srt recipe, or the row digest if it has none. + + Raise ValueError naming the recipe and point when the recipe cannot be rendered. + """ if not entry.get("srt-recipe"): return row_fingerprint(entry) try: @@ -69,6 +78,7 @@ def recipe_fingerprint(entry: Mapping[str, Any], root: Path) -> str: def main() -> None: + """Read matrix rows as JSON from stdin and print their recipe fingerprints as a JSON list.""" root = repository_root() print(json.dumps([recipe_fingerprint(row, root) for row in json.load(sys.stdin)])) diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index c7bed03f91..a47e8922fd 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -10,8 +10,7 @@ from pathlib import Path from typing import Any, Literal -import yaml - +from infx import yamlio from infx.clusters import CLUSTER_LABEL_PREFIX from infx.clusters.slurm import Fabric from infx.srt_slurm.variants import deep_merge, expand_variants @@ -44,10 +43,12 @@ def seq_len_to_str(isl: int, osl: int) -> str: + """Return the short name of an ISL/OSL pair, such as 1k1k, or ``{isl}_{osl}``.""" return seq_len_itos.get((isl, osl), f"{isl}_{osl}") def freeze_config_value(value: Any) -> Any: + """Return ``value`` with dicts and lists turned into sorted tuples so it can be hashed.""" if isinstance(value, dict): return tuple(sorted((key, freeze_config_value(item)) for key, item in value.items())) if isinstance(value, list): @@ -56,9 +57,15 @@ def freeze_config_value(value: Any) -> Any: def trim_conc(entries: list[dict]) -> list[dict]: + """Keep only the lowest-concurrency row of each otherwise identical group of rows. + + Multinode rows shrink to their lowest conc, and the kept row inherits the group's eval flags. + The recipe fingerprint is ignored since single-node bundles bind one variant per conc. + """ ignored_fields = { "conc", "exp-name", + Fields.RECIPE_FINGERPRINT.value, "run-eval", "eval-only", "eval-conc", @@ -70,6 +77,7 @@ def trim_conc(entries: list[dict]) -> list[dict]: out: list[dict] = [] def minimum_concurrency(entry: dict) -> int: + """Return the lowest concurrency of a row whose conc is an int or a list.""" conc = entry["conc"] return min(conc) if isinstance(conc, list) else conc @@ -105,8 +113,13 @@ def minimum_concurrency(entry: dict) -> int: for indices in groups.values(): keep = min(indices, key=lambda index: minimum_concurrency(out[index])) kept_entry = out[keep] - if any(out[index].get("run-eval") is True for index in indices): + eval_rows = [out[index] for index in indices if out[index].get("run-eval") is True] + if eval_rows: kept_entry = {**kept_entry, "run-eval": True} + if not kept_entry.get(Fields.EVAL_FRAMEWORK.value): + for field in (Fields.EVAL_FRAMEWORK.value, Fields.EVAL_SUITE.value): + if field in eval_rows[0]: + kept_entry[field] = eval_rows[0][field] if kept_entry.get("prefill") is not None: kept_entry["eval-conc"] = minimum_concurrency(kept_entry) if any(out[index].get("eval-all-concs") is True for index in indices): @@ -117,7 +130,10 @@ def minimum_concurrency(entry: dict) -> int: def smoke_entries(entries: list[dict]) -> list[dict]: - benchmarks = trim_conc([{**row, "run-eval": False} for row in entries]) + """Return trimmed benchmark rows with evals off, followed by trimmed eval-only rows.""" + benchmarks = trim_conc( + [{**row, "run-eval": False} for row in entries if not row.get(Fields.EVAL_ONLY.value)] + ) evals = [] for row in entries: if not row.get("run-eval"): @@ -132,10 +148,12 @@ def smoke_entries(entries: list[dict]) -> list[dict]: def runner_labels(runner_data: dict) -> dict: + """Return the runner config's map of runner labels to node names.""" return runner_data["labels"] def runner_hardware(runner_data: dict) -> dict: + """Return GPU and CPU DRAM facts keyed by cluster label, for clusters that declare them.""" node_fields = (Fields.GPUS_PER_NODE.value, Fields.AVAILABLE_CPU_DRAM_MIB.value) return { f"{CLUSTER_LABEL_PREFIX}{cluster_id}": { @@ -146,10 +164,12 @@ def runner_hardware(runner_data: dict) -> dict: def runner_nodes_for_label(runner: str, runner_data: dict) -> list[str]: + """Return the node names of runner label ``runner``, or an empty list.""" return runner_labels(runner_data).get(runner, []) def runner_hardware_int(runner: str, runner_data: dict, field: str) -> int: + """Return the integer hardware ``field`` of ``runner``, or raise ValueError if it is unset.""" hardware = runner_hardware(runner_data).get(runner, {}) value = hardware.get(field) if value is None: @@ -162,18 +182,25 @@ def runner_hardware_int(runner: str, runner_data: dict, field: str) -> int: def runner_available_cpu_dram_mib(runner: str, runner_data: dict) -> int: + """Return the CPU DRAM in MiB available to ``runner``.""" return runner_hardware_int(runner, runner_data, Fields.AVAILABLE_CPU_DRAM_MIB.value) def runner_gpus_per_node(runner: str, runner_data: dict) -> int: + """Return the GPUs per node of ``runner``.""" return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) def _hardware_family(label: str) -> str: + """Return the hardware family of a label, such as b200 for cluster:b200-dgxc.""" return label.removeprefix("cluster:").split("-", 1)[0] def scheduling_gpus_per_node(label: str, runner_data: dict) -> int: + """Return the GPUs per node of ``label``, or of its hardware family when that is unambiguous. + + Raise ValueError when no cluster or more than one GPU count matches. + """ hardware = runner_hardware(runner_data) exact = hardware.get(label) if exact is not None: @@ -193,6 +220,10 @@ def scheduling_gpus_per_node(label: str, runner_data: dict) -> int: def _worker_node_override(worker: dict, setting_name: str) -> int | None: + """Return the positive ``setting_name=N`` value in a worker's additional settings, or None. + + Raise ValueError when the settings disagree or are not positive. + """ pattern = re.compile(rf"^{re.escape(setting_name)}=(\d+)$") values = [] for setting in worker.get(Fields.ADDITIONAL_SETTINGS.value, []) or []: @@ -207,13 +238,14 @@ def _worker_node_override(worker: dict, setting_name: str) -> int | None: def recipe_auxiliary_node_count(recipe: dict) -> int: - services = [service for service in recipe.get("services", []) if service.get("enabled", True)] + """Return the nodes a recipe needs beyond its workers, for service pools and dedicated roles.""" + services = [service for service in recipe.get("services") or [] if service.get("enabled", True)] pool_nodes = sum(service.get("nodes") or 0 for service in services) - infra_dedicated = recipe.get("infra", {}).get("etcd_nats_dedicated_node", False) + infra_dedicated = (recipe.get("infra") or {}).get("etcd_nats_dedicated_node", False) discovery = [service for service in services if service.get("type") in ("etcd", "nats")] if discovery: placements = { - service.get("placement", {}).get("node") == "dedicated" for service in discovery + (service.get("placement") or {}).get("node") == "dedicated" for service in discovery } if len(placements) != 1: raise ValueError("Recipe etcd and nats must agree on dedicated placement") @@ -223,34 +255,41 @@ def recipe_auxiliary_node_count(recipe: dict) -> int: ("frontend", "dedicated_node"), ("benchmark", "client_dedicated_node"), ): - settings = recipe.get(section, {}) + settings = recipe.get(section) or {} dedicated_roles += int( - settings.get("placement", {}).get("node") == "dedicated" + (settings.get("placement") or {}).get("node") == "dedicated" or settings.get(legacy_key, False) ) - if dedicated_roles and recipe.get("benchmark", {}).get("colocate_with_frontend", True): + if dedicated_roles and (recipe.get("benchmark") or {}).get("colocate_with_frontend", True): dedicated_roles = 1 return pool_nodes + dedicated_roles def recipe_node_count(root: Path, srt_recipe: str) -> int | None: + """Return the total node count of a schema 2 srt recipe, or None for an unknown variant. + + Raise ValueError when the recipe is not schema 2 or its roles lack node counts. + """ path, _, selector = srt_recipe.partition(":") recipe_path = root / path - recipe = yaml.safe_load(recipe_path.read_text()) - if "base" in recipe: - if not (selector == "base" or selector.startswith("override_")): + recipe = yamlio.safe_load(recipe_path.read_text()) + if isinstance(recipe, dict) and "base" in recipe: + if selector.startswith("zip_override_"): + [(_, recipe)] = expand_variants(recipe, selector) + elif selector == "base" or selector.startswith("override_"): + schema = recipe.get("schema") + recipe = deep_merge(recipe["base"], recipe[selector] or {}) + recipe.setdefault("schema", schema) + else: return None - schema = recipe.get("schema") - recipe = deep_merge(recipe["base"], recipe[selector] or {}) - recipe.setdefault("schema", schema) if recipe.get("schema") != 2: return None - if recipe.get("schema") != 2: + if not isinstance(recipe, dict) or recipe.get("schema") != 2: raise ValueError(f"srt-slurm recipes must declare schema: 2: {recipe_path}") roles = recipe.get("roles") if roles: for name, role in roles.items(): - if "nodes" not in role: + if not isinstance(role, dict) or role.get("nodes") is None: raise ValueError(f"Recipe role {name!r} must specify nodes: {recipe_path}") worker_nodes = sum( 0 if role["nodes"] == "colocate" else int(role["nodes"]) for role in roles.values() @@ -260,6 +299,7 @@ def recipe_node_count(root: Path, srt_recipe: str) -> int | None: def _label_fabrics(label: str, runner_data: dict) -> dict[str, Fabric]: + """Return the srt-slurm fabric of each cluster that runs ``label``, keyed by cluster ID.""" labels = runner_labels(runner_data) runners = set(labels.get(label, [label])) fabrics = {} @@ -273,8 +313,9 @@ def _label_fabrics(label: str, runner_data: dict) -> dict[str, Fabric]: def check_fabric_references(root: Path, srt_recipe: str, label: str, runner_data: dict) -> None: + """Raise ValueError if a recipe variant references a fabric field ``label``'s clusters lack.""" path, _, selector = srt_recipe.partition(":") - variants = expand_variants(yaml.safe_load((root / path).read_text()), selector or None) + variants = expand_variants(yamlio.safe_load((root / path).read_text()), selector or None) fabrics = { f"cluster {cluster_id!r}": fabric.rendered() for cluster_id, fabric in _label_fabrics(label, runner_data).items() @@ -294,6 +335,7 @@ def worker_node_count( runner: str, runner_data: dict, ) -> int: + """Return the nodes a worker role needs, from a ``_NODES`` setting or its GPU count.""" override = _worker_node_override(worker, f"{role.upper()}_NODES") if override is not None: return override @@ -318,6 +360,7 @@ def multinode_node_count( srt_recipe: str | None, root: Path | None, ) -> int: + """Return a multinode entry's node count from its srt recipe, else from its workers.""" if srt_recipe is not None and root is None: raise ValueError(f"Reading {srt_recipe} needs the master config's project root") recipe_count = recipe_node_count(root, srt_recipe) if srt_recipe is not None else None @@ -334,6 +377,10 @@ def add_multinode_node_count( num_nodes: int | None, root: Path | None, ) -> dict: + """Set the entry's node count from ``num_nodes`` or its workers, and return the entry. + + Raise ValueError when ``num_nodes`` is given for a disaggregated entry. + """ if not entry[Fields.DISAGG.value] and num_nodes is not None: entry[Fields.NODE_COUNT.value] = num_nodes elif num_nodes is not None: @@ -351,6 +398,7 @@ def add_multinode_node_count( def effective_gpu_count(benchmark: dict) -> int: + """Return the GPUs one benchmark worker uses, TP times PP times PCP.""" return ( benchmark[Fields.TP.value] * benchmark.get(Fields.PP.value, 1) @@ -359,6 +407,7 @@ def effective_gpu_count(benchmark: dict) -> int: def with_worker_parallelism_defaults(worker: dict) -> dict: + """Return a copy of the worker with PP, DCP, and PCP defaulting to 1.""" return { **worker, Fields.PP.value: worker.get(Fields.PP.value, 1), @@ -368,6 +417,7 @@ def with_worker_parallelism_defaults(worker: dict) -> dict: def multinode_worker_pair(benchmark: dict, disagg: bool) -> tuple[dict, dict]: + """Return the prefill and decode workers, splitting an aggregated worker into a pair.""" if disagg: return ( with_worker_parallelism_defaults(benchmark[Fields.PREFILL.value]), @@ -392,6 +442,10 @@ def multinode_worker_pair(benchmark: dict, disagg: bool) -> tuple[dict, dict]: def worker_gpus_per_node(worker: dict, gpus_per_node: int, role: str = "worker") -> int: + """Return how many of a worker's GPUs land on each node. + + Raise ValueError when its replica size and ``gpus_per_node`` do not divide one another. + """ gpus_per_replica = ( worker[Fields.TP.value] * worker.get(Fields.PP.value, 1) @@ -421,6 +475,10 @@ def worker_gpus_per_node(worker: dict, gpus_per_node: int, role: str = "worker") def agentic_dram_offload_gb( agentic_config: dict, benchmark: dict, runner: str, runner_data: dict ) -> tuple[int, int]: + """Return the CPU DRAM offload in GB for the prefill and decode sides of an agentic point. + + Both are 0 without DRAM offload. Raise ValueError when the workers do not fit the nodes. + """ kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none") if kv_offloading != "dram": return 0, 0 @@ -433,6 +491,7 @@ def agentic_dram_offload_gb( gpus_per_node = runner_gpus_per_node(runner, runner_data) def share_gb(gpu_count: int) -> int: + """Return the GB of offload DRAM that ``gpu_count`` GPUs get as their share of a node.""" proportional_bytes = ( Decimal(available_mib) * BYTES_PER_MIB * utilization * gpu_count / gpus_per_node ) @@ -463,6 +522,7 @@ def agentic_kv_offload_suffix( kv_offloading: str, kv_offload_backend: dict | None, ) -> str: + """Return the experiment-name suffix for a KV offload mode and its backend.""" if kv_offloading == "none": return "kvnone" return f"kv{kv_offloading}-{kv_offload_backend['name']}" @@ -475,8 +535,10 @@ def multinode_agentic_exp_name( conc_batch: list[int], offload_suffix: str, ) -> str: + """Return a multinode agentic experiment name from the model, worker shapes, and concs.""" def _worker_tag(worker: dict, role_prefix: str) -> str: + """Return a short tag of a worker's count, TP, EP, DP attention, and DWDP size.""" ep = worker.get(Fields.EP.value, 1) dpa = worker.get(Fields.DP_ATTN.value, False) tag = f"{role_prefix}{worker[Fields.NUM_WORKER.value]}x{worker[Fields.TP.value]}" @@ -497,6 +559,7 @@ def _worker_tag(worker: dict, role_prefix: str) -> str: def component_metadata(benchmark: dict, config: dict) -> dict: + """Return the router and KV transfer settings of a benchmark, falling back to its config.""" metadata = {} for field in (Fields.ROUTER, Fields.KV_P2P_TRANSFER): value = benchmark.get(field.value, config.get(field.value)) @@ -506,6 +569,7 @@ def component_metadata(benchmark: dict, config: dict) -> dict: def srt_recipe_fields(config: dict, benchmark: dict) -> dict: + """Return the benchmark's srt recipe fields resolved to paths under the config's recipe dir.""" return { field.value: srt_recipe_path( config.get(Fields.MULTINODE.value, False), @@ -518,6 +582,7 @@ def srt_recipe_fields(config: dict, benchmark: dict) -> dict: def _multinode_parallelism_key(entry: dict) -> tuple: + """Return a hashable identity of a multinode row, ignoring conc, eval, and naming fields.""" ignored_fields = { Fields.CONC.value, Fields.RUN_EVAL.value, @@ -538,12 +603,14 @@ def _multinode_parallelism_key(entry: dict) -> tuple: def automatic_agentic_vendor_eval(entry: dict) -> tuple[str, str] | None: + """Return the vendor eval framework and suite run automatically for an agentic row, or None.""" if entry.get(Fields.SCENARIO_TYPE.value) != "agentic-coding": return None return AUTOMATIC_AGENTIC_VENDOR_EVALS.get(entry.get(Fields.MODEL_PREFIX.value)) def app_eval_identity(entry: dict, conc: int) -> tuple: + """Return the identity of the app eval a row runs at ``conc``, to detect duplicate evals.""" if Fields.PREFILL.value in entry: topology = tuple( tuple( @@ -577,6 +644,7 @@ def app_eval_identity(entry: dict, conc: int) -> tuple: def _eval_concs(entry: dict) -> list[int]: + """Return the concurrencies a row would evaluate at.""" conc = entry[Fields.CONC.value] conc_values = conc if isinstance(conc, list) else [conc] if Fields.PREFILL.value in entry and not entry.get(Fields.EVAL_ALL_CONCS.value): @@ -585,6 +653,10 @@ def _eval_concs(entry: dict) -> list[int]: def drop_app_colliding_evals(rows: list[dict]) -> list[dict]: + """Remove eval concurrencies already claimed by another row and return the rows. + + Rows without KV offload claim first. A row left with none stops running evals. + """ claimed: set[tuple] = set() candidates = sorted( (row for row in rows if row.get(Fields.RUN_EVAL.value)), @@ -611,6 +683,11 @@ def drop_app_colliding_evals(rows: list[dict]) -> list[dict]: def mark_eval_entries(matrix_values: list[dict]) -> list[dict]: + """Choose which rows run evals and return the rows, plus extra gsm8k eval-only rows. + + Pick the top and median 8k1k concs per single-node group, the top eligible conc per multinode + shape, one gsm8k point per agentic deployment, and every automatic vendor eval. + """ from collections import defaultdict target_isl, target_osl = seq_len_stoi["8k1k"] @@ -618,6 +695,7 @@ def mark_eval_entries(matrix_values: list[dict]) -> list[dict]: mn_eval_conc = {} def _eligible_eval_concs(entry: dict) -> list[int]: + """Return a row's concurrencies of at least MIN_EVAL_CONC, sorted.""" conc = entry[Fields.CONC.value] conc_values = conc if isinstance(conc, list) else [conc] return sorted(c for c in conc_values if c >= MIN_EVAL_CONC) @@ -762,6 +840,7 @@ def _eligible_eval_concs(entry: dict) -> list[int]: def _gsm8k_eval_only_row(entry: dict, eval_conc: int | None = None) -> dict: + """Return a gsm8k eval-only copy of ``entry``, at ``eval_conc`` when given.""" row = { **entry, Fields.RUN_EVAL.value: True, @@ -776,6 +855,10 @@ def _gsm8k_eval_only_row(entry: dict, eval_conc: int | None = None) -> dict: def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: + """Mark every eligible row to run evals and return the rows, batching multinode concs. + + Agentic vendor evals become separate eval-only rows. + """ expanded_entries: list[dict] = [] multinode_indices: dict[tuple, int] = {} multinode_agentic_indices: dict[tuple, int] = {} @@ -791,6 +874,7 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: eval_entry = { **entry, Fields.RUN_EVAL.value: True, + Fields.EVAL_ONLY.value: True, Fields.EVAL_FRAMEWORK.value: eval_framework, Fields.EVAL_SUITE.value: eval_suite, } @@ -876,6 +960,7 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: def _concurrency_range(start: int, end: int, step: int) -> list[int]: + """Return ``start``, ``start * step``, and so on, capped so the last value is ``end``.""" values = [] while start <= end: values.append(start) @@ -894,6 +979,10 @@ def _fixed_sequence_entries( runner_data: dict, root: Path | None, ) -> list[dict]: + """Return validated fixed-sequence rows for one benchmark, per runner and concurrency. + + Multinode rows carry all concs in one row. Raise ValueError for an invalid power request. + """ is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) isl, osl = sequence[Fields.ISL.value], sequence[Fields.OSL.value] @@ -979,6 +1068,7 @@ def _agentic_entries( conc_filter: list[int] | None = None, root: Path | None = None, ) -> list[dict]: + """Return validated agentic rows for one benchmark at each runner and allowed concurrency.""" is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) model_code = config[Fields.MODEL_PREFIX.value] @@ -1115,6 +1205,7 @@ def generate_full_sweep( runner_data: dict, root: Path | None = None, ) -> list[dict]: + """Return the full-sweep rows selected by the parsed command-line ``args``.""" return expand_full_sweep( all_config_data, runner_data, @@ -1145,6 +1236,10 @@ def expand_full_sweep( options: FullSweepOptions = FullSweepOptions(), root: Path | None = None, ) -> list[dict]: + """Return rows for every master config that passes the full-sweep filters in ``options``. + + Raise ValueError for invalid options or filters that match no config. + """ if options.step_size <= 1: raise ValueError("step_size must be greater than 1") if ( @@ -1161,12 +1256,25 @@ def expand_full_sweep( f"Invalid runner type(s): {invalid_runners}. " f"Valid runner types are: {', '.join(sorted(valid_runner_types))}" ) + precisions = {config[Fields.PRECISION.value] for config in master_config.values()} + frameworks = {config[Fields.FRAMEWORK.value] for config in master_config.values()} + for label, requested, exists in ( + ( + "model prefix", + options.model_prefix, + lambda p: any(k.startswith(f"{p}-") for k in master_config), + ), + ("precision", options.precision, precisions.__contains__), + ("framework", options.framework, frameworks.__contains__), + ): + if missing := [value for value in requested or () if not exists(value)]: + raise ValueError(f"No config has {label}: {', '.join(missing)}") seq_filter = {seq_len_stoi[sl] for sl in options.seq_lens} if options.seq_lens else None configs = ( config for key, config in master_config.items() - if (not options.model_prefix or any(key.startswith(p) for p in options.model_prefix)) + if (not options.model_prefix or any(key.startswith(f"{p}-") for p in options.model_prefix)) and (not options.precision or config[Fields.PRECISION.value] in options.precision) and (not options.framework or config[Fields.FRAMEWORK.value] in options.framework) and (not options.runner_type or config[Fields.RUNNER.value] in options.runner_type) @@ -1189,6 +1297,10 @@ def _fixed_sequence_concurrencies( full_sweep: FullSweepOptions | None, concurrencies: list[int] | None, ) -> list[int] | None: + """Return a fixed-sequence benchmark's concurrencies after the sweep or ``--conc`` filters. + + Return None when the filters leave nothing to run. + """ if full_sweep is None: values = ( benchmark[Fields.CONC_LIST.value] @@ -1238,6 +1350,7 @@ def _runner_values_for_filter( *, include_label: bool = True, ) -> list[str]: + """Return the runner label, or its nodes and itself, that match ``runner_node_filter``.""" if not runner_node_filter: return [runner] candidates = runner_nodes_for_label(runner, runner_data) @@ -1254,6 +1367,7 @@ def generate_test_config_sweep( runner_data: dict | None = None, root: Path | None = None, ) -> list[dict]: + """Return the rows of the config keys named on the ``test-config`` command line.""" return _expand_selected_configs( args.config_keys, all_config_data, @@ -1277,8 +1391,12 @@ def _expand_selected_configs( concurrencies: list[int] | None = None, root: Path | None = None, ) -> list[dict]: + """Return rows for the selected config keys, with the given filters applied. + + Raise ValueError when a requested concurrency is generated by no row. + """ resolved_keys = expand_config_keys(config_keys, all_config_data.keys()) - return _expand_configs( + rows = _expand_configs( (all_config_data[key] for key in resolved_keys), runner_data or {}, runner_node_filter=runner_node_filter, @@ -1287,6 +1405,17 @@ def _expand_selected_configs( concurrencies=concurrencies, root=root, ) + if concurrencies: + generated = set() + for row in rows: + conc = row[Fields.CONC.value] + generated.update(conc if isinstance(conc, list) else [conc]) + if missing := [conc for conc in concurrencies if conc not in generated]: + raise ValueError( + "--conc value(s) not generated by the selected configs: " + + ", ".join(map(str, missing)) + ) + return rows def _expand_configs( @@ -1301,6 +1430,10 @@ def _expand_configs( full_sweep: FullSweepOptions | None = None, root: Path | None = None, ) -> list[dict]: + """Return the fixed-sequence and agentic rows of each config that pass the filters. + + With ``root``, raise ValueError when a selected recipe references an unknown fabric field. + """ rows = [] for config in configs: multinode = config.get(Fields.MULTINODE.value, False) @@ -1400,6 +1533,10 @@ def _expand_configs( def expand_config_keys(config_keys: Iterable[str], available_keys: Iterable[str]) -> list[str]: + """Return the config keys, with glob patterns expanded, in order and without duplicates. + + Raise ValueError for a key or pattern that matches nothing. + """ available = list(available_keys) seen = {} for key in config_keys: @@ -1423,6 +1560,7 @@ def expand_config_keys(config_keys: Iterable[str], available_keys: Iterable[str] def filter_exp_names(entries: list[dict], exp_names: list[str]) -> list[dict]: + """Return the rows named in ``exp_names``, raising ValueError unless each matches one row.""" requested = set(exp_names) if len(requested) != len(exp_names): raise ValueError("--exp-names contains duplicate values") @@ -1443,6 +1581,7 @@ def filter_exp_names(entries: list[dict], exp_names: list[str]) -> list[dict]: def apply_node_type_defaults(args: argparse.Namespace) -> argparse.Namespace: + """Enable both node types in ``args`` when neither is set, and return ``args``.""" if ( hasattr(args, "single_node") and hasattr(args, "multi_node") @@ -1463,6 +1602,10 @@ def select_matrix_evals( mode: EvalMode = "default", trim: bool = False, ) -> list[dict]: + """Apply eval selection ``mode`` to rows, optionally trimming to minimum concurrency. + + Raise ValueError for an unknown mode or smoke combined with trimming. + """ if mode not in ("default", "none", "subset", "all", "smoke"): raise ValueError(f"Unknown eval mode: {mode!r}") if mode == "smoke" and trim: @@ -1474,7 +1617,11 @@ def select_matrix_evals( if mode == "smoke": return smoke_entries(rows) if trim: - rows = trim_conc(rows) + rows = [ + row + for row in drop_app_colliding_evals(trim_conc(rows)) + if row.get(Fields.RUN_EVAL.value) or not row.get(Fields.EVAL_ONLY.value) + ] if mode in ("subset", "all"): rows = [row for row in rows if row.get(Fields.RUN_EVAL.value, False)] for row in rows: @@ -1492,6 +1639,7 @@ def generate_config_matrix( eval_mode: EvalMode = "default", root: Path | None = None, ) -> list[dict]: + """Return the JSON-ready rows of the config keys with ``eval_mode`` eval selection applied.""" rows = _expand_selected_configs( config_keys, master_config, @@ -1507,9 +1655,10 @@ def with_fingerprints(rows: list[dict], root: Path) -> list[dict]: """``rows`` with the ``recipe-fingerprint`` changelog plans give the same throughput rows. Eval selection copies it to the rows it derives. A row whose recipe cannot be bound gets - none and a warning; its job fails when it binds the recipe. + none and a warning, and its job fails when it binds the recipe. The fingerprint module is + imported here because it imports this module through srt_slurm. """ - from infx.matrix.fingerprint import recipe_fingerprint # imports this module via srt_slurm + from infx.matrix.fingerprint import recipe_fingerprint fingerprinted = [] for row in rows: @@ -1524,6 +1673,7 @@ def with_fingerprints(rows: list[dict], root: Path) -> list[dict]: def main() -> list[dict]: + """Parse the command line, print the generated matrix rows as JSON, and return them.""" parent_parser = argparse.ArgumentParser(add_help=False) parent_parser.add_argument( "--config-files", @@ -1551,8 +1701,8 @@ def main() -> list[dict]: "--all-evals", action="store_true", help=( - "Expand eval selection to every generated fixed-sequence config. " - "Can be combined with --evals-only; used alone, it also emits eval-only jobs." + "Expand eval selection to every eligible 8k1k fixed-sequence and agentic config " + "and emit only those eval-only jobs, with or without --evals-only." ), ) parent_parser.add_argument( @@ -1634,25 +1784,31 @@ def main() -> list[dict]: "--min-conc", type=int, required=False, - help="Minimum concurrency value to include (filters out lower concurrency values)", + help=( + "Drop concurrencies below this value. A single-node fixed-sequence " + "conc-start/conc-end range that begins lower starts at this value instead." + ), ) full_sweep_parser.add_argument( "--max-conc", type=int, required=False, - help="Maximum concurrency value to include (filters out higher concurrency values)", + help=( + "Drop concurrencies above this value. A single-node fixed-sequence range that " + "ends higher ends at it, and a fixed-sequence config with nothing left runs at it." + ), ) full_sweep_parser.add_argument( "--max-tp", type=int, required=False, - help="Maximum tensor parallelism value to include (single-node only)", + help="Drop single-node fixed-sequence configs whose TP exceeds this value", ) full_sweep_parser.add_argument( "--max-ep", type=int, required=False, - help="Maximum expert parallelism value to include (single-node only)", + help="Cap the EP of single-node fixed-sequence configs at this value", ) full_sweep_parser.add_argument( "--single-node", diff --git a/inferencex-e2e/infx/matrix/plan.py b/inferencex-e2e/infx/matrix/plan.py index dda30fd68e..35551490fb 100644 --- a/inferencex-e2e/infx/matrix/plan.py +++ b/inferencex-e2e/infx/matrix/plan.py @@ -6,8 +6,7 @@ from collections import defaultdict from contextlib import ExitStack -import yaml - +from infx import yamlio from infx.config import MASTER_CONFIGS, RUNNER_CONFIG, git_path_at_ref from .fingerprint import recipe_fingerprint @@ -16,7 +15,7 @@ drop_app_colliding_evals, freeze_config_value, generate_config_matrix, - seq_len_to_str, + seq_len_itos, trim_conc, ) from .revision import GENERATOR, snapshot @@ -32,6 +31,10 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: + """Return the lines ``filepath`` gained between two git refs, joined by newlines. + + Raise RuntimeError when git diff fails and ValueError when any non-blank line was deleted. + """ result = subprocess.run( [ "git", @@ -43,10 +46,17 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: capture_output=True, text=True, ) + if result.returncode != 0: + raise RuntimeError(f"git diff of {filepath} failed: {result.stderr.strip()}") added_lines = [] + in_hunk = False for line in result.stdout.split("\n"): - if line.startswith("-") and not line.startswith("---"): + if line.startswith("@@"): + in_hunk = True + elif not in_hunk: + continue + elif line.startswith("-"): deleted_content = line[1:] if deleted_content.strip(): raise ValueError( @@ -54,13 +64,14 @@ def get_added_lines(base_ref: str, head_ref: str, filepath: str) -> str: f"Only additions to the changelog are permitted. " f"Found deleted line: {deleted_content}" ) - elif line.startswith("+") and not line.startswith("+++"): + elif line.startswith("+"): added_lines.append(line[1:]) return "\n".join(added_lines) def filter_eval_rows_by_prefill_ep(eval_rows: list[dict], min_prefill_ep: int | None) -> list[dict]: + """Keep rows with no prefill block or a prefill EP of at least ``min_prefill_ep`` if set.""" if min_prefill_ep is None: return eval_rows kept: list[dict] = [] @@ -78,6 +89,10 @@ def filter_eval_rows_by_prefill_ep(eval_rows: list[dict], min_prefill_ep: int | def get_config_keys_from_master(config_keys: list[str], master_config: dict) -> list[str]: + """Expand ``*`` wildcards against the master config and return the matching keys in order. + + Raise ValueError for a key or pattern that matches nothing. + """ resolved_keys = {} for key in config_keys: if "*" in key: @@ -97,6 +112,7 @@ def get_config_keys_from_master(config_keys: list[str], master_config: dict) -> def _matrix_curve_key(entry: dict) -> tuple: + """Return a hashable identity of a matrix row's curve, ignoring its conc and naming fields.""" return tuple( sorted( (key, freeze_config_value(value)) @@ -107,6 +123,7 @@ def _matrix_curve_key(entry: dict) -> tuple: def _matrix_visual_series_key(entry: dict) -> tuple: + """Return the fields that place a matrix row on one visual curve of the dashboard.""" is_agentic = entry.get("scenario-type") == "agentic-coding" kv_offloading = entry.get("kv-offloading", "none") offload_mode = "off" if kv_offloading in (None, "", "none") else "on" @@ -130,6 +147,7 @@ def _matrix_visual_series_key(entry: dict) -> tuple: def _matrix_concurrencies(entry: dict) -> tuple[int, ...]: + """Return a row's concurrencies as a tuple, or raise ValueError if they are not ints.""" conc = entry.get("conc") if isinstance(conc, int): return (conc,) @@ -139,6 +157,11 @@ def _matrix_concurrencies(entry: dict) -> tuple[int, ...]: def append_only_delta(base_entries: list[dict], head_entries: list[dict]) -> list[dict]: + """Return the head rows trimmed to the concurrency points the base revision lacks. + + Raise ValueError when head removes base curves or points, adds nothing, or adds points to a + curve without exactly one unchanged image. + """ base_groups: dict[tuple, set[int]] = defaultdict(set) head_groups: dict[tuple, set[int]] = defaultdict(set) for entry in base_entries: @@ -204,6 +227,7 @@ def validate_append_only_scope( head_master: dict, selected_config_scenarios: dict[str, set[str]], ) -> None: + """Raise ValueError if append-only edits touch unselected configs or scenarios.""" selected_configs = selected_config_scenarios.keys() all_keys = base_master.keys() | head_master.keys() unrelated_changes = [ @@ -249,6 +273,7 @@ def group_unseen_scenarios( scenarios: tuple[str, ...], seen: dict[str, set[str]], ) -> dict[tuple[str, ...], list[str]]: + """Group configs by the requested scenarios not yet in ``seen``, marking them as seen.""" groups: dict[tuple[str, ...], list[str]] = defaultdict(list) for config in config_keys: unseen = tuple( @@ -264,6 +289,7 @@ def group_unseen_scenarios( class MatrixGenerationError(ValueError): def __init__(self, command: list[str], cause: Exception) -> None: + """Wrap the generation error ``cause`` with the equivalent generator ``command``.""" super().__init__(str(cause)) self.command, self.cause = command, cause @@ -279,6 +305,11 @@ def build_plan( all_evals: bool = False, evals_only: bool = False, ) -> ChangelogMatrixEntry: + """Build the sweep matrix for changelog entries added between ``base_ref`` and ``head_ref``. + + Raise ValueError for invalid entry combinations, append-only violations, and fixed-sequence + points with no sweep job in run-sweep.yml, and MatrixGenerationError when generation fails. + """ if not changelog_data: raise ValueError("No valid YAML entries found in the changelog additions.") @@ -323,6 +354,7 @@ def generate_current( mode: EvalMode, scenarios: tuple[str, ...] | None, ) -> list[dict]: + """Generate the working tree's rows for ``keys``, or raise MatrixGenerationError.""" nonlocal runner_data try: if runner_data is None: @@ -425,11 +457,16 @@ def generate_current( for result in all_benchmark_results: result["recipe-fingerprint"] = recipe_fingerprint(result, root) node_type = "multi_node" if result.get("prefill") is not None else "single_node" - scenario = ( - "agentic" - if result.get("scenario-type") == "agentic-coding" - else seq_len_to_str(result["isl"], result["osl"]) - ) + if result.get("scenario-type") == "agentic-coding": + scenario = "agentic" + else: + scenario = seq_len_itos.get((result["isl"], result["osl"])) + if scenario is None: + raise ValueError( + f"{result['exp-name']}: fixed-sequence isl={result['isl']} " + f"osl={result['osl']} has no sweep job; supported: " + f"{', '.join(seq_len_itos.values())}" + ) final_results[node_type][scenario].append(result) all_eval_results = [ @@ -446,6 +483,7 @@ def generate_current( def main() -> None: + """Print the sweep matrix, as JSON, for the changelog lines added between two refs.""" parser = argparse.ArgumentParser() parser.add_argument("--base-ref", type=str, required=True) parser.add_argument("--head-ref", type=str, required=True) @@ -468,7 +506,7 @@ def main() -> None: if not added_yaml.strip(): raise ValueError("No additions found in the changelog file.") - changelog_data = yaml.safe_load(added_yaml) + changelog_data = yamlio.load(added_yaml, yamlio.UniqueKeySafeLoader) try: result = build_plan( @@ -487,6 +525,7 @@ def main() -> None: def _generation_flags(mode: EvalMode, scenarios: tuple[str, ...] | None) -> list[str]: + """Return the generator flags that select ``mode`` evals and the given scenario types.""" flags = ["--no-evals"] if mode == "none" else ["--evals-only"] if mode == "all": flags.append("--all-evals") diff --git a/inferencex-e2e/infx/matrix/revision.py b/inferencex-e2e/infx/matrix/revision.py index 4fe98554ec..ee18fd1bb4 100644 --- a/inferencex-e2e/infx/matrix/revision.py +++ b/inferencex-e2e/infx/matrix/revision.py @@ -4,14 +4,20 @@ import io import json import os +import queue +import signal import subprocess import sys import tempfile +import threading from collections.abc import Iterator, Sequence -from contextlib import contextmanager +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, contextmanager, suppress from dataclasses import dataclass from pathlib import Path +from typing import Self +from infx.bench.proc import status from infx.config import MASTER_CONFIGS, git_repository_root, project_root @@ -22,6 +28,7 @@ class Tool: @property def module_path(self) -> str: + """The module's file path relative to the revision root.""" return self.module.replace(".", "/") + ".py" @@ -39,6 +46,37 @@ def module_path(self) -> str: "benchmarks/multi_node/srt-slurm-recipes", ) INHERITED_ENV = ("PATH", "HOME", "LANG", "TMPDIR") +_FORK_REPLY = "infx-fork-exit " +_FORK_SERVER = """ +import ast, importlib.util, json, os, runpy, sys, types + +module, reply = sys.argv[1], sys.argv[2] +sys.path[0] = os.getcwd() +spec = importlib.util.find_spec(module) +spec.loader.get_code(module) +for node in ast.parse(spec.loader.get_source(module)).body: + if isinstance(node, (ast.Import, ast.ImportFrom)) and getattr(node, "module", "") != "__future__": + try: + exec(compile(ast.Module([node], []), spec.origin, "exec"), + {"__name__": "__infx_preload__", "__package__": spec.parent}) + except BaseException: + pass +for line in sys.stdin: + arguments, output = json.loads(line) + pid = os.fork() + if pid == 0: + null = os.open(os.devnull, os.O_RDWR) + os.dup2(null, 0) + os.dup2(null, 2) + os.dup2(os.open(output, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 1) + sys.argv = [module, *arguments] + sys.modules["__main__"] = types.ModuleType("__main__") + runpy._run_module_as_main(module) + raise SystemExit + print(f"\\n{reply}{os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1])}", flush=True) +""" + +GenerateRequest = tuple[Sequence[str], Sequence[str], Sequence[str] | None] @dataclass(frozen=True) @@ -46,18 +84,22 @@ class Revision: root: Path def __post_init__(self) -> None: + """Resolve ``root`` to an absolute path.""" object.__setattr__(self, "root", Path(self.root).resolve()) @property def master_configs(self) -> list[str]: + """Paths of the revision's master config files.""" return [str(self._configs / Path(path).name) for path in MASTER_CONFIGS] @property def runner_config(self) -> str: + """Path of the revision's runner config.""" return str(self._configs / "runners.yaml") @property def _configs(self) -> Path: + """The revision's config directory, the first that holds runners.yaml.""" return next( ( self.root / name @@ -68,6 +110,10 @@ def _configs(self) -> Path: ) def invocation(self, tool: Tool, args: Sequence[str]) -> tuple[list[str], dict[str, str]]: + """Return the command and environment that run ``tool`` with ``args`` in this revision. + + Prefer the module layout, else the legacy script. Raise ValueError when neither exists. + """ script = self.root / tool.legacy_script if (self.root / tool.module_path).is_file(): entrypoint, imports = ["-m", tool.module], [self.root] @@ -78,36 +124,102 @@ def invocation(self, tool: Tool, args: Sequence[str]) -> tuple[list[str], dict[s return [sys.executable, *entrypoint, *args], self._environment(imports) def _environment(self, imports: Sequence[Path]) -> dict[str, str]: + """Return a minimal environment with ``imports`` on PYTHONPATH and the revision as root.""" env = {name: os.environ[name] for name in INHERITED_ENV if name in os.environ} env["PYTHONPATH"] = os.pathsep.join(map(str, imports)) env["INFERENCEX_REPOSITORY_ROOT"] = str(self.root) return env + def _generator_arguments( + self, config_keys: Sequence[str], flags: Sequence[str], config_files: Sequence[str] | None + ) -> list[str]: + """Return the generator's ``test-config`` arguments for these keys, flags, and configs.""" + return [ + "test-config", + "--config-keys", + *config_keys, + "--config-files", + *(self.master_configs if config_files is None else config_files), + "--runner-config", + self.runner_config, + *flags, + ] + def generate( self, config_keys: Sequence[str], flags: Sequence[str], config_files: Sequence[str] | None = None, ) -> list[dict]: + """Run the revision's generator for ``config_keys`` and return its rows. + + Raise CalledProcessError when the generator fails. + """ command, env = self.invocation( - GENERATOR, - [ - "test-config", - "--config-keys", - *config_keys, - "--config-files", - *(self.master_configs if config_files is None else config_files), - "--runner-config", - self.runner_config, - *flags, - ], + GENERATOR, self._generator_arguments(config_keys, flags, config_files) ) result = subprocess.run( command, cwd=self.root, env=env, capture_output=True, text=True, check=True ) return json.loads(result.stdout) + def generate_each(self, requests: Sequence[GenerateRequest], workers: int) -> list[list[dict]]: + """``generate(*request)`` for every request, in order. + + A module-layout revision runs each request in a fork of a process that already imported + its generator's imports, so a request skips interpreter start-up and imports. A request + that does not exit 0 there is run again as its own process, which raises what + ``generate`` raises. A fork reads stdin from /dev/null and writes stdout to a file, and + shares state its preloaded imports created; generators that depend on any of that need + ``INFX_FORK_GENERATORS=0``, which runs every request as its own process. + """ + if ( + os.environ.get("INFX_FORK_GENERATORS") == "0" + or not hasattr(os, "fork") + or not (self.root / GENERATOR.module_path).is_file() + ): + with ThreadPoolExecutor(workers) as pool: + return list(pool.map(lambda request: self.generate(*request), requests)) + with tempfile.TemporaryDirectory(prefix="infx-generate-") as outputs, ExitStack() as stack: + started = [ + stack.enter_context(_ForkServer(self, GENERATOR)) + for _ in range(max(1, min(workers, len(requests)))) + ] + servers: queue.SimpleQueue[_ForkServer] = queue.SimpleQueue() + for server in started: + servers.put(server) + stopping = threading.Event() + + def run(index: int, request: GenerateRequest) -> list[dict]: + """Return request ``index``'s rows from a fork, or its own process on failure.""" + output = Path(outputs, f"{index}.json") + server = servers.get() + try: + code = server.run(self._generator_arguments(*request), output) + finally: + servers.put(server) + if stopping.is_set(): + raise RuntimeError("generate_each was interrupted") + if code != 0: + return self.generate(*request) + return json.loads(output.read_text()) + + pool = ThreadPoolExecutor(workers) + try: + return list(pool.map(run, range(len(requests)), requests)) + except BaseException: + stopping.set() + for server in started: + server.kill() + raise + finally: + pool.shutdown(wait=True, cancel_futures=True) + def fingerprints(self, rows: list[dict]) -> list[str]: + """Return the revision's recipe fingerprint of each row, or row digests if it has none. + + Raise ValueError when the fingerprinter does not print one fingerprint per row. + """ if not (self.root / f"{FINGERPRINTER.replace('.', '/')}.py").is_file(): from infx.matrix.fingerprint import row_fingerprint @@ -127,8 +239,74 @@ def fingerprints(self, rows: list[dict]) -> list[str]: return fingerprints +class _ForkServer: + """A process in ``revision``'s environment that preloads ``tool``'s imports and forks. + + Each fork runs the tool as ``__main__`` with stdout in the request's output file. + """ + + def __init__(self, revision: Revision, tool: Tool) -> None: + """Start the server in its own session, so a terminal Ctrl-C does not reach its forks. + + Requests are JSON ``[argv, output path]`` lines. Each exit status follows a marker, so + output from importing the tool is never taken for a reply. + """ + _, env = revision.invocation(tool, []) + self._process = subprocess.Popen( + [sys.executable, "-c", _FORK_SERVER, tool.module, _FORK_REPLY], + cwd=revision.root, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + start_new_session=True, + ) + if self._process.stdin is None or self._process.stdout is None: + raise RuntimeError("the fork server was started without pipes") + self._requests, self._replies = self._process.stdin, self._process.stdout + self._noisy = False + + def __enter__(self) -> Self: + """Return the server itself.""" + return self + + def kill(self) -> None: + """Kill the server and its forks, ignoring a server that already exited.""" + with suppress(ProcessLookupError): + os.killpg(self._process.pid, signal.SIGKILL) + + def __exit__(self, error_type: type[BaseException] | None, *_: object) -> None: + """Close the server's stdin and wait for it to exit, killing it first on an error.""" + if error_type is not None: + self.kill() + with suppress(OSError): + self._requests.close() + self._process.wait() + self._replies.close() + + def run(self, arguments: Sequence[str], output: Path) -> int | None: + """The fork's exit status, or None when the server is gone, answered garbage, or its + imports wrote to stdout, which a fresh process would put before every request's rows.""" + try: + request = [[os.fsdecode(argument) for argument in arguments], str(output)] + self._requests.write(json.dumps(request) + "\n") + self._requests.flush() + for line in self._replies: + if line.startswith(_FORK_REPLY): + return None if self._noisy else int(line.removeprefix(_FORK_REPLY)) + self._noisy = self._noisy or line != "\n" + except (OSError, ValueError): + return None + return None + + @contextmanager def snapshot(ref: str) -> Iterator[Revision]: + """Extract the matrix inputs at git ``ref`` into a temporary directory and yield its Revision. + + Raise ValueError when a blob cannot be read or the configs needed to generate are missing. + """ repository = git_repository_root() listing = subprocess.run( [ @@ -188,6 +366,7 @@ def snapshot(ref: str) -> Iterator[Revision]: def main(argv: Sequence[str] | None = None) -> None: + """Run a checkout's generator or planner with its own configs and exit with its status.""" parser = argparse.ArgumentParser( description="Run a checkout's own matrix generator or planner against its own configs." ) @@ -200,7 +379,9 @@ def main(argv: Sequence[str] | None = None) -> None: command, env = revision.invocation(TOOLS[options.tool], options.args) except ValueError as error: parser.error(str(error)) - raise SystemExit(subprocess.run(command, cwd=revision.root, env=env, check=False).returncode) + raise SystemExit( + status(subprocess.run(command, cwd=revision.root, env=env, check=False).returncode) + ) if __name__ == "__main__": diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index ea040e79c1..8d0f08d063 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -15,6 +15,7 @@ model_validator, ) +from infx import yamlio from infx.clusters import CLUSTER_LABEL_PREFIX, RunnerInventory DEFAULT_AGENTIC_DURATION_SECONDS = 3600 @@ -103,6 +104,7 @@ class ComponentMetadata(BaseModel): @field_validator("version") @classmethod def validate_component_version(cls, version: str) -> str: + """Return ``version``, rejecting an ``image:`` reference with ValueError.""" if version.startswith("image:"): raise ValueError( "component version must be a release, package version, or " @@ -120,6 +122,7 @@ class KVOffloadBackendMetadata(BaseModel): @field_validator("version") @classmethod def validate_component_version(cls, version: str | None) -> str | None: + """Return ``version``, rejecting an ``image:`` reference with ValueError.""" if version is not None and version.startswith("image:"): raise ValueError( "component version must be a release, package version, or " @@ -129,6 +132,7 @@ def validate_component_version(cls, version: str | None) -> str | None: def _validate_tp_context_topology(self: Any) -> Any: + """Return the model, raising ValueError unless its TP is divisible by its DCP size.""" if self.tp % self.dcp_size != 0: raise ValueError( f"'{Fields.TP.value}' ({self.tp}) must be divisible by " @@ -173,6 +177,7 @@ class SingleNodeMatrixEntry(BaseModel): @model_validator(mode="after") def validate_single_node_topology(self) -> Self: + """Return the entry, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) @@ -180,7 +185,7 @@ class WorkerConfig(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) num_worker: int = Field(alias=Fields.NUM_WORKER.value) - tp: int + tp: int = Field(gt=0) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -194,6 +199,7 @@ class WorkerConfig(BaseModel): @model_validator(mode="after") def validate_worker_topology(self) -> Self: + """Return the worker, raising ValueError for an invalid DWDP or TP and DCP topology.""" if self.dwdp_size > 1 and ( self.num_worker <= 0 or (self.tp, self.ep, self.pp, self.dcp_size, self.pcp_size) != (1, 1, 1, 1, 1) @@ -209,7 +215,7 @@ class AggregateWorkerConfig(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) num_worker: int = Field(default=1, alias=Fields.NUM_WORKER.value, gt=0, strict=True) - tp: int + tp: int = Field(gt=0) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) pcp_size: int = Field(default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) @@ -222,10 +228,12 @@ class AggregateWorkerConfig(BaseModel): @model_validator(mode="after") def validate_worker_topology(self) -> Self: + """Return the worker, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) def _validate_worker_hardware_pair(self: Any) -> Any: + """Return the model, raising ValueError for DWDP on decode or hardware set on one side only.""" if self.decode.dwdp_size > 1: raise ValueError("DWDP is supported only on prefill workers") if bool(self.prefill.hardware) != bool(self.decode.hardware): @@ -279,10 +287,12 @@ class MultiNodeMatrixEntry(BaseModel): @model_validator(mode="after") def validate_worker_hardware_pair(self) -> Self: + """Return the entry, raising ValueError unless its workers' hardware fields agree.""" return _validate_worker_hardware_pair(self) @model_validator(mode="after") def validate_disagg_transfer(self) -> Self: + """Return the entry, raising ValueError if it is disaggregated without a KV transfer.""" if self.disagg and self.kv_p2p_transfer is None: raise ValueError(f"{Fields.DISAGG.value}=true requires {Fields.KV_P2P_TRANSFER.value}") return self @@ -329,10 +339,12 @@ class SingleNodeAgenticMatrixEntry(BaseModel): @model_validator(mode="after") def validate_kv_offload_fields(self) -> Self: + """Return the entry, raising ValueError for inconsistent KV offload fields.""" return _validate_kv_offload_fields(self) @model_validator(mode="after") def validate_single_node_topology(self) -> Self: + """Return the entry, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) @@ -382,14 +394,17 @@ class MultiNodeAgenticMatrixEntry(BaseModel): @model_validator(mode="after") def validate_worker_hardware_pair(self) -> Self: + """Return the entry, raising ValueError unless its workers' hardware fields agree.""" return _validate_worker_hardware_pair(self) @model_validator(mode="after") def validate_kv_offload_fields(self) -> Self: + """Return the entry, raising ValueError for inconsistent KV offload fields.""" return _validate_kv_offload_fields(self) @model_validator(mode="after") def validate_disagg_transfer(self) -> Self: + """Return the entry, raising ValueError if it is disaggregated without a KV transfer.""" if self.disagg and self.kv_p2p_transfer is None: raise ValueError(f"{Fields.DISAGG.value}=true requires {Fields.KV_P2P_TRANSFER.value}") return self @@ -399,6 +414,7 @@ def validate_disagg_transfer(self) -> Self: def validate_agentic_matrix_entry(entry: dict) -> dict: + """Return an agentic matrix row unchanged, raising ValueError with the row if it is invalid.""" try: if Fields.PREFILL.value in entry: MultiNodeAgenticMatrixEntry(**entry) @@ -412,6 +428,7 @@ def validate_agentic_matrix_entry(entry: dict) -> dict: def validate_matrix_entry(entry: dict, is_multinode: bool) -> dict: + """Return a fixed-sequence matrix row unchanged, raising ValueError with the row if invalid.""" try: if is_multinode: MultiNodeMatrixEntry(**entry) @@ -425,10 +442,11 @@ def validate_matrix_entry(entry: dict, is_multinode: bool) -> dict: def _validate_conc_fields(self: Any) -> Any: + """Return the model, raising ValueError unless it has exactly one valid conc range or list.""" has_range = self.conc_start is not None and self.conc_end is not None has_list = self.conc_list is not None and len(self.conc_list) > 0 - if has_range and has_list: + if has_list and (self.conc_start is not None or self.conc_end is not None): raise ValueError( f"Cannot specify both '{Fields.CONC_LIST.value}' list and " f"'{Fields.CONC_START.value}'/'{Fields.CONC_END.value}'. " @@ -463,10 +481,14 @@ def _validate_conc_fields(self: Any) -> Any: if has_list and not all(x > 0 for x in self.conc_list): raise ValueError(f"Input '{Fields.CONC_LIST.value}' entries must be greater than 0.") + if has_list and len(set(self.conc_list)) != len(self.conc_list): + raise ValueError(f"Input '{Fields.CONC_LIST.value}' entries must not repeat.") + return self def _validate_agentic_runner_is_cluster(runner: str, scenarios: Any) -> None: + """Raise ValueError if an agentic config's ``runner`` is not a cluster label.""" if scenarios.agentic_coding and not runner.startswith(CLUSTER_LABEL_PREFIX): raise ValueError( f"Agentic master configs must use a '{CLUSTER_LABEL_PREFIX}' runner " @@ -475,6 +497,7 @@ def _validate_agentic_runner_is_cluster(runner: str, scenarios: Any) -> None: def _validate_kv_offload_fields(self: Any) -> Any: + """Return the model, raising ValueError unless a KV offload backend comes with offloading.""" backend = getattr(self, "kv_offload_backend", None) if self.kv_offloading is None: if backend is not None: @@ -498,6 +521,7 @@ def _validate_kv_offload_fields(self: Any) -> Any: def _recipe_path(value: str) -> str: + """Return ``value``, raising ValueError unless it is a normalized relative POSIX path.""" path = PurePosixPath(value) if ( not path.parts @@ -511,6 +535,7 @@ def _recipe_path(value: str) -> str: def _recipe_reference(value: str) -> str: + """Return a ``path[:selector]`` recipe reference, raising ValueError for a bad part.""" path, separator, selector = value.partition(":") _recipe_path(path) if separator and not RECIPE_SELECTOR.fullmatch(selector): @@ -527,7 +552,7 @@ def _recipe_reference(value: str) -> str: class SingleNodeSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - tp: int + tp: int = Field(gt=0) srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field(default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) @@ -544,10 +569,12 @@ class SingleNodeSearchSpaceEntry(BaseModel): @model_validator(mode="after") def validate_conc_fields(self) -> Self: + """Return the entry, raising ValueError for an invalid conc range or list.""" return _validate_conc_fields(self) @model_validator(mode="after") def validate_single_node_topology(self) -> Self: + """Return the entry, raising ValueError unless TP is divisible by DCP size.""" return _validate_tp_context_topology(self) @@ -576,10 +603,12 @@ class MultiNodeSearchSpaceEntry(BaseModel): @model_validator(mode="after") def validate_conc_fields(self) -> Self: + """Return the entry, raising ValueError for an invalid conc range or list.""" return _validate_conc_fields(self) @model_validator(mode="after") def validate_worker_hardware_pair(self) -> Self: + """Return the entry, raising ValueError unless it has one worker or a matching pair.""" has_worker = self.worker is not None has_any_disagg_worker = self.prefill is not None or self.decode is not None has_complete_disagg_workers = self.prefill is not None and self.decode is not None @@ -615,7 +644,7 @@ class MultiNodeSeqLenConfig(BaseModel): class AgenticCodingSearchSpaceEntry(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - tp: int | None = None + tp: int | None = Field(default=None, gt=0) srt_recipe: RecipeReference | None = Field(default=None, alias=Fields.SRT_RECIPE.value) eval_srt_recipe: RecipeReference | None = Field( default=None, alias=Fields.EVAL_SRT_RECIPE.value @@ -649,14 +678,17 @@ class AgenticCodingSearchSpaceEntry(BaseModel): @model_validator(mode="after") def validate_conc_fields(self) -> Self: + """Return the entry, raising ValueError for an invalid conc range or list.""" return _validate_conc_fields(self) @model_validator(mode="after") def validate_kv_offload_fields(self) -> Self: + """Return the entry, raising ValueError for inconsistent KV offload fields.""" return _validate_kv_offload_fields(self) @model_validator(mode="after") def validate_topology_fields(self) -> Self: + """Return the entry, raising ValueError unless it sets exactly one valid topology.""" has_single_node = self.tp is not None has_aggregate_worker = self.worker is not None has_any_multinode_field = self.prefill is not None or self.decode is not None @@ -718,6 +750,7 @@ class AgenticCodingConfig(BaseModel): @model_validator(mode="after") def validate_dram_offload_capacity(self) -> Self: + """Return the config, raising ValueError for DRAM offload it cannot size.""" for entry in self.search_space: if entry.kv_offloading != "dram": continue @@ -746,6 +779,7 @@ class SingleNodeScenarios(BaseModel): @model_validator(mode="after") def at_least_one_scenario(self) -> Self: + """Return the scenarios, raising ValueError when none is set.""" if not self.fixed_seq_len and not self.agentic_coding: raise ValueError("At least one scenario type must be specified") return self @@ -763,12 +797,14 @@ class MultiNodeScenarios(BaseModel): @model_validator(mode="after") def at_least_one_scenario(self) -> Self: + """Return the scenarios, raising ValueError when none is set.""" if not self.fixed_seq_len and not self.agentic_coding: raise ValueError("At least one scenario type must be specified") return self def _master_search_space_entries(self: BaseModel) -> list[BaseModel]: + """Return every search-space entry across a master config's scenarios.""" return [ entry for scenario_configs in ( @@ -781,6 +817,7 @@ def _master_search_space_entries(self: BaseModel) -> list[BaseModel]: def _validate_component_metadata_scope(self: BaseModel) -> BaseModel: + """Return the config, raising ValueError if router or KV transfer settings are misplaced.""" search_space_entries = _master_search_space_entries(self) for field in (Fields.ROUTER, Fields.KV_P2P_TRANSFER): @@ -822,6 +859,7 @@ def _validate_component_metadata_scope(self: BaseModel) -> BaseModel: def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: + """Return the config, raising ValueError if an entry's workers do not fit its node type.""" search_space_entries = _master_search_space_entries(self) for entry in search_space_entries: worker = getattr(entry, "worker", None) @@ -870,6 +908,7 @@ def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: def _validate_srt_recipes(self: Any) -> Any: + """Return the config, raising ValueError unless every entry names a valid srt recipe.""" entries = _master_search_space_entries(self) if self.srt_recipe_dir is None: raise ValueError(f"srt-slurm configs require {Fields.SRT_RECIPE_DIR.value}") @@ -906,19 +945,23 @@ class SingleNodeMasterConfigEntry(BaseModel): @model_validator(mode="after") def validate_agentic_runner(self) -> Self: + """Return the config, raising ValueError if agentic scenarios use a non-cluster runner.""" _validate_agentic_runner_is_cluster(self.runner, self.scenarios) return self @model_validator(mode="after") def validate_component_metadata_scope(self) -> Self: + """Return the config, raising ValueError if router or KV transfer settings are misplaced.""" return _validate_component_metadata_scope(self) @model_validator(mode="after") def validate_multinode_entry_scope(self) -> Self: + """Return the config, raising ValueError if an entry's workers do not fit its node type.""" return _validate_multinode_entry_scope(self) @model_validator(mode="after") def validate_srt_recipes(self) -> Self: + """Return the config, raising ValueError unless every entry names a valid srt recipe.""" return _validate_srt_recipes(self) @@ -942,27 +985,33 @@ class MultiNodeMasterConfigEntry(BaseModel): @model_validator(mode="after") def validate_agentic_runner(self) -> Self: + """Return the config, raising ValueError if agentic scenarios use a non-cluster runner.""" _validate_agentic_runner_is_cluster(self.runner, self.scenarios) return self @model_validator(mode="after") def validate_component_metadata_scope(self) -> Self: + """Return the config, raising ValueError if router or KV transfer settings are misplaced.""" return _validate_component_metadata_scope(self) @model_validator(mode="after") def validate_multinode_entry_scope(self) -> Self: + """Return the config, raising ValueError if an entry's workers do not fit its node type.""" return _validate_multinode_entry_scope(self) @model_validator(mode="after") def validate_srt_recipes(self) -> Self: + """Return the config, raising ValueError unless every entry names a valid srt recipe.""" return _validate_srt_recipes(self) def srt_recipe_path(multinode: bool, directory: str, reference: str) -> str: + """Return the repository path of a recipe ``reference`` in its single- or multinode dir.""" return f"{MULTI_NODE_RECIPES if multinode else SINGLE_NODE_RECIPES}/{directory}/{reference}" def _zip_length(group: Any) -> int: + """Return the variant count of a zip override group, its longest list.""" if isinstance(group, list): return len(group) if isinstance(group, dict): @@ -971,12 +1020,16 @@ def _zip_length(group: Any) -> int: def _check_srt_recipe(root: Path, reference: str, recipes: dict[str, Any]) -> None: + """Raise ValueError unless the recipe file exists, parses, and has the selected variant. + + Parsed recipes are cached in ``recipes`` by path. + """ path, _, selector = reference.partition(":") if path not in recipes: if not (root / path).is_file(): raise ValueError(f"{reference} does not exist") try: - recipes[path] = yaml.safe_load((root / path).read_text()) + recipes[path] = yamlio.safe_load((root / path).read_text()) except yaml.YAMLError as error: raise ValueError(f"{path} is not valid YAML") from error recipe = recipes[path] @@ -990,6 +1043,7 @@ def _check_srt_recipe(root: Path, reference: str, recipes: dict[str, Any]) -> No def config_root(config_files: list[str]) -> Path: + """Return the project directory shared by the config files, or raise ValueError.""" roots = {Path(config_file).resolve().parent.parent for config_file in config_files} if len(roots) != 1: raise ValueError(f"Master configs must share one project directory: {sorted(roots)}") @@ -997,6 +1051,10 @@ def config_root(config_files: list[str]) -> Path: def srt_recipe_references(master_configs: dict) -> dict[str, list[str]]: + """Return the srt recipe paths each master config key references. + + Raise ValueError when a master config entry fails validation. + """ references = {} for key, entry in master_configs.items(): model = ( @@ -1018,6 +1076,7 @@ def srt_recipe_references(master_configs: dict) -> dict[str, list[str]]: def validate_master_config(master_configs: dict, root: Path) -> dict: + """Return the master configs, raising ValueError if an entry or its recipes are invalid.""" recipes: dict[str, Any] = {} for key, references in srt_recipe_references(master_configs).items(): for reference in references: @@ -1029,6 +1088,7 @@ def validate_master_config(master_configs: dict, root: Path) -> dict: def _validate_runner_labels(labels: dict) -> None: + """Raise ValueError unless every runner label maps to a non-empty list of strings.""" for key, value in labels.items(): if not isinstance(value, list): raise ValueError( @@ -1043,6 +1103,7 @@ def _validate_runner_labels(labels: dict) -> None: def validate_runner_config(runner_configs: dict) -> dict: + """Return the runner config, raising ValueError if its labels or clusters are invalid.""" labels = runner_configs.get("labels") if not isinstance(labels, dict): raise ValueError("Runner config must define a labels mapping") @@ -1089,6 +1150,7 @@ class ChangelogEntry(BaseModel): @model_validator(mode="after") def validate_append_only_mode(self) -> Self: + """Return the entry, raising ValueError if no-evals or append-only selects evals.""" if self.no_evals and ( self.evals_only or self.all_evals or self.eval_min_prefill_ep is not None ): @@ -1125,11 +1187,15 @@ class ChangelogMatrixEntry(BaseModel): def load_config_files(config_files: list[str], validate: bool = True) -> dict: + """Load and merge the master config files, validating them unless ``validate`` is False. + + Raise ValueError for a missing file, bad or duplicate keys, or an invalid config. + """ all_config_data = {} for config_file in config_files: try: with open(config_file) as f: - config_data = yaml.safe_load(f) + config_data = yamlio.load(f, yamlio.UniqueKeySafeLoader) if not isinstance(config_data, dict): raise ValueError(f"Config file '{config_file}' must contain a dictionary") @@ -1160,9 +1226,13 @@ def load_config_files(config_files: list[str], validate: bool = True) -> dict: def load_runner_file(runner_file: str, validate: bool = True) -> dict: + """Load the runner config, validating it unless ``validate`` is False. + + Raise ValueError when the file is missing or invalid. + """ try: with open(runner_file) as f: - runner_config = yaml.safe_load(f) + runner_config = yamlio.safe_load(f) except FileNotFoundError as e: raise ValueError(f"Runner config file '{runner_file}' does not exist.") from e diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 5cf7469691..67ada94c91 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -13,6 +13,7 @@ def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: + """Return ``env[name]`` as an int, or ``default`` when unset or empty.""" value = env.get(name) if value in (None, ""): return default @@ -20,6 +21,7 @@ def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: def _env_bool(env: Mapping[str, str], name: str, default: bool = False) -> bool: + """Return whether ``env[name]`` is a truthy word, or ``default`` when unset or empty.""" value = env.get(name) if value in (None, ""): return default @@ -27,6 +29,7 @@ def _env_bool(env: Mapping[str, str], name: str, default: bool = False) -> bool: def _required_env(env: Mapping[str, str], name: str) -> str: + """Return ``env[name]``, raising SystemExit when it is unset or empty.""" value = env.get(name) if value in (None, ""): raise SystemExit(f"Missing required environment variable: {name}") @@ -34,6 +37,7 @@ def _required_env(env: Mapping[str, str], name: str) -> str: def _optional_component_metadata(env: Mapping[str, str], env_name: str) -> dict[str, str] | None: + """Parse the component metadata JSON in ``env[env_name]``, or None when unset.""" return parse_component_metadata( env.get(env_name), env_name, @@ -44,6 +48,7 @@ def _optional_component_metadata(env: Mapping[str, str], env_name: str) -> dict[ def _optional_kv_offload_backend_metadata( env: Mapping[str, str], env_name: str ) -> dict[str, str] | None: + """Parse the KV offload backend metadata in ``env[env_name]``, where the version may be absent.""" return parse_component_metadata( env.get(env_name), env_name, @@ -55,6 +60,10 @@ def _optional_kv_offload_backend_metadata( def _validate_kv_offload_env( env: Mapping[str, str], ) -> tuple[str, dict[str, str] | None]: + """Return KV_OFFLOADING and the backend metadata from ``env``. + + Raises SystemExit when the backend name and metadata disagree with the offload mode. + """ kv_offloading = _required_env(env, "KV_OFFLOADING") backend_name = env.get("KV_OFFLOAD_BACKEND", "") backend_metadata = _optional_kv_offload_backend_metadata(env, "KV_OFFLOAD_BACKEND_METADATA") @@ -70,6 +79,10 @@ def _validate_kv_offload_env( def _gpu_shape(env: Mapping[str, str]) -> tuple[dict[str, Any], int, int, int, str]: + """Derive parallelism fields, GPU count, TP, EP and DP-attention from ``env``. + + Multinode runs sum the prefill and decode workers. Raises SystemExit on invalid shapes. + """ is_multinode = _env_bool(env, "IS_MULTINODE") tp = _env_int(env, "TP", 1) ep = _env_int(env, "EP_SIZE", 1) diff --git a/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py b/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py index 213da4ebc9..978f0802ad 100644 --- a/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py +++ b/inferencex-e2e/infx/results/agentic/analyze_benchmark_distributions.py @@ -4,29 +4,24 @@ summary stats. Usage: - python analyze_benchmark_distributions.py path/to/aiperf_artifacts/ -o output_dir/ + python -m infx.results.agentic.analyze_benchmark_distributions -o """ from __future__ import annotations import argparse -import json from pathlib import Path +from .artifacts import load_records as load_profiled_records + def load_records(artifacts_dir: Path) -> list[dict]: - """Load per-request records from profile_export.jsonl.""" - jsonl_path = artifacts_dir / "profile_export.jsonl" - records = [] - with open(jsonl_path) as f: - for line in f: - line = line.strip() - if line: - records.append(json.loads(line)) - return records + """Load successful profiling records from profile_export.jsonl.""" + return load_profiled_records(artifacts_dir / "profile_export.jsonl") def _stats(values: list[int]) -> dict[str, float]: + """Return count, mean, median, p75, p90 and p95 of the integer ``values``.""" sv = sorted(values) n = len(sv) return { @@ -40,6 +35,7 @@ def _stats(values: list[int]) -> dict[str, float]: def _fmt(s: dict[str, float]) -> str: + """Format a ``_stats`` dict as one indented summary line.""" return ( f" n={s['n']:,} mean={s['mean']:,.0f} median={s['median']:,} " f"p75={s['p75']:,} p90={s['p90']:,} p95={s['p95']:,}" @@ -47,6 +43,7 @@ def _fmt(s: dict[str, float]) -> str: def analyze(records: list[dict], output_dir: Path) -> None: + """Print ISL/OSL stats for ``records`` and write them to workload_distribution_summary.txt in ``output_dir``.""" output_dir.mkdir(parents=True, exist_ok=True) all_isl: list[int] = [] @@ -83,6 +80,7 @@ def analyze(records: list[dict], output_dir: Path) -> None: def main() -> None: + """Parse the artifacts dir and output dir from argv and write the distribution summary.""" parser = argparse.ArgumentParser(description="Analyze benchmark workload distributions") parser.add_argument("artifacts_dir", help="Path to aiperf_artifacts/ directory") parser.add_argument( diff --git a/inferencex-e2e/infx/results/agentic/artifacts.py b/inferencex-e2e/infx/results/agentic/artifacts.py index a35ee829ae..dfe8a52bcd 100644 --- a/inferencex-e2e/infx/results/agentic/artifacts.py +++ b/inferencex-e2e/infx/results/agentic/artifacts.py @@ -21,11 +21,13 @@ def resolve_artifact_dir(result_dir: Path) -> Path: def load_aggregate(path: Path) -> dict[str, Any]: + """Return the JSON object stored at ``path``.""" with open(path) as f: return json.load(f) def load_records(path: Path) -> list[dict[str, Any]]: + """Return the profiled, error-free records from the profile_export.jsonl at ``path``.""" records, _ = load_records_with_accounting(path) return records @@ -73,6 +75,7 @@ def load_records_with_accounting( def _error_category(error: Any) -> str: + """Reduce a record's ``error`` field to a short category name, ``"unknown"`` when empty.""" if isinstance(error, dict): for key in ("type", "error_type", "code", "class", "status"): value = error.get(key) @@ -90,6 +93,7 @@ def _error_category(error: Any) -> str: def load_server_metrics(path: Path) -> dict[str, Any]: + """Return the server metrics JSON at ``path``, or an empty dict when the file is missing.""" if not path.exists(): return {} with open(path) as f: @@ -97,6 +101,7 @@ def load_server_metrics(path: Path) -> dict[str, Any]: def load_server_log_head(path: Path, max_bytes: int = 64 * 1024 * 1024) -> str | None: + """Return up to ``max_bytes`` of the log at ``path`` as text without NUL bytes, or None if missing.""" if not path.exists(): return None with open(path, "rb") as f: @@ -105,6 +110,7 @@ def load_server_log_head(path: Path, max_bytes: int = 64 * 1024 * 1024) -> str | def find_server_log_paths(result_dir: Path) -> list[Path]: + """List server.log in ``result_dir`` and watchtower-*.out files from it and three parents, deduplicated.""" paths: list[Path] = [] direct = result_dir / "server.log" if direct.is_file(): @@ -127,6 +133,10 @@ def find_server_log_paths(result_dir: Path) -> list[Path]: def _hf_traces_dir(hf_dataset_name: str | None, env: Mapping[str, str]) -> Path | None: + """Return the single cached snapshot dir of the HF dataset named ``hf_dataset_name``, else None. + + The hub cache root comes from HF_HUB_CACHE, HUGGINGFACE_HUB_CACHE or HF_HOME in ``env``. + """ if not hf_dataset_name: return None @@ -146,6 +156,7 @@ def _hf_traces_dir(hf_dataset_name: str | None, env: Mapping[str, str]) -> Path def _iter_trace_blobs(traces_dir: Path) -> Iterator[dict[str, Any]]: + """Yield every JSON object from the *.jsonl and *.json files in ``traces_dir``, skipping unreadable ones.""" for path in sorted(traces_dir.glob("*.jsonl")): try: with open(path) as f: diff --git a/inferencex-e2e/infx/results/agentic/backends/__init__.py b/inferencex-e2e/infx/results/agentic/backends/__init__.py index 0cde104ff8..d873d9c4b3 100644 --- a/inferencex-e2e/infx/results/agentic/backends/__init__.py +++ b/inferencex-e2e/infx/results/agentic/backends/__init__.py @@ -24,6 +24,7 @@ def detect_backend( metrics: dict[str, dict[str, Any]], framework: str, ) -> ServerMetricsBackend | None: + """Return the first backend that claims ``metrics`` for ``framework``, or None.""" for backend in BACKENDS: if backend.matches(metrics, framework): return backend diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 947024b111..9e5c661de2 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -23,6 +23,7 @@ class AtomBackend(ServerMetricsBackend): name = "atom" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for atom metric names, or for an empty scrape when ``framework`` is atom.""" metric_names = set(metrics) return any(name.startswith(("atom:", "atom_")) for name in metric_names) or ( not metrics and framework.lower() == "atom" @@ -34,6 +35,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, prefix and LMCache hit rates and KV usage from ATOM ``metrics``.""" prompt_total = sum_stat( metrics, _atom_counter_names("prompt_tokens"), diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 427a4c1606..8aafdc0885 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -28,6 +28,7 @@ class ServerMetricsBackend: name = "" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return whether this adapter handles ``metrics`` scraped from ``framework``.""" raise NotImplementedError def populate( @@ -36,6 +37,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill the ``flat`` and ``nested`` server metric dicts in place from ``metrics``.""" raise NotImplementedError def gpu_kv_capacity_tokens( @@ -43,10 +45,12 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], # noqa: ARG002 server_logs: Iterable[str | None], # noqa: ARG002 ) -> int | None: + """Return the total GPU KV cache capacity in tokens from ``metrics`` or ``server_logs``, None by default.""" return None def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: + """Return the nested server metrics skeleton with every value None, tagged ``present`` and ``metric_count``.""" return { "present": present, "adapter": "none", @@ -98,6 +102,7 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: def apply_profile_totals(flat: dict[str, Any], records: list[dict[str, Any]]) -> None: + """Fill missing prompt and generation totals in ``flat`` from per-record lengths and set the completed count.""" input_tokens = _record_token_sum(records, "input_sequence_length") output_tokens = _record_token_sum(records, "output_sequence_length") if flat["total_prompt_tokens"] is None and input_tokens is not None: @@ -108,12 +113,14 @@ def apply_profile_totals(flat: dict[str, Any], records: list[dict[str, Any]]) -> def counter_int(value: float | None) -> int | None: + """Round a counter ``value`` to an int, keeping None.""" if value is None: return None return round(value) def _record_token_sum(records: list[dict[str, Any]], metric_name: str) -> int | None: + """Sum the integer ``metric_name`` values across ``records``, or None when no record has one.""" total = 0 found = False for record in records: diff --git a/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py b/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py index 273be93f58..5e09446461 100644 --- a/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/dynamo_vllm.py @@ -13,6 +13,7 @@ class DynamoVllmBackend(VllmBackend): name = "dynamo-vllm" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for dynamo frameworks or dynamo_ metrics, except dynamo-sglang with sglang metrics.""" metric_names = set(metrics) framework = framework.lower() if framework == "dynamo-sglang" and any( @@ -27,6 +28,7 @@ def prompt_generation_totals( self, metrics: dict[str, dict[str, Any]], ) -> tuple[float | None, float | None]: + """Return prompt and generation token totals, preferring Dynamo frontend counters over vLLM ones.""" prompt_total = first_counter_total( metrics, ["dynamo_frontend_input_sequence_tokens", "vllm:prompt_tokens"], @@ -43,6 +45,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill the vLLM fields, then the Dynamo frontend and router fields, from ``metrics``.""" super().populate(metrics, flat, nested) self._populate_dynamo(metrics, flat, nested) @@ -52,6 +55,7 @@ def _populate_dynamo( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Add Dynamo GPU usage, frontend and router hit rates where vLLM metrics left gaps.""" dynamo_gpu_usage = normalize_fraction( gauge_stat( metrics, @@ -92,7 +96,9 @@ def _populate_dynamo( ) ) if flat["server_overall_cache_hit_rate"] is None: - flat["server_overall_cache_hit_rate"] = frontend_hit_rate or router_shared_hit_rate + overall = frontend_hit_rate if frontend_hit_rate is not None else router_shared_hit_rate + flat["server_overall_cache_hit_rate"] = overall + nested["cache"]["overall_cache_hit_rate"] = overall nested["cache"].update( { diff --git a/inferencex-e2e/infx/results/agentic/backends/sglang.py b/inferencex-e2e/infx/results/agentic/backends/sglang.py index c49d0ea038..a6990a16d0 100644 --- a/inferencex-e2e/infx/results/agentic/backends/sglang.py +++ b/inferencex-e2e/infx/results/agentic/backends/sglang.py @@ -26,6 +26,7 @@ class SglangBackend(ServerMetricsBackend): _DP_SIZE_RE = re.compile(r"\bdp_size=(?P\d+)\b") def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for sglang: metric names, or for an empty scrape when ``framework`` is sglang.""" metric_names = set(metrics) return any(name.startswith("sglang:") for name in metric_names) or ( not metrics and framework.lower() == "sglang" @@ -37,6 +38,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, cache hit rates by source and GPU/host KV usage from SGLang ``metrics``.""" prompt_total = sum_stat( metrics, "sglang:prompt_tokens", @@ -146,6 +148,7 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], # noqa: ARG002 server_logs: Iterable[str | None], ) -> int | None: + """Return the summed SGLang KV pool size parsed from ``server_logs``, ignoring ``metrics``.""" return sum_server_log_capacities( server_logs, self.kv_cache_pool_tokens_from_server_log, @@ -153,6 +156,10 @@ def gpu_kv_capacity_tokens( @classmethod def kv_cache_pool_tokens_from_server_log(cls, server_log: str | None) -> int | None: + """Return the KV pool tokens from max_total_num_tokens lines in ``server_log``, or None. + + Sums distinct DP/TP/EP ranks and scales a single logged rank by dp_size. + """ if not server_log: return None @@ -187,6 +194,7 @@ def kv_cache_pool_tokens_from_server_log(cls, server_log: str | None) -> int | N @classmethod def _dp_size(cls, server_log: str) -> int | None: + """Return the positive dp_size logged in ``server_log``, or None.""" match = cls._DP_SIZE_RE.search(server_log) if not match: return None diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index aba19a03c6..cb85924d28 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -20,6 +20,7 @@ class TrtllmBackend(ServerMetricsBackend): name = "trtllm" def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for trtllm_ metric names, or for an empty scrape from a TRT-LLM ``framework``.""" metric_names = set(metrics) return any(name.startswith("trtllm_") for name in metric_names) or ( not metrics and framework.lower() in ("trtllm", "dynamo-trt", "dynamo-trtllm") @@ -31,6 +32,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, KV hit rate, usage and offload bytes and per-endpoint sources from ``metrics``.""" prompt_total = _first_counter_total( metrics, ["dynamo_frontend_input_sequence_tokens", "trtllm_prompt_tokens_total"], @@ -130,6 +132,7 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], server_logs: Iterable[str | None], ) -> int | None: + """Return KV max blocks times tokens per block from ``metrics``, or None if either is missing.""" del server_logs max_blocks = sum_stat( metrics, @@ -151,6 +154,7 @@ def _first_counter_total( metrics: dict[str, dict[str, Any]], metric_names: list[str], ) -> float | None: + """Return the first non-None counter total among ``metric_names`` in ``metrics``.""" for metric_name in metric_names: value = sum_stat( metrics, @@ -163,6 +167,7 @@ def _first_counter_total( def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Return per-endpoint token, hit rate and KV usage rows from TRT-LLM ``metrics``, sorted by endpoint.""" endpoints: set[str] = set() roles: dict[str, str] = {} for entry in metrics.values(): @@ -186,6 +191,7 @@ def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: for endpoint in sorted(endpoints): def series_filter(series: dict, endpoint: str = endpoint) -> bool: + """Return whether ``series`` was scraped from ``endpoint``.""" return str(series.get("endpoint_url", "")) == endpoint prompt_tokens = sum_stat( @@ -233,6 +239,7 @@ def series_filter(series: dict, endpoint: str = endpoint) -> bool: def _normalize_role(mode: str) -> str: + """Map a disaggregation ``mode`` label to prefill, decode or combined, else return it lowercased.""" lowered = mode.lower() if lowered in ("prefill", "context", "ctx"): return "prefill" diff --git a/inferencex-e2e/infx/results/agentic/backends/vllm.py b/inferencex-e2e/infx/results/agentic/backends/vllm.py index 5baf5a47f2..bbafcfd2b8 100644 --- a/inferencex-e2e/infx/results/agentic/backends/vllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/vllm.py @@ -26,6 +26,7 @@ class VllmBackend(ServerMetricsBackend): _GPU_KV_SIZE_RE = re.compile(r"GPU KV cache size:\s*(?P[\d,]+)\s*tokens") def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: + """Return True for vllm: metric names, or for an empty scrape when ``framework`` is vllm.""" metric_names = set(metrics) return any(name.startswith("vllm:") for name in metric_names) or ( not metrics and framework.lower() == "vllm" @@ -37,6 +38,7 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: + """Fill token totals, prefix and external hit rates, KV usage, offload stats and sources from ``metrics``.""" prompt_total, generation_total = self.prompt_generation_totals(metrics) flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) @@ -177,6 +179,7 @@ def prompt_generation_totals( self, metrics: dict[str, dict[str, Any]], ) -> tuple[float | None, float | None]: + """Return the vLLM prompt and generation token totals from ``metrics``.""" prompt_total = sum_stat( metrics, "vllm:prompt_tokens", @@ -194,6 +197,7 @@ def gpu_kv_capacity_tokens( metrics: dict[str, dict[str, Any]], # noqa: ARG002 server_logs: Iterable[str | None], ) -> int | None: + """Return the summed vLLM GPU KV cache size parsed from ``server_logs``, ignoring ``metrics``.""" return sum_server_log_capacities( server_logs, self.kv_cache_pool_tokens_from_server_log, @@ -201,6 +205,7 @@ def gpu_kv_capacity_tokens( @classmethod def kv_cache_pool_tokens_from_server_log(cls, server_log: str | None) -> int | None: + """Return the GPU KV cache size in tokens from ``server_log``, summed per EngineCore, or None.""" if not server_log: return None @@ -239,6 +244,7 @@ def first_counter_total( metrics: dict[str, dict[str, Any]], metric_names: list[str], ) -> float | None: + """Return the first non-None counter total among ``metric_names`` in ``metrics``.""" for metric_name in metric_names: value = sum_stat( metrics, @@ -251,6 +257,7 @@ def first_counter_total( def _vllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Return per-source token, hit rate and KV usage rows from vLLM ``metrics``, sorted by source id.""" source_ids = set() for metric_name in ( "vllm:prompt_tokens", @@ -269,6 +276,7 @@ def _vllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: continue def series_filter(series: dict, source_id: str = source_id) -> bool: + """Return whether ``series`` belongs to ``source_id``.""" return _source_id(series) == source_id prompt_tokens = sum_stat(metrics, "vllm:prompt_tokens", series_filter=series_filter) @@ -297,6 +305,7 @@ def series_filter(series: dict, source_id: str = source_id) -> bool: def _source_id(series: dict[str, Any]) -> str: + """Build a ``role|endpoint|key=value`` id for ``series`` from its component, endpoint and rank labels.""" labels = series.get("labels") if isinstance(series.get("labels"), dict) else {} component = str(labels.get("dynamo_component") or labels.get("component") or "") if component == "prefill": diff --git a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py index da1ee90af8..82c20e6d72 100644 --- a/inferencex-e2e/infx/results/agentic/collect_sweep_results.py +++ b/inferencex-e2e/infx/results/agentic/collect_sweep_results.py @@ -14,6 +14,7 @@ python collect_sweep_results.py """ +import contextlib import json import sys from pathlib import Path @@ -21,7 +22,7 @@ import pandas as pd -def _load_custom_client_csv(client_csv: Path, exp_dir: Path) -> pd.DataFrame | None: # noqa: ARG001 +def _load_custom_client_csv(client_csv: Path) -> pd.DataFrame | None: """Load per-request metrics from custom benchmark client CSV.""" df = pd.read_csv(client_csv) if len(df) == 0: @@ -54,6 +55,7 @@ def _load_aiperf_summary_csv(csv_path: Path) -> dict | None: break def metric_stat(metric_name: str, stat: str) -> float: + """Return the ``stat`` column of ``metric_name`` as a float, or 0 when missing or unparsable.""" if metric_name in per_metric: try: return float(per_metric[metric_name].get(stat, 0)) @@ -62,6 +64,7 @@ def metric_stat(metric_name: str, stat: str) -> float: return 0 def scalar_val(metric_name: str) -> float: + """Return the scalar row ``metric_name`` as a float, or 0 when missing or unparsable.""" if metric_name in scalars: try: return float(scalars[metric_name]) @@ -141,19 +144,15 @@ def load_experiment(exp_dir: Path) -> dict | None: return result result.update(aiperf_metrics) elif client_csv.exists(): - df = _load_custom_client_csv(client_csv, exp_dir) + df = _load_custom_client_csv(client_csv) if df is None or len(df) == 0: return result metadata_file = exp_dir / "benchmark_metadata.json" total_time_sec = None if metadata_file.exists(): - try: - with open(metadata_file) as f: - metadata = json.load(f) - total_time_sec = metadata.get("benchmark_runtime_sec") - except Exception: # noqa: BLE001, S110 - pass + with contextlib.suppress(Exception), open(metadata_file) as f: + total_time_sec = json.load(f).get("benchmark_runtime_sec") if not total_time_sec or total_time_sec <= 0: first_start_ms = df["start_time_ms"].min() @@ -219,6 +218,7 @@ def load_experiment(exp_dir: Path) -> dict | None: def main() -> None: + """Write summary.csv to the output dir for every experiment under the artifacts dir from argv, then plot overviews.""" if len(sys.argv) < 3: print(f"Usage: {sys.argv[0]} ") sys.exit(1) diff --git a/inferencex-e2e/infx/results/agentic/common.py b/inferencex-e2e/infx/results/agentic/common.py index eb498c9fd6..26bb6ea368 100644 --- a/inferencex-e2e/infx/results/agentic/common.py +++ b/inferencex-e2e/infx/results/agentic/common.py @@ -9,6 +9,7 @@ def percentile(data: list[float], p: float) -> float: + """Return the linearly interpolated ``p``th percentile of ``data``, or 0.0 when empty.""" if not data: return 0.0 sorted_data = sorted(data) @@ -21,6 +22,7 @@ def percentile(data: list[float], p: float) -> float: def stats_for(prefix: str, values: list[float]) -> dict[str, float]: + """Return mean, p50, p75, p90, p95 and std of ``values`` keyed with ``prefix``, or {} when empty.""" if not values: return {} return { @@ -34,6 +36,7 @@ def stats_for(prefix: str, values: list[float]) -> dict[str, float]: def to_float(value: Any) -> float | None: + """Return ``value`` as a float, or None when it is None or not numeric.""" if value is None: return None try: @@ -43,6 +46,7 @@ def to_float(value: Any) -> float | None: def to_int(value: Any) -> int | None: + """Return ``value`` as an int, or None when it is None or not convertible.""" if value is None: return None try: @@ -52,6 +56,7 @@ def to_int(value: Any) -> int | None: def rate(numerator: float | None, denominator: float | None) -> float | None: + """Return ``numerator / denominator``, or None when either is None or the denominator is not positive.""" if numerator is None or denominator is None: return None if denominator <= 0: @@ -98,6 +103,7 @@ def metric_series( metrics: dict[str, dict[str, Any]], metric_names: str | list[str], ) -> list[dict[str, Any]]: + """Return every series dict of the named metric or metrics in ``metrics``.""" names = [metric_names] if isinstance(metric_names, str) else metric_names out: list[dict[str, Any]] = [] for name in names: @@ -115,6 +121,7 @@ def series_stat( series: dict[str, Any], preferred_keys: tuple[str, ...] = ("total", "sum", "max", "avg"), ) -> float | None: + """Return the first numeric stat of ``series`` among ``preferred_keys``, or None.""" stats = series.get("stats") if not isinstance(stats, dict): return None @@ -132,6 +139,7 @@ def sum_stat( preferred_keys: tuple[str, ...] = ("total", "sum", "max", "avg"), series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: + """Sum the preferred stat over matching series of ``metric_names``, or None when none has one.""" total = 0.0 found = False for series in metric_series(metrics, metric_names): @@ -153,6 +161,7 @@ def gauge_stat( combine: str = "max", series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: + """Combine the preferred stat of matching series by max, avg or sum per ``combine``, or None.""" values: list[float] = [] for series in metric_series(metrics, metric_names): if series_filter is not None and not series_filter(series): @@ -170,6 +179,7 @@ def gauge_stat( def label_value(series: dict[str, Any], key: str) -> str | None: + """Return the label ``key`` of ``series`` as a string, or None.""" labels = series.get("labels") if not isinstance(labels, dict): return None @@ -180,6 +190,7 @@ def label_value(series: dict[str, Any], key: str) -> str | None: def label_equals(key: str, value: str) -> Callable[[dict[str, Any]], bool]: + """Return a series filter that keeps series whose label ``key`` equals ``value``.""" return lambda series: label_value(series, key) == value @@ -190,6 +201,7 @@ def sum_by_label( *, preferred_keys: tuple[str, ...] = ("total", "sum", "max", "avg"), ) -> dict[str, float]: + """Sum the preferred stat of ``metric_names`` grouped by the first present label among ``label_keys``.""" keys = [label_keys] if isinstance(label_keys, str) else label_keys out: dict[str, float] = {} for series in metric_series(metrics, metric_names): @@ -215,6 +227,7 @@ def sum_server_log_capacities( logs: Iterable[str | None], parser: Callable[[str | None], int | None], ) -> int | None: + """Sum the capacities ``parser`` extracts from each of ``logs``, or None when none parse.""" total = 0 found = False for text in logs: diff --git a/inferencex-e2e/infx/results/agentic/power_adapter.py b/inferencex-e2e/infx/results/agentic/power_adapter.py index 9973c0ed7a..59860c74e7 100644 --- a/inferencex-e2e/infx/results/agentic/power_adapter.py +++ b/inferencex-e2e/infx/results/agentic/power_adapter.py @@ -85,7 +85,9 @@ def build_power_window( try: aggregate = load_aggregate(aggregate_path) records = load_records(records_path) - except (OSError, json.JSONDecodeError, TypeError, ValueError): + except (AttributeError, OSError, json.JSONDecodeError, TypeError, ValueError): + return None, ["profile_artifacts_invalid"] + if not isinstance(aggregate, dict): return None, ["profile_artifacts_invalid"] raw_start = aggregate.get("start_time") @@ -140,6 +142,7 @@ def build_power_window( def _fail_multinode_adapter(message: str, *, require_power: bool) -> int: + """Log ``message`` to stderr and return exit code 1 when ``require_power`` is set, else 0.""" print(f"[agentx_power] {message}", file=sys.stderr) return 1 if require_power else 0 @@ -192,6 +195,7 @@ def _window_payload( end: float | None, duration: float | None, ) -> dict[str, Any]: + """Build the formal custom-benchmark window JSON for ``result_path`` with the given status and times.""" return { "schema_version": 1, "benchmark_type": "custom", @@ -281,6 +285,7 @@ def _record_multinode_adapter_failure( validation_result: Path, reasons: list[str], ) -> None: + """Mark ``agg_result`` power-invalid and always write ``validation_result`` with ``reasons``.""" try: aggregate = json.loads(agg_result.read_text(encoding="utf-8")) if not isinstance(aggregate, dict): @@ -298,6 +303,7 @@ def _record_multinode_adapter_failure( def _write_multinode_failure_validation(validation_result: Path, reasons: list[str]) -> None: + """Write a power-invalid validation JSON with ``reasons`` to ``validation_result``.""" _write_json_atomic( validation_result, { @@ -309,6 +315,7 @@ def _write_multinode_failure_validation(validation_result: Path, reasons: list[s def _formal_result_for_directory(result_dir: Path) -> Path | None: + """Return the single agentic_power_concurrency_*.json in ``result_dir``, or None if absent or ambiguous.""" candidates = [ path for path in result_dir.glob("agentic_power_concurrency_*.json") @@ -318,6 +325,7 @@ def _formal_result_for_directory(result_dir: Path) -> Path | None: def _gpu_count(value: Any) -> int | None: + """Return ``value`` when it is a non-negative int (not bool), else None.""" if isinstance(value, bool) or not isinstance(value, int) or value < 0: return None return value @@ -431,6 +439,7 @@ def run_native_agentic_power( def main() -> int: + """Run the window, missing-contract or power-join mode chosen by argv and return the exit code.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--result-dir", type=Path, required=True) parser.add_argument("--agg-result", type=Path) diff --git a/inferencex-e2e/infx/results/agentic/process_agentic_result.py b/inferencex-e2e/infx/results/agentic/process_agentic_result.py index a44dda3f68..d7dac83644 100644 --- a/inferencex-e2e/infx/results/agentic/process_agentic_result.py +++ b/inferencex-e2e/infx/results/agentic/process_agentic_result.py @@ -23,6 +23,7 @@ def main() -> int: + """Build the AgentX aggregate from RESULT_DIR artifacts, write it to AGENTIC_OUTPUT_DIR and return the exit code.""" result_filename = os.environ.get("RESULT_FILENAME", "") if not result_filename: print("ERROR: RESULT_FILENAME env var not set", file=sys.stderr) diff --git a/inferencex-e2e/infx/results/agentic/request_metrics.py b/inferencex-e2e/infx/results/agentic/request_metrics.py index 66075e50e2..7d032b4e7a 100644 --- a/inferencex-e2e/infx/results/agentic/request_metrics.py +++ b/inferencex-e2e/infx/results/agentic/request_metrics.py @@ -12,6 +12,7 @@ def _metric_value(record: dict[str, Any], key: str) -> Any: + """Return the value of metric ``key`` in ``record``, unwrapping ``{"value": ...}`` dicts.""" metric = record.get("metrics", {}).get(key) if isinstance(metric, dict): return metric.get("value") @@ -19,6 +20,7 @@ def _metric_value(record: dict[str, Any], key: str) -> Any: def extract_per_record_floats(records: list[dict[str, Any]], key: str) -> list[float]: + """Return the float values of metric ``key`` across ``records``, skipping missing ones.""" out: list[float] = [] for record in records: value = to_float(_metric_value(record, key)) @@ -28,6 +30,7 @@ def extract_per_record_floats(records: list[dict[str, Any]], key: str) -> list[f def extract_per_record_ints(records: list[dict[str, Any]], key: str) -> list[int]: + """Return the int values of metric ``key`` across ``records``, skipping missing ones.""" out: list[int] = [] for record in records: value = to_int(_metric_value(record, key)) @@ -37,6 +40,7 @@ def extract_per_record_ints(records: list[dict[str, Any]], key: str) -> list[int def _ms_to_s(values_ms: Iterable[float]) -> list[float]: + """Convert finite positive millisecond values to seconds, dropping the rest.""" return [ value / 1000.0 for value in values_ms @@ -45,6 +49,7 @@ def _ms_to_s(values_ms: Iterable[float]) -> list[float]: def _distribution(prefix: str, values: list[int]) -> dict[str, float]: + """Return mean, p50, p75, p90, p95 and std of integer ``values`` keyed with ``prefix``, or {} when empty.""" if not values: return {} return { @@ -58,6 +63,7 @@ def _distribution(prefix: str, values: list[int]) -> dict[str, float]: def _nest_stats(prefix: str, flat: dict[str, Any]) -> dict[str, Any]: + """Return the entries of ``flat`` ending in ``_`` with that suffix stripped.""" suffix = f"_{prefix}" return {key[: -len(suffix)]: value for key, value in flat.items() if key.endswith(suffix)} @@ -115,6 +121,7 @@ def _e2e_normalized_interactivity_stats( def compute_latency_stats( records: list[dict[str, Any]], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested TTFT, E2EL, ITL, TPOT and interactivity stats in seconds for ``records``.""" ttfts = _ms_to_s(extract_per_record_floats(records, "time_to_first_token")) e2els = _ms_to_s(extract_per_record_floats(records, "request_latency")) itls = _ms_to_s(extract_per_record_floats(records, "inter_token_latency")) @@ -161,6 +168,7 @@ def compute_latency_stats( def compute_qps_stats( records: list[dict[str, Any]], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested completions-per-second stats over 1 s windows of ``records``' end times.""" ends_ns = [ int(record["metadata"]["request_end_ns"]) for record in records @@ -192,6 +200,7 @@ def compute_qps_stats( def compute_workload_stats( records: list[dict[str, Any]], traces: Iterable[dict[str, Any]] ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested input, actual output and trace-expected output token distributions.""" input_tokens = extract_per_record_ints(records, "input_sequence_length") output_tokens = extract_per_record_ints(records, "output_sequence_length") @@ -214,6 +223,7 @@ def compute_workload_stats( def compute_throughput_stats( records: list[dict[str, Any]], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested input, output and total tokens per second over the ``records`` time span.""" input_tokens = extract_per_record_ints(records, "input_sequence_length") output_tokens = extract_per_record_ints(records, "output_sequence_length") starts_ns = [ @@ -254,6 +264,7 @@ def _aiperf_percent_metric_as_rate( aggregate: dict[str, Any], metric_name: str, ) -> float | None: + """Return aiperf ``metric_name`` from ``aggregate`` as a 0..1 rate, or None when absent.""" metric = aggregate.get(metric_name) if not isinstance(metric, dict): return None @@ -271,9 +282,9 @@ def _aiperf_percent_metric_as_rate( def compute_cache_stats( - records: list[dict[str, Any]], # noqa: ARG001 aggregate: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested theoretical prefix cache hit rate from the aiperf ``aggregate``.""" flat: dict[str, Any] = { "theoretical_cache_hit_rate": _aiperf_percent_metric_as_rate( aggregate, @@ -292,13 +303,14 @@ def compute_request_metrics( *, traces: Iterable[dict[str, Any]] = (), ) -> tuple[dict[str, Any], dict[str, Any]]: + """Return flat and nested QPS, latency, token, cache and throughput metrics for ``records``.""" aggregate = aggregate or {} flat: dict[str, Any] = {} nested: dict[str, Any] = {} qps_flat, qps_nested = compute_qps_stats(records) latency_flat, latency_nested = compute_latency_stats(records) workload_flat, workload_nested = compute_workload_stats(records, traces) - cache_flat, cache_nested = compute_cache_stats(records, aggregate) + cache_flat, cache_nested = compute_cache_stats(aggregate) throughput_flat, throughput_nested = compute_throughput_stats(records) for part in (qps_flat, latency_flat, workload_flat, cache_flat, throughput_flat): @@ -340,6 +352,7 @@ def _trace_output_lengths(traces: Iterable[dict[str, Any]]) -> dict[str, list[in def _expected_output_lengths( records: list[dict[str, Any]], traces: Iterable[dict[str, Any]] ) -> list[int]: + """Return the trace-declared output length for each record's conversation turn, skipping unmatched ones.""" metadata = _trace_output_lengths(traces) if not metadata: return [] diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index d14962a61f..0ba5ffdb55 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -21,6 +21,10 @@ def compute_server_metrics( records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: + """Return flat and nested server metrics plus warnings from aiperf's ``server_metrics`` export. + + Token totals fall back to ``records``. Raises ValueError when no backend adapter matches. + """ metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() warnings: list[str] = [] diff --git a/inferencex-e2e/infx/results/agentic/validate_agentic_result.py b/inferencex-e2e/infx/results/agentic/validate_agentic_result.py index 1d7f76df02..6e5a114c48 100644 --- a/inferencex-e2e/infx/results/agentic/validate_agentic_result.py +++ b/inferencex-e2e/infx/results/agentic/validate_agentic_result.py @@ -89,6 +89,7 @@ def validate_result(artifact_dir: Path, failed_request_threshold: float) -> list def main() -> int: + """Validate the artifact dir from argv against the error threshold and return 1 on any error, else 0.""" parser = argparse.ArgumentParser() parser.add_argument("artifact_dir", type=Path) parser.add_argument( diff --git a/inferencex-e2e/infx/results/artifacts.py b/inferencex-e2e/infx/results/artifacts.py index 0abfd5e55d..187ed1b8c5 100644 --- a/inferencex-e2e/infx/results/artifacts.py +++ b/inferencex-e2e/infx/results/artifacts.py @@ -172,6 +172,8 @@ def agentic_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("infmax_model_prefix"), row.get("framework"), row.get("precision"), + row.get("spec_decoding", "none"), + as_bool(row.get("disagg", False)), as_int(row.get("tp")), as_int(row.get("pp", 1), 1), as_int(row.get("dcp_size", 1), 1), @@ -255,7 +257,7 @@ def duplicate_identity_errors( label: str, identities: Iterable[tuple[Any, ...]], ) -> list[str]: - """Reject duplicate rows that set equality would otherwise hide.""" + """Return errors naming each ``identities`` entry seen more than once, which set equality would hide.""" counts = Counter(identities) duplicates = {identity: count for identity, count in counts.items() if count > 1} if not duplicates: @@ -276,7 +278,7 @@ def duplicate_identity_errors( def validate_fixed_artifacts( artifacts_dir: Path, ) -> list[str]: - """Validate fixed-sequence aggregate rows for duplicate identities.""" + """Return duplicate-identity errors for the fixed-sequence rows in ``artifacts_dir``.""" actual_rows = actual_benchmark_key_rows(artifacts_dir) return duplicate_identity_errors("fixed-sequence", actual_rows) @@ -284,8 +286,12 @@ def validate_fixed_artifacts( def validate_agentic_artifacts( artifacts_dir: Path, ) -> list[str]: - """Validate agentic point, raw, and aggregate artifacts agree.""" - point_rows = agentic_keys_from_paths(agentic_point_files(artifacts_dir)) + """Return errors where the agentic point, raw and aggregate artifacts in ``artifacts_dir`` disagree. + + Raw dirs of attempts replaced by a later rerun still download and are ignored. + """ + point_files = agentic_point_files(artifacts_dir) + point_rows = agentic_keys_from_paths(point_files) errors = duplicate_identity_errors("agentic point", point_rows) results_bmk = artifacts_dir / "results_bmk" @@ -301,14 +307,17 @@ def validate_agentic_artifacts( ) point_names = { - path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") - for path in agentic_point_files(artifacts_dir) + path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") for path in point_files } + superseded_names = { + path.relative_to(artifacts_dir).parts[0].removeprefix("bmk_") + for path in artifacts_dir.glob("bmk_agentic_*/**/*.json") + } - point_names raw_names = { path.name for path in artifacts_dir.iterdir() if path.is_dir() and path.name.startswith("agentic_") - } + } - superseded_names if point_names != raw_names: missing_raw = point_names - raw_names extra_raw = raw_names - point_names @@ -321,7 +330,7 @@ def validate_agentic_artifacts( def validate_run_stats(artifacts_dir: Path, required: bool) -> list[str]: - """Require run-stats when fixed-sequence collection should have run.""" + """Return an error when ``required`` and ``artifacts_dir`` holds no run-stats JSON, else [].""" if not required: return [] if list((artifacts_dir / "run-stats").glob("*.json")): diff --git a/inferencex-e2e/infx/results/collect_eval_results.py b/inferencex-e2e/infx/results/collect_eval_results.py index 334cff83cc..fb20a1d8cf 100644 --- a/inferencex-e2e/infx/results/collect_eval_results.py +++ b/inferencex-e2e/infx/results/collect_eval_results.py @@ -1,3 +1,4 @@ +import contextlib import json import sys from pathlib import Path @@ -59,12 +60,12 @@ def find_eval_sets(root: Path) -> list[Path]: extracted flat into root (no subdirectory), so check root itself too. """ out: list[Path] = [] - try: + with contextlib.suppress(Exception): if (root / "meta_env.json").exists(): out.append(root) - out.extend(d for d in root.iterdir() if d.is_dir() and (d / "meta_env.json").exists()) - except Exception: # noqa: BLE001, S110 - pass + out.extend( + d for d in sorted(root.iterdir()) if d.is_dir() and (d / "meta_env.json").exists() + ) return out @@ -83,6 +84,7 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> list[Path]: def _read_results(directory: Path) -> dict[Path, dict[str, Any]]: + """Map each readable eval result JSON in ``directory``, except meta_env.json, to its parsed content.""" return read_eval_results( (path for path in directory.glob("*.json") if path.name != "meta_env.json"), skip_errors=(Exception,), @@ -132,6 +134,7 @@ def collect_eval_rows(root: Path) -> list[dict[str, Any]]: def main() -> None: + """Print single-node and multinode eval tables for the results dir in argv and write agg_eval_.json.""" if len(sys.argv) < 3: print("Usage: collect_eval_results.py [sort_by: model_prefix|hw]") sys.exit(1) diff --git a/inferencex-e2e/infx/results/collect_events.py b/inferencex-e2e/infx/results/collect_events.py index e15ca916c2..f272f50ff8 100644 --- a/inferencex-e2e/infx/results/collect_events.py +++ b/inferencex-e2e/infx/results/collect_events.py @@ -23,8 +23,6 @@ FILENAME = "job_event.json" SCHEMA_VERSION = 2 NO_RECORD = "NoLauncherRecord" -# The names benchmark-tmpl.yml and benchmark-multinode-tmpl.yml give their jobs: an optional -# klaud marker, the priority, model prefix, precision and runner, then the concurrency last. _BENCHMARK_JOB = re.compile( r"(?:^|/ )(?:klaud \| )?p[0-9.]+ \| (?P\S+) (?P\S+) (?P\S+)" r" .* c(?P[0-9]+(?:x[0-9]+)*)(?: \| (?Peval|eval-only))?$" @@ -51,14 +49,26 @@ def read_events(directory: Path) -> list[dict[str, Any]]: def reconcile( events: list[dict[str, Any]], jobs: Iterable[dict[str, Any]], *, run_url: str ) -> list[dict[str, Any]]: - """``events`` with their jobs' outcomes, then a record for each benchmark job without one.""" + """``events`` with their jobs' outcomes, then a record for each benchmark job without one. + + Benchmark jobs are recognized by the names benchmark-tmpl.yml and benchmark-multinode-tmpl.yml give them. + """ events = [dict(event) for event in events] + starts = [_time(event.get("started_at")) for event in events] synthesized = [] for job in jobs: name = job.get("name") or "" if job.get("conclusion") in {None, "skipped"} or not _BENCHMARK_JOB.search(name): continue - event = next((event for event in events if _ran(event, job)), None) + began, ended = _time(job.get("started_at")), _time(job.get("completed_at")) + event = next( + ( + event + for event, started in zip(events, starts, strict=True) + if _ran(event, started, job, began, ended) + ), + None, + ) if event is None: synthesized.append(_synthesized(job, run_url)) elif isinstance(event.get("github"), dict): @@ -66,16 +76,18 @@ def reconcile( return [*events, *synthesized] -def _ran(event: dict[str, Any], job: dict[str, Any]) -> bool: +def _ran( + event: dict[str, Any], + started: datetime | None, + job: dict[str, Any], + began: datetime | None, + ended: datetime | None, +) -> bool: """Whether ``event`` is ``job``'s: by check-run id, else by runner and start time.""" recorded = event.get("github") job_id = recorded.get("job_id") if isinstance(recorded, dict) else None if job_id is not None: return job_id == job.get("id") - started, began, ended = ( - _time(value) - for value in (event.get("started_at"), job.get("started_at"), job.get("completed_at")) - ) return ( event.get("runner") is not None and event.get("runner") == job.get("runner_name") @@ -153,6 +165,7 @@ def _time(value: object) -> datetime | None: def main(argv: list[str] | None = None) -> int: + """Write the job events under the artifacts dir in ``argv`` as JSON Lines, reconciled with ``--run`` jobs, and return 0.""" parser = argparse.ArgumentParser( prog="python -m infx.results.collect_events", description=__doc__ ) diff --git a/inferencex-e2e/infx/results/collect_results.py b/inferencex-e2e/infx/results/collect_results.py index e30faf7972..5ad1c8a631 100644 --- a/inferencex-e2e/infx/results/collect_results.py +++ b/inferencex-e2e/infx/results/collect_results.py @@ -4,11 +4,12 @@ def main() -> None: + """Concatenate every JSON result under the results dir in argv into agg_.json.""" results_dir = Path(sys.argv[1]) exp_name = sys.argv[2] agg_results = [] - for result_path in results_dir.rglob("*.json"): + for result_path in sorted(results_dir.rglob("*.json")): with open(result_path) as f: result = json.load(f) agg_results.append(result) diff --git a/inferencex-e2e/infx/results/compare_results.py b/inferencex-e2e/infx/results/compare_results.py index ad49479b8a..d7b19cafc6 100644 --- a/inferencex-e2e/infx/results/compare_results.py +++ b/inferencex-e2e/infx/results/compare_results.py @@ -7,8 +7,11 @@ import psycopg2 from tabulate import tabulate +from infx.results.identity import public_framework + def parse_bool(value: object) -> bool: + """Return ``value`` if it is a bool, else whether its string form is ``"true"`` in any case.""" if isinstance(value, bool): return value return str(value).lower() == "true" @@ -44,8 +47,8 @@ def compute_delta_str( def extract_hardware(runner: str) -> str: - """Strip suffixes like -multinode, -trt, -disagg from runner to get hardware name.""" - return re.split(r"-(multinode|trt|disagg)$", runner)[0].lower() + """The GPU of a runner label such as cluster:b200-nscale, the key ingestion stores.""" + return runner.lower().rpartition(":")[2].split("-", 1)[0] def build_config_params(result: dict) -> dict: @@ -53,7 +56,7 @@ def build_config_params(result: dict) -> dict: is_multinode = result.get("is_multinode", False) hw = extract_hardware(result["hw"]) model = result["infmax_model_prefix"].lower() - framework = result["framework"].lower() + framework = public_framework(result["framework"]) precision = result["precision"].lower() spec_method = result.get("spec_decoding", "none").lower() disagg = parse_bool(result.get("disagg", False)) @@ -102,7 +105,7 @@ def build_config_params(result: dict) -> dict: JOIN workflow_runs wr ON wr.id = br.workflow_run_id WHERE c.hardware = %(hardware)s AND c.framework = %(framework)s - AND c.model LIKE %(model)s || '%%' + AND c.model = %(model)s AND c.precision = %(precision)s AND c.spec_method = %(spec_method)s AND c.disagg = %(disagg)s @@ -182,6 +185,7 @@ def compute_metric_delta( def main() -> None: + """Print markdown tables comparing fixed-sequence results in the argv dir to DATABASE_URL baselines.""" if len(sys.argv) < 2: print("Usage: python compare_results.py ") sys.exit(1) diff --git a/inferencex-e2e/infx/results/eval_artifacts.py b/inferencex-e2e/infx/results/eval_artifacts.py index 790b17d893..9ef9a09bde 100644 --- a/inferencex-e2e/infx/results/eval_artifacts.py +++ b/inferencex-e2e/infx/results/eval_artifacts.py @@ -31,6 +31,11 @@ def normalized_runner(value: Any) -> str: return str(value or "").lower() +def _lowered(value: Any) -> Any: + """Match aggregate rows, which lowercase framework and precision.""" + return value.lower() if isinstance(value, str) else value + + LEGACY_EVAL_SUITE = "" @@ -47,8 +52,8 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: "multi", normalized_runner(row.get("hw")), row.get("model_prefix", row.get("infmax_model_prefix")), - row.get("framework"), - row.get("precision"), + _lowered(row.get("framework")), + _lowered(row.get("precision")), row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), @@ -78,8 +83,8 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: "single", normalized_runner(row.get("hw")), row.get("model_prefix", row.get("infmax_model_prefix")), - row.get("framework"), - row.get("precision"), + _lowered(row.get("framework")), + _lowered(row.get("precision")), row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), @@ -190,10 +195,12 @@ class _RawEvalArtifact: @cached_property def results(self) -> dict[Path, dict[str, Any]]: + """Parsed results*.json files of this raw dir, keyed by path.""" return read_eval_results(self.path.glob("results*.json")) @classmethod def read(cls, path: Path) -> _RawEvalArtifact: + """Load the raw eval dir at ``path``, recording meta_env.json problems in ``errors``.""" artifact = cls(path, {}, [], False, []) prefix = f"raw eval artifact {path.name!r}" meta_path = path / "meta_env.json" @@ -400,6 +407,7 @@ def _prune_raw_eval_dir( name = artifact_dir.name def superseded(key: tuple[Any, ...]) -> bool: + """Return whether a dir other than this one won identity ``key``.""" winner = winners.get(key) return winner is not None and winner.parent.name != name diff --git a/inferencex-e2e/infx/results/evals.py b/inferencex-e2e/infx/results/evals.py index 08c765eeba..e95fdbfd44 100644 --- a/inferencex-e2e/infx/results/evals.py +++ b/inferencex-e2e/infx/results/evals.py @@ -27,6 +27,7 @@ def read_eval_results( *, skip_errors: tuple[type[Exception], ...] = (OSError, UnicodeDecodeError, json.JSONDecodeError), ) -> dict[Path, dict[str, Any]]: + """Map each of ``paths`` holding a recognized eval result to its JSON, skipping ``skip_errors`` failures.""" results = {} for path in paths: try: @@ -103,6 +104,11 @@ def select_latest_results(paths: Iterable[Path], *, batched: bool = False) -> li _SCORE_NAMES = {"strict": "em_strict", "accuracy": "accuracy", "flex": "em_flexible"} +_PARALLELISM_SIZE_KEYS = ( + "pp", "dcp_size", "pcp_size", + "prefill_pp", "prefill_dcp_size", "prefill_pcp_size", "prefill_dwdp_size", + "decode_pp", "decode_dcp_size", "decode_pcp_size", +) # fmt: skip def is_valid_score(value: object) -> bool: @@ -125,6 +131,16 @@ def is_valid_effective_count(value: object) -> bool: ) +def _is_standard_error(value: object) -> bool: + """Return whether ``value`` is a finite non-negative number, excluding booleans.""" + return ( + not isinstance(value, bool) + and isinstance(value, (int, float)) + and math.isfinite(value) + and value >= 0 + ) + + def metric_family(name: str) -> str | None: """Classify a filter name or metric key; strict takes precedence.""" if "strict" in name: @@ -135,6 +151,7 @@ def metric_family(name: str) -> str | None: def _primary_metric(metrics: dict[str, Any]) -> str | None: + """Return the first of strict, accuracy and flex set in ``metrics``, or None.""" return next((name for name in _SCORE_NAMES if metrics.get(name) is not None), None) @@ -235,7 +252,10 @@ def as_bool(x: Any, default: bool = False) -> bool: def build_row(meta: dict[str, Any], m: dict[str, Any]) -> dict[str, Any]: - """Build a result row from metadata and extracted metrics.""" + """Build a result row from metadata and extracted metrics. + + Parallelism sizes in ``meta`` are copied because raw artifact identities include them. + """ is_multinode = as_bool(meta.get("is_multinode"), False) prefill_tp = as_int(meta.get("prefill_tp", meta.get("tp", 1)), 1) prefill_ep = as_int(meta.get("prefill_ep", meta.get("ep", 1)), 1) @@ -293,6 +313,9 @@ def build_row(meta: dict[str, Any], m: dict[str, Any]) -> dict[str, Any]: if "eval_suite" in meta: row["eval_suite"] = meta["eval_suite"] + for key in _PARALLELISM_SIZE_KEYS: + if key in meta: + row[key] = as_int(meta[key], 1) primary = _primary_metric(m) row["score"] = m[primary] if primary is not None else None @@ -312,6 +335,7 @@ def build_rows( Primary scores prefer strict, accuracy, then flexible metrics. An invalid primary produces a failed row rather than falling back to a secondary score. + Non-numeric stderrs, such as lm-eval's "N/A" for one-sample metrics, become None. Inputs are not modified; discovery, concurrency selection and writes belong to the caller. """ @@ -327,6 +351,9 @@ def build_rows( "type": "InvalidPrimaryScore", "message": f"invalid primary score: {score!r}", } + for name in _SCORE_NAMES: + if not _is_standard_error(metrics[f"{name}_se"]): + metrics[f"{name}_se"] = None rows.append(build_row(meta, metrics)) return rows diff --git a/inferencex-e2e/infx/results/fixed_sequence.py b/inferencex-e2e/infx/results/fixed_sequence.py index 2a3a511230..e9a19c4adb 100644 --- a/inferencex-e2e/infx/results/fixed_sequence.py +++ b/inferencex-e2e/infx/results/fixed_sequence.py @@ -296,6 +296,10 @@ def aggregate_power_result( def process_result(env: Mapping[str, str]) -> int: + """Write agg_.json from the benchmark JSON and ``env``, join power when configured. + + Returns 1 when power is required and fails or the benchmark failed, else 0. + """ require_environment(env, _BASE_ENV_VARS) result_filename = env["RESULT_FILENAME"] bench_path = Path(f"{result_filename}.json") @@ -409,6 +413,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int: def main() -> int: + """Process every multinode point with ``--all``, else the single result named by the environment.""" return ( process_multinode_results(os.environ) if sys.argv[1:] == ["--all"] diff --git a/inferencex-e2e/infx/results/identity.py b/inferencex-e2e/infx/results/identity.py new file mode 100644 index 0000000000..cc9ea75ff0 --- /dev/null +++ b/inferencex-e2e/infx/results/identity.py @@ -0,0 +1,27 @@ +"""Benchmark identity names as InferenceX-app ingestion stores them.""" + +from __future__ import annotations + +FRAMEWORK_ALIASES = { + "atom-disagg": "mooncake-atom", + "sglang-disagg": "mori-sglang", + "trtllm": "trt", + "dynamo-trtllm": "dynamo-trt", +} +_LONGEST_FIRST = sorted(FRAMEWORK_ALIASES.items(), key=lambda alias: -len(alias[0])) + + +def public_framework(framework: str) -> str: + """The framework name the database and public API hold for a config's ``framework``. + + The aliases mirror InferenceX-app's FRAMEWORK_ALIASES used at ingestion. + """ + framework = framework.lower() + return FRAMEWORK_ALIASES.get(framework, framework) + + +def public_config_key(key: str) -> str: + """A changelog config key as ingestion stores it: every alias replaced, longest first.""" + for alias, canonical in _LONGEST_FIRST: + key = key.replace(alias, canonical) + return key diff --git a/inferencex-e2e/infx/results/power/common.py b/inferencex-e2e/infx/results/power/common.py index a9b33c2eeb..3a8fb651ee 100644 --- a/inferencex-e2e/infx/results/power/common.py +++ b/inferencex-e2e/infx/results/power/common.py @@ -27,6 +27,7 @@ class BenchmarkData: @property def integration_duration_s(self) -> float: + """Seconds between the benchmark start and end timestamps.""" return self.end_unix - self.start_unix @@ -264,6 +265,8 @@ def patch_power_metrics( ) -> None: """Validate replacement metrics before atomically updating an aggregate.""" data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"aggregate is not a JSON object: {path}") data = with_power_metrics( data, metric_keys=metric_keys, diff --git a/inferencex-e2e/infx/results/power/multinode.py b/inferencex-e2e/infx/results/power/multinode.py index b118c1245e..c857ed4e64 100644 --- a/inferencex-e2e/infx/results/power/multinode.py +++ b/inferencex-e2e/infx/results/power/multinode.py @@ -113,10 +113,12 @@ def _dedupe(values: list[str]) -> tuple[str, ...]: + """Return ``values`` as a tuple without duplicates, keeping first-seen order.""" return tuple(dict.fromkeys(values)) def _is_safe_relative_subpath(value: str) -> bool: + """Return whether ``value`` is a non-empty relative POSIX path with no ``..`` or empty parts.""" if not value or value.startswith(("/", "~")): return False parts = PurePosixPath(value).parts @@ -124,6 +126,7 @@ def _is_safe_relative_subpath(value: str) -> bool: def _stays_below(root: Path, relative: str) -> bool: + """Return whether ``root / relative`` resolves inside ``root``.""" try: (root / relative).resolve().relative_to(root.resolve()) except (ValueError, OSError): @@ -132,10 +135,12 @@ def _stays_below(root: Path, relative: str) -> bool: def _is_finite(value: Any) -> bool: + """Return whether ``value`` is a finite int or float, excluding booleans.""" return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) def _is_positive_finite(value: Any) -> bool: + """Return whether ``value`` is finite and greater than zero.""" return _is_finite(value) and value > 0 @@ -155,6 +160,7 @@ class ExpectedDevice: @property def key(self) -> tuple[str, int]: + """The (hostname, gpu_index) pair identifying this device.""" return (self.hostname, self.gpu_index) @@ -165,6 +171,7 @@ class ExpectedWindow: @property def key(self) -> tuple[str, int]: + """The (benchmark_type, concurrency) pair identifying this window.""" return (self.benchmark_type, self.concurrency) @@ -179,6 +186,7 @@ class SampleRow: @property def key(self) -> tuple[int, str, int]: + """The (scrape_seq, hostname, gpu_index) triple identifying this sample.""" return (self.scrape_seq, self.hostname, self.gpu_index) @@ -193,9 +201,11 @@ class ObservedDevice: @property def key(self) -> tuple[str, int]: + """The (hostname, gpu_index) pair identifying this device.""" return (self.hostname, self.gpu_index) def to_dict(self) -> dict: + """Return this device as a JSON-ready dict without its sample times.""" return { "hostname": self.hostname, "gpu_index": self.gpu_index, @@ -206,24 +216,31 @@ def to_dict(self) -> dict: def _text(value: Any, label: str) -> str: + """Return ``value`` if it is a non-empty string, else raise ValueError naming ``label``.""" if not isinstance(value, str) or not value: raise ValueError(f"{label} is not a non-empty string: {value!r}") return value def _whole(value: Any, label: str, *, minimum: int) -> int: + """Return ``value`` if it is an int of at least ``minimum``, else raise ValueError naming ``label``.""" if not isinstance(value, int) or isinstance(value, bool) or value < minimum: raise ValueError(f"{label} is not an integer >= {minimum}: {value!r}") return value def _role(value: Any) -> str: + """Return ``value`` if it is a known worker role, else raise ValueError.""" if value not in WORKER_ROLES: raise ValueError(f"worker_role is not one of {WORKER_ROLES}: {value!r}") return value def _parse_expected_devices(manifest: dict) -> list[ExpectedDevice]: + """Parse the manifest's expected_devices into ExpectedDevice entries. + + Raises TypeError, ValueError or KeyError on malformed entries. + """ entries = manifest.get("expected_devices") or [] if not isinstance(entries, list): raise TypeError(f"expected_devices is not a list: {entries!r}") @@ -256,6 +273,10 @@ def _parse_expected_devices(manifest: dict) -> list[ExpectedDevice]: def _parse_expected_windows(manifest: dict) -> list[ExpectedWindow]: + """Parse the manifest's expected_windows into ExpectedWindow entries. + + Raises TypeError, ValueError or KeyError on malformed entries. + """ entries = manifest.get("expected_windows") or [] if not isinstance(entries, list): raise TypeError(f"expected_windows is not a list: {entries!r}") @@ -368,6 +389,7 @@ def _parse_sample_row(raw: list[str], expected_version: int) -> SampleRow | None def read_samples(path: Path) -> tuple[tuple[SampleRow, ...], tuple[str, ...]]: + """Return the valid unique sample rows of the samples CSV at ``path`` and the reasons it is unsound.""" if not path.is_file(): return (), ("samples_csv_missing",) @@ -417,6 +439,7 @@ def read_samples(path: Path) -> tuple[tuple[SampleRow, ...], tuple[str, ...]]: def derive_observed_devices(rows: tuple[SampleRow, ...]) -> list[ObservedDevice]: + """Group sample ``rows`` per (hostname, gpu_index) into ObservedDevice entries sorted by key.""" ordered: dict[tuple[str, int], list[SampleRow]] = {} for row in rows: ordered.setdefault((row.hostname, row.gpu_index), []).append(row) @@ -441,6 +464,7 @@ def derive_observed_devices(rows: tuple[SampleRow, ...]) -> list[ObservedDevice] def _resolve_roles( devices: list[ExpectedDevice], ) -> tuple[dict[tuple[str, int], str], list[str]]: + """Map each expected device to its single worker role, or return {} and a conflict reason.""" roles: dict[tuple[str, int], str] = {} for device in devices: distinct = {assignment.worker_role for assignment in device.assignments} @@ -453,6 +477,7 @@ def _resolve_roles( def _resolve_het_groups( devices: list[ExpectedDevice], ) -> tuple[dict[str, int | None], list[str]]: + """Map each expected host to its single het group, or return {} and a conflict reason.""" groups: dict[str, int | None] = {} for device in devices: distinct = {assignment.het_group for assignment in device.assignments} @@ -469,6 +494,7 @@ def _validate_devices( expected: list[ExpectedDevice], observed: list[ObservedDevice], ) -> tuple[list[str], dict[tuple[str, int], str]]: + """Compare ``expected`` and ``observed`` devices and return the mismatch reasons and the role per device.""" reasons: list[str] = [] expected_keys = {device.key for device in expected} observed_keys = {device.key for device in observed} @@ -555,6 +581,7 @@ class ParsedWindow: def _window_status_invariants_hold(status: str, end: Any, duration: Any, reason: Any) -> bool: + """Return whether ``end``, ``duration`` and ``reason`` are consistent with window ``status``.""" if status == WINDOW_STATUS_RUNNING: return end is None and duration is None and reason is None if status == "interrupted": @@ -569,6 +596,10 @@ def _window_status_invariants_hold(status: str, end: Any, duration: Any, reason: def _parse_window( path: Path, relative: str, result_root: Path ) -> tuple[ParsedWindow | None, list[str]]: + """Parse the window JSON at ``path`` whose result must stay below ``result_root``. + + Returns the window, or None with the reason codes when it is malformed. + """ try: payload = json.loads(path.read_text()) except (OSError, ValueError): @@ -621,6 +652,10 @@ def _scan_windows( result_root: Path, artifact_errors: list[dict], ) -> tuple[dict[tuple[str, int], ParsedWindow], set[tuple[str, int]]]: + """Parse every window file in ``windows_dir`` keyed by (type, concurrency), dropping duplicated keys. + + Returns the windows and the duplicated keys, appending file problems to ``artifact_errors``. + """ parsed: dict[tuple[str, int], ParsedWindow] = {} duplicates: set[tuple[str, int]] = set() if not windows_dir.is_dir(): @@ -672,6 +707,7 @@ def _scan_windows( def _check_window_result(window: ParsedWindow, result_root: Path) -> list[str]: + """Return reasons the window's result JSON under ``result_root`` is missing or disagrees with ``window``.""" result_file = result_root / window.result_path if not result_file.is_file(): return ["measurement_window_result_missing"] @@ -697,6 +733,7 @@ def _check_window_result(window: ParsedWindow, result_root: Path) -> list[str]: def _bracketing_sequence(times: tuple[float, ...], start: float, end: float) -> list[float] | None: + """Return the sample ``times`` from the last at or before ``start`` to the first at or after ``end``, or None.""" before = [value for value in times if value <= start] after = [value for value in times if value >= end] if not before or not after: @@ -711,6 +748,7 @@ def _check_coverage( expected_device_keys: set[tuple[str, int]], observed_devices: list[ObservedDevice], ) -> tuple[dict[str, float], list[str]]: + """Return the largest sample gap per expected device across ``start``..``end`` and the coverage reasons.""" by_key = {device.key: device for device in observed_devices} gaps: dict[str, float] = {} reasons: list[str] = [] @@ -821,6 +859,7 @@ def _check_stored_evidence( validations: list[dict], artifact_errors: list[dict], ) -> list[str]: + """Return mismatches between the ``manifest``'s stored evidence and the values recomputed from disk.""" failures: list[str] = [] stored_rows = manifest.get("sample_row_count") @@ -894,12 +933,14 @@ class MultinodePowerAudit: def _canonical_sha256(payload: dict) -> str: + """Return the SHA-256 hex digest of ``payload`` as sorted, compact JSON.""" return hashlib.sha256( json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() def _add_reason(audit: MultinodePowerAudit, reason: str, detail: str | None = None) -> None: + """Record ``reason`` once on ``audit`` and append ``detail`` to its failures when given.""" _append_reason(audit.reasons, reason) if detail: audit.failures.append(detail) @@ -1239,6 +1280,7 @@ def _select_window_for_result( def _patch_agg(agg_path: Path, audit: MultinodePowerAudit) -> None: + """Write the ``audit`` verdict and power metrics into the aggregate JSON at ``agg_path``.""" patch_power_metrics( agg_path, metric_keys=_ALL_POWER_METRIC_KEYS, @@ -1254,6 +1296,7 @@ def _sidecar_payload( bench_result: Path, benchmark: BenchmarkData | None, ) -> dict: + """Build the power validation sidecar JSON from ``audit`` and the benchmark inputs.""" return { "schema_version": 1, "power_valid": audit.power_valid, @@ -1383,6 +1426,7 @@ def run( def main() -> int: + """Parse the multinode power options from argv and return the exit code of ``run``.""" parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--power-dir", diff --git a/inferencex-e2e/infx/results/result_filename.py b/inferencex-e2e/infx/results/result_filename.py index c4093589ad..ad3e421532 100644 --- a/inferencex-e2e/infx/results/result_filename.py +++ b/inferencex-e2e/infx/results/result_filename.py @@ -8,6 +8,10 @@ def compact(value: str, budget: int) -> str: + """Return ``value`` if it fits ``budget`` UTF-8 bytes, else a truncated prefix plus a hash suffix. + + Raises ValueError for empty values, path separators or a budget too small for the suffix. + """ if not value or any(char in value for char in "/\\\r\n\x00"): raise ValueError("Expected a nonempty filename component") encoded = value.encode("utf-8") @@ -20,6 +24,7 @@ def compact(value: str, budget: int) -> str: def result_stem(base: str, fingerprint: str = "") -> str: + """Return the result basename for ``base`` and an optional recipe ``fingerprint``, capped at 120 bytes.""" value = base + (f"_recipe-{fingerprint}" if fingerprint else "") return compact(value, 120) @@ -27,6 +32,10 @@ def result_stem(base: str, fingerprint: str = "") -> str: def point_filename( stem: str, config: str, conc: str, gpus: str, ctx: str = "", gen: str = "" ) -> str: + """Return the point result filename for ``stem``, ``config`` and numeric point fields within 255 bytes. + + Raises ValueError when a point field is not decimal. + """ numbers = [conc, gpus, *([ctx, gen] if ctx or gen else [])] if any(not number.isascii() or not number.isdecimal() for number in numbers): raise ValueError("Expected numeric point identity") @@ -36,6 +45,7 @@ def point_filename( def main() -> None: + """Print a point filename for ``--point``, else the result stem from RESULT_FILENAME_BASE.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--point", nargs=6, metavar=("STEM", "CONFIG", "CONC", "GPUS", "CTX", "GEN") diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index dc5b9d9b1b..f84bd6e8e4 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -24,7 +24,6 @@ ) SINGLE_NODE_ENGINES = {**ENGINES, "atom": "atom"} -# SGLang spells decode context parallelism --dcp-size, with the vLLM name as an alias. DCP_FLAGS = { "sglang": ("dcp-size", "decode-context-parallel-size"), "vllm": ("decode-context-parallel-size",), @@ -100,6 +99,8 @@ def select_recipe( validate_recipe(recipe, environment) except ValueError as exc: errors.append(f"{name}: {exc}") + except (KeyError, TypeError) as exc: + errors.append(f"{name}: missing or malformed field {exc}") else: matches.append((f"{path}:{name}" if name else path, recipe)) if len(matches) != 1: @@ -200,6 +201,8 @@ def runtime_arguments(config: str, environment: Mapping[str, str]) -> list[str]: def submission_fields(path: Path) -> tuple[str, str]: """Accept exactly one successful native JSON submission, never scrape prose.""" record = json.loads(path.read_text()) + if not isinstance(record, dict): + raise ValueError("SRT submission manifest is not a JSON object") if record.get("status") != "submitted": raise ValueError("SRT did not submit a job") job_id = str(record["slurm_job_id"]) @@ -212,6 +215,7 @@ def submission_fields(path: Path) -> tuple[str, str]: def main() -> None: + """Run ``prepare`` to bind a recipe, or ``submission`` to print a job's id and output dir.""" parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) prepare = commands.add_parser("prepare") @@ -221,6 +225,8 @@ def main() -> None: submitted = commands.add_parser("submission") submitted.add_argument("manifest", type=Path) parsed = parser.parse_args() + if parsed.command == "prepare" and parsed.output.name == BOUND_RECIPE: + parser.error(f"{parsed.output} would overwrite the bound recipe written beside it") try: if parsed.command == "prepare": selected, recipe = select_recipe(parsed.recipe, os.environ) diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 6ab1a17a40..e7da9b72bb 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -15,6 +15,8 @@ import yaml +from infx import yamlio +from infx.bench.proc import status from infx.golden_al_distribution import GOLDEN_DIR, golden_length from infx.srt_slurm.variants import ZIP_VARIANT, without_nulls @@ -42,6 +44,11 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: + """Return a role's speculative config for ``engine`` as method, draft length and draft model. + + Empty when the role does not speculate. DFlash's draft length is its verify block minus the + current token. Raises ValueError on a malformed or conflicting config. + """ args = role.get("args", {}) if engine == "tilert": if args.get("with-mtp") is not True: @@ -79,7 +86,6 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: if not algorithm: return {} if algorithm == "dflash": - # DFlash verifies a block of the current token plus its drafts. width = args.get("speculative-num-draft-tokens") return { "method": "dflash", @@ -114,7 +120,30 @@ def acceptance_length( *, golden_dir: Path = GOLDEN_DIR, ) -> float | None: - """The golden AL a speculative AgentX throughput run of ``recipe`` simulates, else None.""" + """The golden AL the generation role of a speculative AgentX throughput run simulates, + else None.""" + roles = recipe.get("roles", {}) + generation = roles.get("decode", roles.get("agg", {})) + if ( + ENGINES.get(framework) == "vllm" + and spec_parameters(generation, "vllm").get("enable_adaptive_verification") is True + ): + return None + return _simulated_length(recipe, framework, environment, golden_dir=golden_dir) + + +def _simulated_length( + recipe: Mapping[str, Any], + framework: str, + environment: Mapping[str, str], + *, + golden_dir: Path = GOLDEN_DIR, +) -> float | None: + """The golden AL ``build_overrides`` sets on non-adaptive speculative roles, else None. + + Raises ValueError unless a Qwen3.5 SGLang MTP generation role's window is its draft steps + plus one verification token. + """ engine = ENGINES.get(framework) if engine is None: return None @@ -128,13 +157,18 @@ def acceptance_length( spec = spec_parameters(generation, engine) if not (synthetic and spec): return None + if engine == "vllm" and all( + spec_parameters(role, engine).get("enable_adaptive_verification") is True + for name, role in roles.items() + if name in ("agg", "prefill", "decode") and spec_parameters(role, engine) + ): + return None al = golden_length(environment["MODEL_PREFIX"], spec, environment["THINKING_MODE"], golden_dir) if ( engine == "sglang" and environment["MODEL_PREFIX"] == "qwen3.5" and spec.get("method") in ("eagle", "nextn", "mtp") ): - # The generation role's MTP window includes one verification token. drafts = generation.get("args", {}).get("speculative-num-draft-tokens") expected = spec["num_speculative_tokens"] + 1 if not isinstance(drafts, int) or isinstance(drafts, bool) or drafts != expected: @@ -157,7 +191,7 @@ def build_overrides( if engine is None: return [] roles = recipe.get("roles", {}) - al = acceptance_length(recipe, framework, environment, golden_dir=golden_dir) + al = _simulated_length(recipe, framework, environment, golden_dir=golden_dir) overrides = [] variables = { "sglang": SGLANG_VARIABLES, @@ -294,7 +328,7 @@ def plan_commands( from srtctl.core.overrides import apply_overrides_to_recipe, parse_overrides path, _, selector = config.partition(":") - raw = yaml.safe_load(Path(path).read_text()) + raw = yamlio.safe_load(Path(path).read_text()) if not isinstance(raw, dict): raise ValueError("Recipe must be a mapping") parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) @@ -324,6 +358,7 @@ def plan_commands( def main(argv: list[str] | None = None) -> int: + """Plan srtctl commands with golden acceptance from ``argv``, run them and return the rc.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--record", type=Path, help="write the plan's golden acceptance length here as JSON" @@ -345,7 +380,7 @@ def main(argv: list[str] | None = None) -> int: for plan in plans: result = subprocess.run(plan.command, check=False) if result.returncode: - return result.returncode + return status(result.returncode) return 0 diff --git a/inferencex-e2e/infx/srt_slurm/variants.py b/inferencex-e2e/infx/srt_slurm/variants.py index 412ce54749..c9c0d43cc1 100644 --- a/inferencex-e2e/infx/srt_slurm/variants.py +++ b/inferencex-e2e/infx/srt_slurm/variants.py @@ -8,9 +8,10 @@ from __future__ import annotations import re -from copy import deepcopy from typing import Any +from infx.fastcopy import deepcopy + ZIP_VARIANT = re.compile(r"(zip_override_[\w-]+)\[(\d+)\]") @@ -28,6 +29,7 @@ def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any] def _list_lengths(group: dict[str, Any]) -> list[int]: + """Return the length of every list value in ``group``, nested mappings included.""" lengths = [] for value in group.values(): if isinstance(value, list): @@ -49,6 +51,7 @@ def _zip_length(group: dict[str, Any]) -> int: def _zip_slice(group: dict[str, Any], index: int) -> dict[str, Any]: + """Return ``group`` with each list replaced by its ``index`` item, or its only item.""" sliced = {} for key, value in group.items(): if isinstance(value, list): @@ -80,6 +83,7 @@ def _block_variants(raw: dict[str, Any], key: str) -> list[tuple[str, dict[str, def without_nulls(value: Any) -> Any: + """Return ``value`` with null-valued keys removed from every nested mapping.""" if isinstance(value, dict): return {key: without_nulls(item) for key, item in value.items() if item is not None} return value diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 6a1f2ea22e..e5eb375757 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -13,13 +13,14 @@ import os import re from collections.abc import Callable, Iterator, Mapping -from copy import deepcopy from pathlib import Path from typing import Any import yaml +from infx import yamlio from infx.config import repository_root +from infx.fastcopy import deepcopy from infx.results.metadata import parse_component_metadata from infx.srt_slurm.synthetic_acceptance import selected_recipes, spec_parameters @@ -108,6 +109,7 @@ def merge_blocks(shared: Mapping[str, Any], fragment: Mapping[str, Any]) -> dict def _lookup(block: Any, key: tuple[str, ...]) -> tuple[bool, Any]: + """Follow ``key`` into nested mappings and return whether it is present and its value.""" for part in key: if not isinstance(block, Mapping) or part not in block: return False, None @@ -158,7 +160,8 @@ def check_fragment(raw: Mapping[str, Any], source: Path, *, agentic: bool, multi def _block(path: Path) -> dict[str, Any]: - block = yaml.safe_load(path.read_text()) + """Load the YAML mapping at ``path``, or raise ValueError if it is not a mapping.""" + block = yamlio.safe_load(path.read_text()) if not isinstance(block, dict): raise ValueError(f"{path}: shared block must be a mapping") return block @@ -193,7 +196,7 @@ def compose_recipe( ``power_port`` adds the DCGM telemetry block with its exporter on that port. """ - raw = yaml.safe_load(path.read_text()) + raw = yamlio.safe_load(path.read_text()) if not isinstance(raw, dict): raise ValueError(f"{path}: recipe must be a mapping") check_fragment(raw, path, agentic=agentic, multinode=multinode) @@ -209,6 +212,7 @@ def compose_recipe( def _required(environment: Mapping[str, str], name: str) -> str: + """Return the input ``name`` from ``environment``, or raise ValueError if it is blank.""" value = environment.get(name, "") if not value.strip(): raise ValueError(f"Missing workload input: {name}") @@ -216,6 +220,7 @@ def _required(environment: Mapping[str, str], name: str) -> str: def _positive(value: str, name: str) -> int: + """Return ``value`` as a positive int, or raise ValueError naming ``name``.""" if not value.isascii() or not value.isdecimal() or int(value) <= 0: raise ValueError(f"{name} must be a positive integer: {value!r}") return int(value) @@ -407,6 +412,7 @@ def add_fabric_argument(parser: argparse.ArgumentParser) -> None: def _assignment(text: str) -> tuple[str, str]: + """Split a ``NAME=VALUE`` argument, or raise ArgumentTypeError if malformed.""" name, separator, value = text.partition("=") if not separator or not name: raise argparse.ArgumentTypeError(f"expected NAME=VALUE, got {text!r}") @@ -414,6 +420,7 @@ def _assignment(text: str) -> tuple[str, str]: def main(argv: list[str] | None = None) -> None: + """Bind the multi-node recipe variant ``argv`` names and write it to the output path.""" parser = argparse.ArgumentParser(description="Bind one multi-node recipe variant") parser.add_argument("recipe", help="fragment[:selector]") parser.add_argument("output", type=Path) diff --git a/inferencex-e2e/infx/tests/bench/test_fixed_seq.py b/inferencex-e2e/infx/tests/bench/test_fixed_seq.py index 9919f20b5c..1765ea4ce7 100644 --- a/inferencex-e2e/infx/tests/bench/test_fixed_seq.py +++ b/inferencex-e2e/infx/tests/bench/test_fixed_seq.py @@ -151,8 +151,13 @@ def sweep_env( "CONC_LIST": "4 16", "PREFILL_NUM_WORKERS": "1", "PREFILL_TP": "2", + "PREFILL_PP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_DWDP_SIZE": "1", "DECODE_NUM_WORKERS": "2", "DECODE_TP": "4", + "DECODE_PP_SIZE": "1", + "DECODE_PCP_SIZE": "1", "TOKENIZER": None, "SRT_MEASUREMENT_WINDOW_DIR": None, **overrides, @@ -340,6 +345,30 @@ def test_sweep_stops_at_the_first_failing_point(tmp_path, tools, http_server, mo assert [path.name for path in point_dir.iterdir()] == ["results_concurrency_4_gpus_4_ctx_4_gen_0.json"] +@pytest.mark.parametrize( + ("topology", "name"), + [ + ( + {"PREFILL_TP": "2", "PREFILL_PP_SIZE": "2", "DECODE_TP": "4", "DECODE_PCP_SIZE": "2"}, + "results_concurrency_4_gpus_20_ctx_4_gen_16.json", + ), + ( + {"PREFILL_NUM_WORKERS": "3", "PREFILL_TP": "1", "PREFILL_DWDP_SIZE": "4"}, + "results_concurrency_4_gpus_20_ctx_12_gen_8.json", + ), + ], +) +def test_sweep_names_every_physical_gpu_of_each_worker( + tmp_path, tools, http_server, monkeypatch, topology, name +): + url = http_server(frontend("served/model-id")) + use_env(monkeypatch, sweep_env(tmp_path, tools, url, CONC_LIST="4", **topology)) + + assert bench(["fixed-seq", "srt-sweep", "--logs-dir", str(tmp_path / "logs")]) == 0 + point_dir = tmp_path / "logs" / "sa-bench_isl_1024_osl_128" + assert [path.name for path in point_dir.iterdir()] == [name] + + def test_sweep_without_a_served_model_runs_nothing( tmp_path, tools, http_server, monkeypatch, capsys ): diff --git a/inferencex-e2e/infx/tests/bench/test_proc.py b/inferencex-e2e/infx/tests/bench/test_proc.py new file mode 100644 index 0000000000..00f93878c3 --- /dev/null +++ b/inferencex-e2e/infx/tests/bench/test_proc.py @@ -0,0 +1,38 @@ +"""Child-process helpers: the echoed command line and start failures.""" + +from __future__ import annotations + +from infx.bench import proc + + +def test_echo_masks_credentials_but_not_token_count_settings(capsys): + env = { + "HF_TOKEN": "hf_abcdef", + "GITHUB_TOKEN": "ghp_123456", + "OPENAI_API_KEY": "sk-proj-9", + "CLIENT_SECRET": "shh-quiet", + "DB_PASSWORD": "hunter22", + "TOKENIZER": "/models/tok", + "MAX_NUM_BATCHED_TOKENS": "8192", + } + argv = ["run", "--tokenizer", "/models/tok", "--max-tokens=8192"] + argv += ["hf_abcdef", "ghp_123456", "api_key=sk-proj-9", "shh-quiet", "hunter22"] + + proc.echo(argv, env) + + assert capsys.readouterr().err == ( + "+ run --tokenizer /models/tok --max-tokens=8192 *** *** api_key=*** *** ***\n" + ) + + +def test_tee_reports_a_command_that_cannot_start(tmp_path, capsys): + missing = str(tmp_path / "missing") + log = tmp_path / "out.log" + + rc = proc.tee([missing, "--flag"], log) + + assert rc == 127 + err = capsys.readouterr().err + assert err.startswith(f"+ {missing} --flag\nERROR: cannot run {missing}: ") + assert "No such file or directory" in err + assert log.read_bytes() == b"" diff --git a/inferencex-e2e/infx/tests/bench/test_server.py b/inferencex-e2e/infx/tests/bench/test_server.py index 1675ccfb8f..72cb8261e2 100644 --- a/inferencex-e2e/infx/tests/bench/test_server.py +++ b/inferencex-e2e/infx/tests/bench/test_server.py @@ -7,6 +7,7 @@ import sys import time from pathlib import Path +from types import SimpleNamespace import pytest @@ -41,6 +42,48 @@ def respond(method: str, path: str) -> tuple[int, object]: assert capsys.readouterr().out == "loading\nGET /health\n" +@pytest.mark.parametrize( + ("before", "mode", "written", "out"), + [(b"loading\n", "wb", b"ok\n", "loading\nok\n"), (b"caf\xc3", "ab", b"\xa9\n", "caf\u00e9\n")], + ids=["truncated", "split-utf8"], +) +def test_wait_follows_a_truncated_log_and_split_characters( + http_server, tmp_path, capsys, before, mode, written, out +): + log = tmp_path / "server.log" + log.write_bytes(before) + statuses = iter([503, 200]) + + def respond(_method: str, _path: str) -> tuple[int, object]: + status = next(statuses) + if status == 503: + with log.open(mode) as stream: + stream.write(written) + return status, {} + + server.wait_ready(f"{http_server(respond)}/health", log=log, poll_s=0.01) + + assert capsys.readouterr().out == out + + +def test_wait_restarts_a_log_replaced_by_a_longer_file(http_server, tmp_path, capsys): + log = tmp_path / "server.log" + log.write_bytes(b"old\n") + statuses = iter([503, 200]) + + def respond(_method: str, _path: str) -> tuple[int, object]: + status = next(statuses) + if status == 503: + fresh = tmp_path / "server.log.new" + fresh.write_bytes(b"restarted server\n") + fresh.replace(log) + return status, {} + + server.wait_ready(f"{http_server(respond)}/health", log=log, poll_s=0.01) + + assert capsys.readouterr().out == "old\nrestarted server\n" + + def test_wait_fails_once_the_server_process_is_gone(): dead = subprocess.Popen(["true"]) dead.wait() @@ -55,6 +98,23 @@ def test_wait_fails_once_the_server_process_is_gone(): assert result.stderr == f"ERROR: process {dead.pid} died before {url} became ready\n" +def test_wait_streams_what_a_dying_server_wrote_before_failing(http_server, tmp_path, capsys): + log = tmp_path / "server.log" + log.write_text("loading\n") + dead = subprocess.Popen(["true"]) + dead.wait() + + def respond(_method: str, _path: str) -> tuple[int, object]: + with log.open("a") as out: + out.write("CUDA out of memory\n") + return 503, {} + + with pytest.raises(server.NotReadyError): + server.wait_ready(f"{http_server(respond)}/health", pid=dead.pid, log=log, poll_s=0.01) + + assert capsys.readouterr().out == "loading\nCUDA out of memory\n" + + @pytest.mark.parametrize( ("models", "health", "stabilization_s", "waiting"), [(["m"], 503, 0, False), (["other"], 200, 1, True)], @@ -74,6 +134,24 @@ def test_chat_route_is_ready_once_the_model_is_served_or_health_holds( assert capsys.readouterr().out == f"{waited}OpenAI chat endpoint ready for model 'm': {chat}\n" +def test_chat_route_keeps_waiting_through_a_null_model_list(http_server, capsys): + health = iter([503, 200]) + + def respond(_method: str, path: str) -> tuple[int, object]: + if path == "/v1/models": + return 200, {"data": None} + return (next(health), {}) if path == "/health" else (404, {}) + + url = http_server(respond) + chat = f"{url}/v1/chat/completions" + + server.wait_chat_route(url, "m", (5, 0), poll_s=0.01) + + assert capsys.readouterr().out == ( + f"Waiting for {chat} ('m'): 0/5s\nOpenAI chat endpoint ready for model 'm': {chat}\n" + ) + + def test_chat_route_gives_up_after_its_timeout(http_server, capsys): url = http_server(frontend(None, chat=404, health=503)) chat = f"{url}/v1/chat/completions" @@ -83,3 +161,27 @@ def test_chat_route_gives_up_after_its_timeout(http_server, capsys): assert str(raised.value) == f"chat endpoint for model 'm' not ready within 1s: {chat}" assert capsys.readouterr().out == f"Waiting for {chat} ('m'): 0/1s\n" + + +def test_chat_route_deadline_counts_time_spent_in_slow_probes(http_server, monkeypatch, capsys): + clock = SimpleNamespace(now=0.0) + + def sleep(seconds: float) -> None: + clock.now += seconds + + monkeypatch.setattr(server, "time", SimpleNamespace(monotonic=lambda: clock.now, sleep=sleep)) + respond = frontend(None, chat=404, health=503) + + def slow_health(method: str, path: str) -> tuple[int, object]: + if path == "/health": + clock.now += 8 + return respond(method, path) + + url = http_server(slow_health) + + with pytest.raises(server.NotReadyError): + server.wait_chat_route(url, "m", (10, 0), poll_s=1) + + assert clock.now == 17 + chat = f"{url}/v1/chat/completions" + assert capsys.readouterr().out == f"Waiting for {chat} ('m'): 8/10s\n" diff --git a/inferencex-e2e/infx/tests/bench_serving/test_backend_request_func.py b/inferencex-e2e/infx/tests/bench_serving/test_backend_request_func.py new file mode 100644 index 0000000000..720cf09258 --- /dev/null +++ b/inferencex-e2e/infx/tests/bench_serving/test_backend_request_func.py @@ -0,0 +1,92 @@ +"""OpenAI streaming request functions against a local SSE server.""" + +import asyncio +import json + +import pytest +from aiohttp import web + +from infx.bench_serving import backend_request_func as req + +CHOICE = { + "completions": {"choices": [{"text": "hi"}]}, + "chat/completions": {"choices": [{"delta": {"content": "hi"}}]}, +} +REQUEST = { + "completions": req.async_request_openai_completions, + "chat/completions": req.async_request_openai_chat_completions, +} + + +def stream(endpoint: str, events: list[object]) -> req.RequestFuncOutput: + """Serve ``events`` as one SSE stream on HTTP 200 and run the request function on it. + + ``bytes`` events are written verbatim, anything else as one ``data:`` event. + """ + + async def handler(request: web.Request) -> web.StreamResponse: + await request.read() + response = web.StreamResponse() + await response.prepare(request) + for event in events: + if isinstance(event, bytes): + await response.write(event) + continue + data = event if isinstance(event, str) else json.dumps(event) + await response.write(f"data: {data}\n\n".encode()) + await response.write_eof() + return response + + async def run() -> req.RequestFuncOutput: + app = web.Application() + app.router.add_post(f"/v1/{endpoint}", handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = runner.addresses[0][1] + try: + return await REQUEST[endpoint]( + req.RequestFuncInput( + prompt="p", + api_url=f"http://127.0.0.1:{port}/v1/{endpoint}", + prompt_len=1, + output_len=4, + model="m", + ) + ) + finally: + await runner.cleanup() + + return asyncio.run(run()) + + +@pytest.mark.parametrize("endpoint", ["completions", "chat/completions"]) +def test_error_event_mid_stream_fails_the_request(endpoint): + error = {"error": {"message": "context length exceeded", "code": 400}} + + output = stream(endpoint, [CHOICE[endpoint], error, "[DONE]"]) + + assert output.success is False + assert "context length exceeded" in output.error + + +def test_chat_stream_without_a_token_chunk_fails_the_request(): + usage = {"choices": [], "usage": {"completion_tokens": 0}} + + output = stream("chat/completions", [usage, "[DONE]"]) + + assert output.success is False + assert output.error.startswith("Never received a valid chunk") + + +@pytest.mark.parametrize("endpoint", ["completions", "chat/completions"]) +def test_sse_comment_and_event_lines_are_not_parsed_as_data(endpoint): + keepalive = b": keep-alive\n\n" + typed = b"event: message\n" + + output = stream(endpoint, [keepalive, typed, CHOICE[endpoint], keepalive, "[DONE]"]) + + assert output.error == "" + assert output.success is True + assert output.generated_text == "hi" diff --git a/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py b/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py index 507e3c5d50..2eacb4ed17 100644 --- a/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py +++ b/inferencex-e2e/infx/tests/bench_serving/test_benchmark_serving.py @@ -1,5 +1,6 @@ """Exercise client result persistence without loading a model or serving requests.""" +import asyncio import json from argparse import Namespace from unittest.mock import AsyncMock @@ -9,6 +10,33 @@ from infx.bench_serving import benchmark_serving as client +def cli_args(tmp_path, **overrides): + """``main`` arguments for a one-prompt random run saved to ``tmp_path/result.json``.""" + return Namespace(**{ + 'seed': 0, 'backend': 'vllm', 'model': 'fixture', 'served_model_name': None, + 'tokenizer': None, 'tokenizer_mode': 'auto', 'base_url': 'http://unused', + 'endpoint': '/completions', 'trust_remote_code': False, 'dataset_name': 'random', + 'random_prefix_len': 0, 'random_input_len': 1, 'random_output_len': 1, 'num_prompts': 1, + 'random_range_ratio': 1, 'use_chat_template': False, 'dsv4': False, + 'random_num_workers': 1, 'goodput': None, 'logprobs': None, 'best_of': 1, + 'request_rate': float('inf'), 'burstiness': 1, 'disable_tqdm': True, 'num_warmups': 0, + 'profile': False, 'percentile_metrics': 'ttft', 'metric_percentiles': '99', + 'ignore_eos': False, 'max_concurrency': 1, 'lora_modules': None, 'save_result': True, + 'metadata': None, 'save_detailed': True, 'result_filename': 'result.json', + 'result_dir': str(tmp_path), **overrides, + }) + + +def stub_benchmark(monkeypatch, raw): + """Skip tokenizer loading, prompt sampling and serving; ``benchmark`` returns ``raw``.""" + benchmark = AsyncMock(return_value=raw) + monkeypatch.setattr(client, '_load_tokenizer', lambda *a, **kw: object()) + monkeypatch.setattr(client, 'sample_random_requests', lambda **kw: []) + monkeypatch.setattr(client, 'benchmark', benchmark) + monkeypatch.setattr(client.gc, 'freeze', lambda: None) + return benchmark + + @pytest.mark.parametrize('requested,completed,status', [ (100, 100, 'passed'), (100, 95, 'passed'), (100, 94, 'failed'), (100, 0, 'failed'), (0, 0, 'failed'), (100, 101, 'failed'), @@ -18,28 +46,14 @@ def test_client_preserves_outcome_before_failure( tmp_path, monkeypatch, requested, completed, status, save_result, ): - args = Namespace( - seed=0, backend='vllm', model='fixture', served_model_name=None, - tokenizer=None, tokenizer_mode='auto', base_url='http://unused', endpoint='/completions', - trust_remote_code=False, dataset_name='random', random_prefix_len=0, - random_input_len=1, random_output_len=1, num_prompts=requested, random_range_ratio=1, - use_chat_template=False, dsv4=False, random_num_workers=1, goodput=None, - logprobs=None, best_of=1, request_rate=float('inf'), burstiness=1, - disable_tqdm=True, num_warmups=0, profile=False, percentile_metrics='ttft', - metric_percentiles='99', ignore_eos=False, max_concurrency=1, lora_modules=None, - save_result=save_result, metadata=None, save_detailed=True, - result_filename='result.json', result_dir=str(tmp_path), - ) + args = cli_args(tmp_path, num_prompts=requested, save_result=save_result) metrics = {name: 0 for name in ( 'median_ttft_ms', 'mean_ttft_ms', 'std_ttft_ms', 'p99_ttft_ms', 'mean_tpot_ms', 'median_tpot_ms', 'std_tpot_ms', 'p99_tpot_ms', 'median_itl_ms', 'mean_itl_ms', 'std_itl_ms', 'p99_itl_ms', )} raw = {**metrics, 'completed': completed, 'errors': ['retained diagnostic']} - monkeypatch.setattr(client, '_load_tokenizer', lambda *a, **kw: object()) - monkeypatch.setattr(client, 'sample_random_requests', lambda **kw: []) - monkeypatch.setattr(client, 'benchmark', AsyncMock(return_value=raw)) - monkeypatch.setattr(client.gc, 'freeze', lambda: None) + stub_benchmark(monkeypatch, raw) invalid = requested == 0 or completed is None or completed < 0 or completed > requested if status == 'failed': @@ -68,3 +82,131 @@ def test_client_preserves_outcome_before_failure( assert 'error' not in outcome assert outcome['failed'] == {100: 0, 95: 5, 94: 6, 0: 100}[completed] assert outcome['max_failure_rate'] == 0.05 + + +def test_metadata_value_keeps_its_equals_signs(tmp_path, monkeypatch): + stub_benchmark(monkeypatch, {'completed': 1}) + + client.main(cli_args(tmp_path, metadata=['cmd=--a=b', ' tp = 8 '])) + + saved = json.loads((tmp_path / 'result.json').read_text()) + assert saved['cmd'] == '--a=b' + assert saved['tp'] == '8' + + +@pytest.mark.parametrize('rate', [0.0, -2.0]) +def test_non_positive_request_rate_is_rejected_before_the_run(tmp_path, monkeypatch, rate): + benchmark = stub_benchmark(monkeypatch, {'completed': 1}) + + with pytest.raises(ValueError, match=f'--request-rate must be positive, got {rate}'): + client.main(cli_args(tmp_path, request_rate=rate)) + + assert benchmark.await_count == 0 + + +def test_missing_result_dir_is_rejected_before_the_run(tmp_path, monkeypatch): + benchmark = stub_benchmark(monkeypatch, {'completed': 1}) + missing = tmp_path / 'missing' + + with pytest.raises(ValueError, match=f'--result-dir {missing} is not an existing directory'): + client.main(cli_args(tmp_path, result_dir=str(missing))) + + assert benchmark.await_count == 0 + assert not missing.exists() + + +def test_random_sampling_rejects_zero_prompts(): + with pytest.raises(ValueError, match='--num-prompts must be positive, got 0'): + client.sample_random_requests( + prefix_len=0, input_len=4, output_len=4, num_prompts=0, range_ratio=1.0, + tokenizer=Namespace(vocab_size=10), num_workers=1, + ) + + +class WordTokenizer: + """One token per whitespace-separated word.""" + + def __call__(self, text, add_special_tokens): + return Namespace(input_ids=text.split()) + + +def test_metrics_retokenize_a_success_without_reported_usage(): + outputs = [ + client.RequestFuncOutput( + generated_text='one two three four', success=True, latency=1.0, ttft=0.1, + ), + client.RequestFuncOutput( + generated_text='ignored', success=True, latency=1.0, ttft=0.1, output_tokens=7, + ), + ] + requests = [('p', 5, 4, None), ('p', 5, 7, None)] + + metrics, output_lens = client.calculate_metrics( + requests, outputs, 2.0, WordTokenizer(), [50], {}, + ) + + assert output_lens == [4, 7] + assert metrics.total_output == 11 + assert metrics.output_throughput == 5.5 + assert metrics.mean_tpot_ms == pytest.approx((0.9 / 3 + 0.9 / 6) / 2 * 1000) + + +@pytest.mark.parametrize('enabled', [True, False]) +def test_pytorch_records_skip_metrics_the_run_did_not_select(tmp_path, monkeypatch, enabled): + if enabled: + monkeypatch.setenv('SAVE_TO_PYTORCH_BENCHMARK_FORMAT', '1') + else: + monkeypatch.delenv('SAVE_TO_PYTORCH_BENCHMARK_FORMAT', raising=False) + results = {'mean_ttft_ms': 12.5, 'p90_ttft_ms': 20.0, 'completed': 3} + result_file = tmp_path / 'result.json' + + client.save_to_pytorch_benchmark_format(Namespace(model='m'), results, str(result_file)) + + records_file = tmp_path / 'result.pytorch.json' + if not enabled: + assert not records_file.exists() + return + [record] = json.loads(records_file.read_text()) + assert record['metric']['name'] == 'mean_ttft_ms' + assert record['metric']['benchmark_values'] == [12.5] + assert record['metric']['extra_info'] == {'p90_ttft_ms': 20.0, 'completed': 3} + + +def run_benchmark(monkeypatch, ttfts, **overrides): + """Run ``benchmark`` against a fake server and clock; each prompt's request takes 2 s.""" + clock = Namespace(now=0.0) + monkeypatch.setattr(client, 'time', Namespace(perf_counter=lambda: clock.now, time=lambda: 0)) + + async def serve(request_func_input, pbar=None): + cost = {'/start_profile': 0.0, '/stop_profile': 100.0} + clock.now += cost.get(request_func_input.api_url.removeprefix('http://x'), 2.0) + ttft = ttfts.get(request_func_input.prompt, 0.0) + return client.RequestFuncOutput(success=True, latency=1.0, ttft=ttft, output_tokens=3) + + monkeypatch.setitem(client.ASYNC_REQUEST_FUNCS, 'fake', serve) + kwargs = { + 'backend': 'fake', 'api_url': 'http://x/v1/completions', 'base_url': 'http://x', + 'model_id': 'm', 'model_name': None, 'tokenizer': None, 'logprobs': None, 'best_of': 1, + 'input_requests': [(prompt, 5, 3, None) for prompt in ttfts], + 'request_rate': float('inf'), 'burstiness': 1.0, 'disable_tqdm': True, 'num_warmups': 0, + 'profile': False, 'selected_percentile_metrics': ['ttft'], 'selected_percentiles': [99], + 'ignore_eos': False, 'goodput_config_dict': {}, 'max_concurrency': None, + 'lora_modules': None, **overrides, + } + return asyncio.run(client.benchmark(**kwargs)) + + +def test_goodput_counts_requests_within_the_slo(monkeypatch): + result = run_benchmark( + monkeypatch, {'fast': 0.1, 'slow': 0.5}, goodput_config_dict={'ttft': 200.0}, + ) + + assert result['duration'] == 4.0 + assert result['request_goodput'] == 0.25 + + +def test_profiler_shutdown_is_not_part_of_the_measured_duration(monkeypatch): + result = run_benchmark(monkeypatch, {'a': 0.1, 'b': 0.1}, profile=True) + + assert result['duration'] == 4.0 + assert result['request_throughput'] == 0.5 diff --git a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py index 4fc4ac0c38..dc0d4df1fe 100644 --- a/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py +++ b/inferencex-e2e/infx/tests/datasets/test_build_weka_hf_dataset.py @@ -1,7 +1,173 @@ from __future__ import annotations +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from infx.datasets import build_weka_hf_dataset from infx.datasets.build_weka_hf_dataset import _filter_trace_256k +STUB_SAMPLER = """ +import json, os, sys +from pathlib import Path + +argv = sys.argv[1:] +out = Path(argv[argv.index("--out") + 1]) +Path(os.environ["STUB_SAMPLER_RECORD"]).write_text( + json.dumps({"argv": argv, "db_url": os.environ.get("AGENTIC_PROXY_DB_URL")}) +) +for n, sid in enumerate(os.environ["STUB_SESSIONS"].split(",")): + if n == int(os.environ.get("STUB_SAMPLER_CRASH_AT", "-1")): + sys.exit(1) + row = { + "timestamp": "2026-05-01T00:00:00+00:00", + "t_sec": 0.0, + "model": "claude-test", + "input_tokens": 10, + "output_tokens": 5, + "duration_ms": 1000, + "hash_ids": ["h1"], + } + if sid == os.environ.get("STUB_BAD_SESSION"): + del row["model"] + (out / f"{sid}.jsonl").write_text(json.dumps(row) + "\\n") +(out / "manifest.json").write_text("{}") +""" + + +STUB_PLOT = """ +import os, sys +from pathlib import Path + +if os.environ.get("STUB_PLOT_FAIL"): + sys.exit(1) +out = Path(sys.argv[sys.argv.index("--out-dir") + 1]) +(out / f"{Path(sys.argv[0]).stem}.png").write_bytes(b"png") +""" + + +@pytest.fixture +def build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Run the build CLI with the DB sampler stubbed to write STUB_SESSIONS and the plot + scripts stubbed to write a PNG, or to fail when STUB_PLOT_FAIL is set.""" + sampler = tmp_path / "stub_sampler.py" + sampler.write_text(STUB_SAMPLER) + record = tmp_path / "sampler-record.json" + plot = tmp_path / "stub_plot.py" + plot.write_text(STUB_PLOT) + monkeypatch.delenv("STUB_PLOT_FAIL", raising=False) + monkeypatch.delenv("STUB_SAMPLER_CRASH_AT", raising=False) + monkeypatch.delenv("STUB_BAD_SESSION", raising=False) + monkeypatch.setattr(build_weka_hf_dataset, "SAMPLER", sampler) + monkeypatch.setattr(build_weka_hf_dataset, "PLOT_WEKA", plot) + monkeypatch.setattr(build_weka_hf_dataset, "PLOT_SUBAGENT", plot) + monkeypatch.setenv("STUB_SAMPLER_RECORD", str(record)) + # setenv first so teardown also removes the value main() exports. + monkeypatch.setenv("AGENTIC_PROXY_DB_URL", "") + monkeypatch.delenv("AGENTIC_PROXY_DB_URL") + + def run(sessions: str, *flags: str) -> dict: + monkeypatch.setenv("STUB_SESSIONS", sessions) + argv = ["build", "--repo-base", "org/base", "--work-dir", str(tmp_path / "work")] + monkeypatch.setattr(sys, "argv", [*argv, "--skip-upload", *flags]) + assert build_weka_hf_dataset.main() == 0 + return json.loads(record.read_text()) + + return run + + +def _trace_ids(traces_jsonl: Path) -> list[str]: + return [json.loads(line)["id"] for line in traces_jsonl.read_text().splitlines()] + + +def test_rebuild_in_same_work_dir_publishes_only_this_runs_sessions(build, tmp_path: Path) -> None: + filters = ("--repo-256k", "org/256k", "--base-max-isl", "990016") + build("old-a,old-b", *filters) + build("new-c", *filters) + + work = tmp_path / "work" + assert _trace_ids(work / "base/traces.jsonl") == ["new-c"] + assert _trace_ids(work / "256k/traces.jsonl") == ["new-c"] + + +def test_failed_plots_do_not_republish_the_previous_builds_pngs( + build, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + build("s1") + plots = tmp_path / "work/base/plots" + assert [p.name for p in plots.iterdir()] == ["stub_plot.png"] + + monkeypatch.setenv("STUB_PLOT_FAIL", "1") + build("s2") + + assert list(plots.iterdir()) == [] + assert _trace_ids(tmp_path / "work/base/traces.jsonl") == ["s2"] + + +def test_skip_sample_does_not_reuse_a_crashed_samplers_partial_dump( + build, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("STUB_SAMPLER_CRASH_AT", "1") + with pytest.raises(subprocess.CalledProcessError): + build("partial-a,partial-b") + monkeypatch.delenv("STUB_SAMPLER_CRASH_AT") + + build("full-a,full-b", "--skip-sample") + + assert _trace_ids(tmp_path / "work/base/traces.jsonl") == ["full-a", "full-b"] + + +def test_skip_convert_does_not_publish_a_crashed_converters_partial_output( + build, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("STUB_BAD_SESSION", "b-bad") + with pytest.raises(subprocess.CalledProcessError): + build("a-good,b-bad") + assert (tmp_path / "work/per_trace/a-good.json").is_file() + + with pytest.raises(subprocess.CalledProcessError): + build("unused", "--skip-sample", "--skip-convert") + + assert not (tmp_path / "work/base").exists() + + +def test_db_url_reaches_sampler_without_being_echoed( + build, capfd: pytest.CaptureFixture[str] +) -> None: + url = "postgresql://reader:hunter2-secret@db.example/proxy" + record = build("s1", "--db-url", url) + + assert record["db_url"] == url + assert all("hunter2-secret" not in arg for arg in record["argv"]) + out, err = capfd.readouterr() + assert "hunter2-secret" not in out + err + + +def test_random_sampling_forwards_seed_to_sampler_and_readme(build, tmp_path: Path) -> None: + record = build("s1", "--sampling", "random", "--seed", "7", "--limit", "3") + + argv = record["argv"] + assert argv[argv.index("--sampling") + 1] == "random" + assert argv[argv.index("--seed") + 1] == "7" + readme = (tmp_path / "work/base/README.md").read_text() + assert "--seed 7" in readme + + +def test_dwbug_min_peak_without_the_filter_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + argv = ["build", "--repo-base", "org/base", "--work-dir", str(tmp_path), "--dwbug-min-peak", "2"] + monkeypatch.setattr(sys, "argv", argv) + + with pytest.raises(SystemExit) as exit_info: + build_weka_hf_dataset.parse_args() + + assert exit_info.value.code == 2 + assert "--dwbug-min-peak requires --exclude-dynamic-workflow-bug" in capsys.readouterr().err + def _request(t: float, api_time: float, tokens: int = 1) -> dict: return { @@ -86,3 +252,18 @@ def test_partial_subagent_filter_recomputes_group_metadata() -> None: assert child["t"] == 4.0 assert child["duration_ms"] == 5_000 assert child["total_tokens"] == 300 + + +def test_filter_drops_models_left_without_requests() -> None: + opus_main = {**_request(0.0, 1.0, tokens=300_000), "model": "opus"} + sonnet_main = {**_request(1.0, 1.0), "model": "sonnet"} + haiku_inner = {**_request(2.0, 1.0), "model": "haiku"} + opus_inner = {**_request(3.0, 1.0, tokens=300_000), "model": "opus"} + child = {**_subagent("child", [haiku_inner, opus_inner]), "models": ["haiku", "opus"]} + trace = {"id": "models", "models": ["haiku", "opus", "sonnet"]} + trace["requests"] = [opus_main, sonnet_main, child] + + filtered = _filter_trace_256k(trace, cap=256_000) + + assert filtered["requests"][1]["models"] == ["haiku"] + assert filtered["models"] == ["haiku", "sonnet"] diff --git a/inferencex-e2e/infx/tests/datasets/test_plot_distributions.py b/inferencex-e2e/infx/tests/datasets/test_plot_distributions.py new file mode 100644 index 0000000000..95e3deeafa --- /dev/null +++ b/inferencex-e2e/infx/tests/datasets/test_plot_distributions.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from types import ModuleType + +import pytest +from matplotlib.figure import Figure + +from infx.datasets import plot_subagent_distributions, plot_weka_distributions + + +@pytest.mark.parametrize("module", [plot_weka_distributions, plot_subagent_distributions]) +@pytest.mark.parametrize("values", [[0.0002, 0.0005], [64.0, 64.0, 64.0]]) +def test_log_histogram_counts_every_positive_value( + module: ModuleType, values: list[float] +) -> None: + ax = Figure().subplots() + + module._draw_histogram( + ax=ax, + values=values, + title="metric", + xlabel="x", + bins=40, + log_x=True, + value_fmt="{:,.4f}", + linear_clip_pct=99.0, + ) + + assert sum(bar.get_height() for bar in ax.patches) == len(values) diff --git a/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py b/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py new file mode 100644 index 0000000000..d4c650cece --- /dev/null +++ b/inferencex-e2e/infx/tests/datasets/test_sample_proxy_traces.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import argparse +import sys +import types +from datetime import UTC, datetime + +import pytest + +if "psycopg" not in sys.modules: + # The sampler's DB driver is not a test dependency; nothing here opens a connection. + _psycopg = types.ModuleType("psycopg") + _psycopg.rows = types.ModuleType("psycopg.rows") + _psycopg.rows.dict_row = object() + sys.modules["psycopg"] = _psycopg + sys.modules["psycopg.rows"] = _psycopg.rows + +from infx.datasets.sample_proxy_traces import ( # noqa: E402 + _redact_url, + _sort_and_limit, + parse_args, +) + +TIED_TS = datetime(2026, 5, 1, tzinfo=UTC) +OLDER_TS = datetime(2026, 4, 1, tzinfo=UTC) + + +@pytest.mark.parametrize("sampling", ["top", "recent"]) +def test_limit_picks_the_same_sessions_whatever_order_the_db_returns(sampling: str) -> None: + rows = [ + {"session_id": "sess-b", "req_count": 10, "last_ts": TIED_TS}, + {"session_id": "sess-a", "req_count": 10, "last_ts": TIED_TS}, + {"session_id": "sess-c", "req_count": 5, "last_ts": OLDER_TS}, + ] + args = argparse.Namespace(sampling=sampling, limit=1, seed=None) + + picked = [_sort_and_limit(order, args)[0]["session_id"] for order in (rows, rows[::-1])] + + assert picked == ["sess-a", "sess-a"] + + +def test_zero_limit_is_rejected( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + argv = ["sample", "--out", "out", "--sampling", "top", "--limit", "0"] + monkeypatch.setattr(sys, "argv", argv) + + with pytest.raises(SystemExit) as exit_info: + parse_args() + + assert exit_info.value.code == 2 + assert "--limit must be >= 1" in capsys.readouterr().err + + +def test_keyword_conninfo_password_is_not_logged() -> None: + redacted = _redact_url("host=db.example dbname=proxy user=reader password=hunter2-secret") + + assert "hunter2-secret" not in redacted diff --git a/inferencex-e2e/infx/tests/evals/test_batched_eval.py b/inferencex-e2e/infx/tests/evals/test_batched_eval.py index 245e22dc1b..ea6888496e 100644 --- a/inferencex-e2e/infx/tests/evals/test_batched_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_batched_eval.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +import pytest + from infx.evals.validate_scores import main as validate_scores_main from infx.evals.validate_scores import validate_batch_manifest @@ -212,3 +214,223 @@ def test_validate_scores_accepts_legacy_result_without_effective_count( assert validate_scores_main() == 0 captured = capsys.readouterr() assert "PASS: gsm8k exact_match,strict-match" in captured.out + + +@pytest.mark.parametrize( + ("score", "shown"), [(float("nan"), "nan"), (float("inf"), "inf"), (True, "True")] +) +def test_validate_scores_fails_a_non_finite_or_boolean_score( + tmp_path: Path, + monkeypatch, + capsys, + score: object, + shown: str, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({"results": {"gsm8k": {"exact_match,strict-match": score}}})) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: gsm8k exact_match,strict-match = {shown} is not a finite number" in captured.err + assert "PASS" not in captured.out + + +def test_validate_scores_fails_when_thresholds_cannot_be_loaded( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + thresholds = tmp_path / "thresholds.yaml" + thresholds.write_text("default: [gsm8k\n") + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.87}}})) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--thresholds", + str(thresholds), + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: could not load thresholds from {thresholds}: invalid YAML" in captured.err + assert "PASS" not in captured.out + + +@pytest.mark.parametrize( + ("thresholds_text", "expected"), + [ + ( + "default:\n gsm8k: 0.9\nmodels:\n", + "FAIL: gsm8k exact_match,strict-match = 0.8700 (< 0.9 from default)", + ), + ( + "default:\n gsm8k: 0.9\nmodels:\n dsr1:\n", + "FAIL: gsm8k exact_match,strict-match = 0.8700 (< 0.9 from default)", + ), + ( + "default:\n gsm8k: 0.9\nmodels: [dsr1]\n", + "thresholds config 'models' must be a mapping", + ), + ( + "default: []\n", + "thresholds config 'default' must be a mapping", + ), + ( + "default:\n gsm8k: 0.9\nmodels:\n dsr1: 0\n", + "thresholds config 'models.dsr1' must be a mapping", + ), + ( + "default:\n gsm8k: .nan\n", + "thresholds config default.gsm8k must be a finite number, got nan", + ), + ( + "default:\n gsm8k: 0.8\nmodels:\n dsr1:\n gsm8k: '0.9'\n", + "thresholds config models.dsr1.gsm8k must be a finite number, got '0.9'", + ), + ( + "default:\n gsm8k: 0.8\nmodel:\n dsr1:\n gsm8k: 0.9\n", + "thresholds config has unknown section(s): model", + ), + ], +) +def test_validate_scores_handles_empty_or_malformed_threshold_sections( + tmp_path: Path, + monkeypatch, + capsys, + thresholds_text: str, + expected: str, +) -> None: + thresholds = tmp_path / "thresholds.yaml" + thresholds.write_text(thresholds_text) + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.87}}})) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--thresholds", + str(thresholds), + "--model-prefix", + "dsr1", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + assert expected in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("content", "reason"), + [ + (b'{"results": {"gsm8k": ', "invalid JSON"), + (b"[1]", "expected a JSON object with a 'results' mapping"), + (b'{"results": null}', "expected a JSON object with a 'results' mapping"), + (b"\xff{}", "'utf-8' codec can't decode byte 0xff"), + ], +) +def test_validate_scores_fails_a_malformed_result_file_and_checks_the_rest( + tmp_path: Path, + monkeypatch, + capsys, + content: bytes, + reason: str, +) -> None: + bad = tmp_path / "results_a.json" + bad.write_bytes(content) + (tmp_path / "results_b.json").write_text( + json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.95}}}) + ) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(tmp_path / "results*.json"), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: unreadable result file {bad}: {reason}" in captured.err + assert "PASS: gsm8k exact_match,strict-match = 0.9500" in captured.out + + +@pytest.mark.parametrize( + ("data", "reason"), + [ + ({"results": {"gsm8k": None}}, "'results' entry is not a mapping"), + ({"configs": [], "results": {"gsm8k": {}}}, "'configs' is not a mapping"), + ({"configs": {"gsm8k": []}, "results": {"gsm8k": {}}}, "'configs' entry is not a mapping"), + ( + {"configs": {"gsm8k": {"metric_list": 5}}, "results": {"gsm8k": {}}}, + "'metric_list' is not a list", + ), + ], +) +def test_validate_scores_fails_a_malformed_task_entry_and_checks_the_rest( + tmp_path: Path, + monkeypatch, + capsys, + data: dict, + reason: str, +) -> None: + (tmp_path / "results_a.json").write_text(json.dumps(data)) + (tmp_path / "results_b.json").write_text( + json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.95}}}) + ) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(tmp_path / "results*.json"), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert f"FAIL: gsm8k malformed result: {reason}" in captured.err + assert "PASS: gsm8k exact_match,strict-match = 0.9500" in captured.out + + +def test_validate_scores_fails_a_result_file_with_no_checked_metric( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "eval_concs": [1, 4], + "completed_eval_concs": [1, 4], + "failed_eval_concs": [], + })) + (tmp_path / "results_test_conc1.json").write_text( + json.dumps({"results": {"gsm8k": {"exact_match,strict-match": 0.95}}}) + ) + (tmp_path / "results_test_conc4.json").write_text( + json.dumps({"results": {"gsm8k": {"acc,none": 0.95}}}) + ) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(tmp_path / "results*.json"), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "PASS: [conc=1] gsm8k exact_match,strict-match = 0.9500" in captured.out + assert ( + "FAIL: [conc=4] no metrics matched 'declared task metrics' in " + f"{tmp_path / 'results_test_conc4.json'}" + ) in captured.err diff --git a/inferencex-e2e/infx/tests/evals/test_bfcl_eval.py b/inferencex-e2e/infx/tests/evals/test_bfcl_eval.py index 92197eca70..0f0fb0efdf 100644 --- a/inferencex-e2e/infx/tests/evals/test_bfcl_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_bfcl_eval.py @@ -982,8 +982,9 @@ def test_kimi_suite_caps_multi_turn_steps(monkeypatch: pytest.MonkeyPatch) -> No assert prompts.MAXIMUM_STEP_LIMIT == 10 +@pytest.mark.parametrize("model", ["model-a", "zai-org/glm5_2"]) def test_upstream_registration_uses_exact_stock_openai_handler( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, model: str ) -> None: project_root = tmp_path / "bfcl" be._write_id_map(project_root, be.SMOKE_CASE_IDS) @@ -1008,8 +1009,13 @@ def generate(**_: Any) -> None: encoding="utf-8", ) - def evaluate(**_: Any) -> None: - pass + evaluated: list[str] = [] + + def evaluate(**kwargs: Any) -> None: + # Upstream evaluation reloads configs from the "/"-to-"_" result dir name. + [name] = kwargs["model"] + escaped = name.replace("/", "_").replace("_", "/") + evaluated.append(model_config_mapping[escaped].model_name) modules = { "bfcl_eval": ModuleType("bfcl_eval"), @@ -1047,14 +1053,15 @@ def evaluate(**_: Any) -> None: monkeypatch.setattr(be, "_function_defaults", lambda _: {}) be._run_upstream( - model="model-a", + model=model, project_root=project_root, base_url="http://127.0.0.1:8000/v1", api_key="EMPTY", num_threads=4, ) - assert model_config_mapping["model-a"].model_handler is OpenAICompletionsHandler + assert model_config_mapping[model].model_handler is OpenAICompletionsHandler + assert evaluated == [model] def test_source_mount_archives_license_without_installed_distribution(tmp_path: Path) -> None: diff --git a/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py b/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py index f2429cc8fa..d32c8ff2dc 100644 --- a/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py +++ b/inferencex-e2e/infx/tests/evals/test_kimi_verifier_archive.py @@ -66,7 +66,9 @@ def _archive(*, missing: str | None = None, extra: tarfile.TarInfo | None = None @contextmanager -def _serve(payload: bytes, *, transient_failures: int = 0) -> Iterator[tuple[str, list[str]]]: +def _serve( + payload: bytes, *, transient_failures: int = 0, truncate: str | None = None +) -> Iterator[tuple[str, list[str]]]: paths: list[str] = [] class Handler(BaseHTTPRequestHandler): @@ -77,8 +79,16 @@ def do_GET(self) -> None: self.end_headers() return self.send_response(200) + if truncate == "chunked" and len(paths) == transient_failures + 1: + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(f"{len(payload):x}\r\n".encode() + payload[:10]) + return self.send_header("Content-Length", str(len(payload))) self.end_headers() + if truncate == "short" and len(paths) == transient_failures + 1: + self.wfile.write(payload[: len(payload) // 2]) + return self.wfile.write(payload) def log_message(self, *args: object) -> None: @@ -102,11 +112,15 @@ def _fetch( *, sha256: str | None = None, transient_failures: int = 0, + truncate: str | None = None, ) -> list[str]: """Fetch ``payload`` from a local archive server into ``checkout``; return the paths asked.""" checkout.mkdir() monkeypatch.setattr(archive.time, "sleep", lambda _: None) - with _serve(payload, transient_failures=transient_failures) as (repo_url, paths): + with _serve(payload, transient_failures=transient_failures, truncate=truncate) as ( + repo_url, + paths, + ): digest = sha256 or hashlib.sha256(payload).hexdigest() monkeypatch.setattr(sys, "argv", ["archive", repo_url, REF, digest, str(checkout)]) archive.main() @@ -127,6 +141,32 @@ def test_extracts_only_the_required_subset_after_a_transient_server_error( assert (checkout / "pyproject.toml").read_text() == "pyproject.toml" +@pytest.mark.parametrize("truncate", ["short", "chunked"]) +def test_retries_a_download_cut_short_by_the_connection( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], truncate: str +) -> None: + checkout = tmp_path / "checkout" + + paths = _fetch(monkeypatch, checkout, _archive(), truncate=truncate) + + assert len(paths) == 2 + assert "archive download attempt 1/3 failed" in capsys.readouterr().err + extracted = {p.relative_to(checkout).as_posix() for p in checkout.rglob("*") if p.is_file()} + assert extracted == REQUIRED_FILES + + +def test_accepts_an_uppercase_pinned_archive_sha256( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + checkout = tmp_path / "checkout" + payload = _archive() + + paths = _fetch(monkeypatch, checkout, payload, sha256=hashlib.sha256(payload).hexdigest().upper()) + + assert len(paths) == 1 + assert (checkout / "pyproject.toml").read_text() == "pyproject.toml" + + def _escaping_member() -> tarfile.TarInfo: member = tarfile.TarInfo("verifier-pinned/../../escaped") member.size = 6 diff --git a/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py b/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py index 35f32bd1a3..1a8c1ced97 100644 --- a/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_minimax_m3_full_eval.py @@ -138,6 +138,40 @@ def open(self, request, *, timeout): assert sleeps == [0.5] +def test_fetch_source_retries_a_truncated_download(monkeypatch: pytest.MonkeyPatch) -> None: + content = b"pinned source" + monkeypatch.setattr( + full, "REQUIRED_SOURCE_SHA256", {"verify.py": hashlib.sha256(content).hexdigest()} + ) + monkeypatch.setattr(full.time, "sleep", lambda _: None) + bodies = [content[:6], content] + + class Response: + status = 200 + headers: dict[str, str] = {} + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def geturl(self) -> str: + return full.source_url("verify.py") + + def read(self, _: int) -> bytes: + return bodies.pop(0) + + class Opener: + def open(self, request, *, timeout): + return Response() + + monkeypatch.setattr(full, "_NO_REDIRECT_OPENER", Opener()) + + assert full._fetch_source("verify.py") == content + assert bodies == [] + + def test_verifier_command_is_one_102_row_run_with_fixed_m3_settings( tmp_path: Path, ) -> None: @@ -271,6 +305,23 @@ def test_projection_rejects_incomplete_or_transport_failed_native_results( full.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3") +@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85"]) +def test_projection_counts_rows_whose_text_holds_unicode_line_separators( + tmp_path: Path, separator: str +) -> None: + _write_native_run(tmp_path) + rows = [{"data_index": index, "status": "success"} for index in range(1, 103)] + rows[4]["response"] = f"first{separator}second" + (tmp_path / full.NATIVE_RESULTS_FILENAME).write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8" + ) + + compatibility_path = full.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3") + + compatibility = json.loads(compatibility_path.read_text(encoding="utf-8")) + assert compatibility["n-samples"]["minimax_m3_full"] == {"original": 102, "effective": 102} + + def test_upstream_process_failure_is_nonzero_and_publishes_all_failure_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py b/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py index 87c3e7f5ff..3333e7b8b0 100644 --- a/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py +++ b/inferencex-e2e/infx/tests/evals/test_minimax_provider_eval.py @@ -48,6 +48,21 @@ def _compatibility(output_dir: Path) -> dict[str, Any]: return json.loads(paths[0].read_text(encoding="utf-8")) +def test_projection_reads_a_row_whose_text_holds_a_unicode_line_separator( + tmp_path: Path, +) -> None: + _native_outputs(tmp_path, match_rate=0.5) + (tmp_path / mpe.NATIVE_RESULTS_FILENAME).write_text( + json.dumps({"data_index": 1, "status": "success", "response": "a\u2028b"}, ensure_ascii=False) + + "\n", + encoding="utf-8", + ) + + mpe.project_native_artifacts(output_dir=tmp_path, model="MiniMax-M3") + + assert _compatibility(tmp_path)["results"][mpe.TASK_NAME]["exact_match,strict-match"] == 0.5 + + def test_prepare_smoke_input_preserves_request_and_unicode( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py b/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py index 1f6046cb9d..dbab825262 100644 --- a/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py +++ b/inferencex-e2e/infx/tests/golden_al_distribution/test_golden_al_distribution.py @@ -90,3 +90,31 @@ def test_cli_list_and_show(capsys) -> None: def test_cli_reports_errors(capsys) -> None: assert main(["show", "missing_curve"]) == 1 assert "No committed golden curve" in capsys.readouterr().err + + +def test_cli_show_rejects_a_curve_outside_the_golden_dir(tmp_path, capsys) -> None: + golden = tmp_path / "golden" + golden.mkdir() + (tmp_path / "outside.yaml").write_text("some-model:\n thinking_on:\n 3: 2.5\n") + + assert main(["--golden-dir", str(golden), "show", "../outside"]) == 1 + assert "Invalid golden curve identity" in capsys.readouterr().err + + +def test_a_thinking_mode_written_as_a_list_is_rejected(tmp_path) -> None: + (tmp_path / "m_mtp.yaml").write_text("m:\n thinking_on: [1.5, 2.3, 2.9]\n") + spec = {"method": "mtp", "num_speculative_tokens": 2} + + with pytest.raises(ValueError, match="thinking_on of golden curve m_mtp.yaml"): + golden_length("m", spec, "thinking_on", tmp_path) + + +def test_cli_show_marks_every_cell_of_an_empty_thinking_mode(tmp_path, capsys) -> None: + (tmp_path / "m_mtp.yaml").write_text("m:\n thinking_on:\n 1: 1.5\n thinking_off:\n") + + assert main(["--golden-dir", str(tmp_path), "show", "m_mtp"]) == 0 + assert capsys.readouterr().out.splitlines() == [ + "m_mtp (m)", + "tokens thinking_on thinking_off", + "1 1.50 -", + ] diff --git a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py index 9a2ddc22f8..167346a1d0 100644 --- a/inferencex-e2e/infx/tests/klaud/test_klaud_github.py +++ b/inferencex-e2e/infx/tests/klaud/test_klaud_github.py @@ -65,6 +65,37 @@ def run(args, **kwargs): assert github.write("example/project", "git/refs/heads/claim", "DELETE") == {} +def test_session_listings_see_the_state_after_a_write(monkeypatch): + session = lifecycle.Session( + "example/project", {"id": 42}, OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40) + ) + refs = [{"ref": f"refs/heads/{session.branch}"}] + monkeypatch.setattr(github, "items", lambda *_args: list(refs)) + monkeypatch.setattr(github.github, "api", lambda *_args, **_kwargs: refs.clear() or {}) + + assert session.branch_exists() is True + github.write("example/project", f"git/refs/heads/{session.branch}", "DELETE") + assert session.branch_exists() is False + + +@pytest.mark.parametrize(("contents", "data"), [ + ({"type": "file", "encoding": "base64", "size": 3, "content": "YWJj\n"}, b"abc"), + ([{"type": "file", "name": "recipe.yaml", "path": "recipes/recipe.yaml"}], None), +], ids=["file", "directory"]) # fmt: skip +def test_source_reads_return_file_bytes_or_a_fixed_failure(monkeypatch, contents, data): + def run(args, **_kwargs): + assert args[4] == f"repos/example/project/contents/recipes?ref={'a' * 40}" + return subprocess.CompletedProcess(args, 0, json.dumps(contents), "") + + monkeypatch.setattr(github.subprocess, "run", run) + + if data is None: + with pytest.raises(github.VerificationError, match="Source data unavailable or too large"): + github.file_at("example/project", "a" * 40, "recipes") + else: + assert github.file_at("example/project", "a" * 40, "recipes") == data + + @pytest.mark.parametrize( "failure,reason", [ @@ -222,6 +253,23 @@ def test_finish_requires_specific_baseline_reason_before_touching_session( ) +@pytest.mark.parametrize("command", ["report", "check-final"]) +def test_pr_commands_fail_cleanly_before_the_candidate_pr_exists( + tmp_path, monkeypatch, capfd, command +): + record = tmp_path / "record.json" + record.write_text(reporting.Baseline( + family=FAMILY, date="2026-09-19", image="example/image:1", + goal=reporting.Prose(en="Update the image.", zh="更新镜像。"), sources=[], points=[], + ).model_dump_json(by_alias=True)) # fmt: skip + options = (["--kind", "baseline", "--file"] if command == "report" else ["--matrix-file"]) + monkeypatch.setattr(sys, "argv", ["klaud", command, *options, str(record)]) + monkeypatch.setattr(lifecycle, "current_session", lambda: SimpleNamespace(pulls=lambda: [])) + + assert klaud.main() == 1 + assert capfd.readouterr().out == "::error::Klaud: Candidate PR not found.\n" + + def test_cooldown_skips_recent_blocks_across_bases_and_retries_capacity_deferrals( monkeypatch, ): @@ -828,6 +876,108 @@ def test_recovery_keeps_the_pr_claim_and_lease_of_a_rerun_final_sweep(monkeypatc "sweep). [Parent run / 父运行](https://github.com/example/project/actions/runs/42)\n\n") +@pytest.mark.parametrize("stale", ["deleted-claim-parent", "corrupt-artifact"]) +def test_recovery_skips_an_unreadable_owner_and_reconciles_the_others( + monkeypatch, tmp_path, capsys, stale +): + world = ended_session_sweep(monkeypatch, jobs=["success", "cancelled"]) + family = "configs/amd-master.yaml:other" + other = OwnedCandidate(id=f"{identity(family)[:16]}-{'6' * 16}", family=family, base="a" * 40) + owners = [Ownership(run_id=42, candidates=[world.session.candidate])] + if stale == "deleted-claim-parent": + owners.append(Ownership(run_id=77, candidates=[other])) + monkeypatch.setattr(claims, "family_owners", lambda _r: owners) + serve, read = github.items, github.read + + def items(repository, path, key=None): + if path.startswith("actions/artifacts?") and stale == "corrupt-artifact": + return [{"id": 5, "expired": False, "size_in_bytes": 10, "workflow_run": {"id": 42}}] + return serve(repository, path, key) + + def gone(repository, path): + if path == "actions/runs/77": + raise subprocess.CalledProcessError(1, ["gh", "api"]) + return read(repository, path) + + def corrupt(*_args): + raise github.VerificationError("Invalid artifact archive") + + monkeypatch.setattr(github, "items", items) + monkeypatch.setattr(github, "read", gone) + monkeypatch.setattr(github, "download_json", corrupt) + + summary, _ = recovery_pass(monkeypatch, tmp_path) + + assert world.writes == [("POST", "actions/runs/9/rerun-failed-jobs")] + rerun = (f"Klaud `{world.session.candidate.id}`: pending / 等待后续收尾 (Reran preempted " + "final sweep). [Parent run / 父运行](https://github.com/example/project/actions/runs/42)" + "\n\n") + if stale == "deleted-claim-parent": + assert json.loads((tmp_path / "klaud-recovery.json").read_text()) == [ + "configs/amd-master.yaml:other", FAMILY] + assert summary.read_text() == ( + f"Klaud `{other.id}`: needs inspection / 需要检查 (state unavailable or invalid). " + "[Parent run / 父运行](https://github.com/example/project/actions/runs/77)\n\n" + rerun) + else: + assert json.loads((tmp_path / "klaud-recovery.json").read_text()) == [FAMILY] + assert summary.read_text() == rerun + assert "::warning::Klaud: ownership artifact 5 is unreadable; skipped.\n" in ( + capsys.readouterr().out) + + +def test_recovery_leaves_the_draft_of_a_planner_run_it_could_not_read(monkeypatch, tmp_path): + world = ended_session_sweep(monkeypatch, sessions=("failure",)) + read, failed = github.read, [] + + def flaky(repository, path): + if path == "actions/runs/42" and not failed: + failed.append(path) + raise subprocess.CalledProcessError(1, ["gh", "api"]) + return read(repository, path) + + monkeypatch.setattr(github, "read", flaky) + + recovery_pass(monkeypatch, tmp_path) + + assert world.writes == world.released == [] + assert json.loads((tmp_path / "klaud-recovery.json").read_text()) == [FAMILY] + + +def test_abandoned_work_sweep_keeps_a_claim_whose_planner_run_is_gone(monkeypatch): + world = ended_session_sweep(monkeypatch, sessions=("failure",)) + candidate = world.session.candidate + family = "configs/amd-master.yaml:other" + other = OwnedCandidate(id=f"{identity(family)[:16]}-{'6' * 16}", family=family, base="a" * 40) + monkeypatch.setattr(claims, "family_owners", lambda _r: [ + Ownership(run_id=42, candidates=[candidate]), Ownership(run_id=77, candidates=[other])]) + read, serve = github.read, github.items + draft = {**world.pull, "number": 8, + "head": {**world.pull["head"], "ref": f"klaud/auto-{other.id}"}} + + def gone(repository, path): + if path == "actions/runs/77": + raise subprocess.CalledProcessError(1, ["gh", "api"]) + return read(repository, path) + + def items(repository, path, key=None): + if path == "pulls?state=open&per_page=100": + return [world.pull, draft] + if path == "pulls/8/commits?per_page=100": + return [{"committer": {"login": "github-actions[bot]"}}] + return serve(repository, path, key) + + monkeypatch.setattr(github, "read", gone) + monkeypatch.setattr(github, "items", items) + + assert lifecycle.sweep_abandoned("example/project", set()) == {other.id} + assert world.released == [(candidate.id, 42)] + assert world.writes == [ + ("DELETE", f"git/refs/heads/klaud/recovery-42-{candidate.id}"), + ("POST", "issues/7/comments"), ("PATCH", "pulls/7"), + ("DELETE", f"git/refs/heads/klaud/auto-{candidate.id}"), + ] + + def test_recovery_warns_why_a_passed_final_sweep_failed_validation(monkeypatch, capsys): world = ended_session_sweep(monkeypatch, attempt=2, conclusion="success", artifacts=[{ "id": 1, "name": "klaud-sweep-manifest", "expired": False, @@ -1281,7 +1431,7 @@ def test_select_continues_after_one_baseline_state_failure(tmp_path, monkeypatch ] } - def resolve(_repository, candidate, _context, _model, _goal, _producers): + def resolve(_repository, candidate, _context, _model, _goal, _producers, _matrix): if candidate.id == first_id: raise OSError("transient") return reporting.Baseline( @@ -1344,7 +1494,7 @@ def test_select_needs_headroom_on_every_target_cluster_and_shares_it_within_a_wa for context in contexts ]} # fmt: skip - def resolve(_repository, candidate, _context, _model, _goal, _producers): + def resolve(_repository, candidate, _context, _model, _goal, _producers, _matrix): return reporting.Baseline( family=candidate.family, date="2026-09-19", image="example/image:1", goal=reporting.Prose(en="Check the baseline.", zh="检查基线。"), @@ -1389,7 +1539,7 @@ def test_select_leaves_the_slots_of_resumed_sessions_to_them( for context in contexts ]} # fmt: skip - def resolve(_repository, candidate, _context, _model, _goal, _producers): + def resolve(_repository, candidate, _context, _model, _goal, _producers, _matrix): return reporting.Baseline( family=candidate.family, date="2026-09-19", image="example/image:1", goal=reporting.Prose(en="Check the baseline.", zh="检查基线。"), @@ -1517,8 +1667,12 @@ def test_prepare_baseline_keeps_the_baseline_the_candidate_pr_froze(tmp_path, mo published if frozen else preflight.model_copy(update={"goal": goal})) +@pytest.mark.parametrize(("family_key", "published_key", "fingerprint"), [ + ("test-family", "test-family", "fingerprint"), + ("model-fp8-h200-trtllm-mtp", "model-fp8-h200-trt-mtp", None), +], ids=["fingerprinted", "legacy-row-under-aliased-key"]) # fmt: skip def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( - monkeypatch, + monkeypatch, family_key, published_key, fingerprint ): base = "a" * 40 historical_head = "b" * 40 @@ -1527,7 +1681,7 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( entry = { "model-prefix": "Model", "runner": "h200", - "framework": "trtllm", + "framework": "trt", "precision": "fp8", "spec-decoding": "mtp", "disagg": False, @@ -1556,7 +1710,7 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( public_row = { "model": "Model", "hardware": "h200", - "framework": "trtllm", + "framework": "trt", "precision": "fp8", "spec_method": "mtp", "disagg": False, @@ -1575,24 +1729,19 @@ def test_baseline_normalizes_enroot_image_and_rejects_unverified_provenance( "decode_ep": 1, "decode_dp_attention": False, "decode_num_workers": 0, - "recipe_fingerprint": "fingerprint", + "recipe_fingerprint": fingerprint, "run_url": "https://github.com/example/project/actions/runs/42", "tput_per_gpu": 10.0, "output_tput_per_gpu": 8.0, "mean_ttft": 0.1, "mean_tpot": 0.02, - "errors": 0, } info = { "runs": [{"github_run_id": "42", "run_attempt": 1}], "runConfigs": [{"github_run_id": "42", "head_sha": historical_head}], - "changelogs": [{"workflow_run_id": "42", "config_keys": ["test-family"]}], + "changelogs": [{"workflow_run_id": "42", "config_keys": [published_key]}], } - def canonical_matrix(_repository, head, _family): - assert head == base - return current - feeds = { "benchmarks": [ {**public_row, "image": None}, @@ -1610,12 +1759,11 @@ def fetch(resource, **_kwargs): payload=feeds[resource], ) - monkeypatch.setattr(validation, "canonical_matrix", canonical_matrix) monkeypatch.setattr(api, "fetch", fetch) producers = {historical_head: historical_matrix} candidate = OwnedCandidate( id="1" * 16 + "-" + "2" * 16, - family="configs/nvidia-master.yaml:test-family", + family=f"configs/nvidia-master.yaml:{family_key}", base=base, ) context = { @@ -1624,7 +1772,7 @@ def fetch(resource, **_kwargs): "image": public_image, "model": "Model", "hardware": "h200", - "framework": "trtllm", + "framework": "trt", "precision": "fp8", "spec_method": "mtp", "disagg": False, @@ -1638,6 +1786,7 @@ def fetch(resource, **_kwargs): "Model", reporting.Prose(en="Update the image.", zh="更新镜像。"), producers, + current, ) assert [(point.conc, point.result) for point in baseline.points] == [ @@ -1661,6 +1810,7 @@ def fetch(resource, **_kwargs): "Model", reporting.Prose(en="Update the image.", zh="更新镜像。"), producers, + current, ) @@ -1698,6 +1848,26 @@ def test_multinode_identity_matches_published_aggregate_encodings( assert reporting._matches_public_point(row, reporting.public_point(entry)) is matches +def test_sglang_disagg_identity_matches_its_published_mori_sglang_rows(): + worker = {"num-worker": 1, "tp": 4, "ep": 1, "dp-attn": False} + entry = { + "model-prefix": "dsv4", "runner": "cluster:mi355x-amds", "framework": "sglang-disagg", + "precision": "fp4", "spec-decoding": "draft_model", "disagg": True, + "scenario-type": "agentic-coding", "kv-offloading": "dram", "conc": 16, + "image": "example/image:1", "prefill": worker, "decode": worker, + } # fmt: skip + row = { + "model": "dsv4", "hardware": "mi355x", "framework": "mori-sglang", "precision": "fp4", + "spec_method": "draft_model", "disagg": True, "is_multinode": True, + "benchmark_type": "agentic_traces", "isl": None, "osl": None, "offload_mode": "on", + "conc": 16, "image": "example/image:1", "prefill_tp": 4, "prefill_ep": 1, + "prefill_dp_attention": False, "prefill_num_workers": 1, "decode_tp": 4, + "decode_ep": 1, "decode_dp_attention": False, "decode_num_workers": 1, + } # fmt: skip + + assert reporting._matches_public_point(row, reporting.public_point(entry)) is True + + @pytest.fixture def family_configs(): master = { @@ -1795,7 +1965,7 @@ def publish(monkeypatch, head): "decode_ep": 1, "decode_dp_attention": False, "decode_num_workers": 0, "run_url": "https://github.com/example/project/actions/runs/42", "tput_per_gpu": 5.0 * conc, "output_tput_per_gpu": 4.0 * conc, "mean_ttft": 0.1, - "mean_tpot": 0.02, "errors": 0, + "mean_tpot": 0.02, } for conc in (2, 6) ] @@ -1966,7 +2136,7 @@ def serve_dsr1_baseline(monkeypatch, current, markdown): "recipe_fingerprint": f"{isl:064x}", "run_url": "https://github.com/example/project/actions/runs/42", "tput_per_gpu": 10.0, "output_tput_per_gpu": 8.0, "mean_ttft": 0.1, - "mean_tpot": 0.02, "errors": 0, + "mean_tpot": 0.02, } for isl in (1024, 8192) ] @@ -2089,6 +2259,7 @@ def test_final_coverage_exempts_retired_points_and_reports_their_evidence(monkey "DeepSeek-R1", reporting.Prose(en="Update the image.", zh="更新镜像。"), producers, + {"single_node": {"all": current}}, ) retired, kept = baseline.points @@ -2179,11 +2350,12 @@ def test_baselines_without_retirements_keep_the_earlier_record_format(): def frozen_dsr1_baseline(monkeypatch): """The planner's baseline for a base family of 8k/1k c1 and c2 whose MODELS.md deprecated 1k/1k after the 2026-05-22 publication: points are 1k/1k c1 (retired), 8k/1k c1 and c2.""" - producers, source = serve_dsr1_baseline(monkeypatch, [dsr1_point(8192, [1, 2])], models_md()) + current = [dsr1_point(8192, [1, 2])] + producers, source = serve_dsr1_baseline(monkeypatch, current, models_md()) return reporting.resolve_baseline( "example/project", OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40), {"source": source}, "DeepSeek-R1", reporting.Prose(en="Update the image.", zh="更新镜像。"), - producers, + producers, {"single_node": {"all": current}}, ) @@ -2286,7 +2458,7 @@ def attempt_comment(number, en, zh, *, author="Klaud-Cold", parent=42, candidate f"\nRendered.")} -def finish_report(monkeypatch, outcome, comments, recovering=False): +def finish_report(monkeypatch, outcome, comments, recovering=False, repairs=0): session = lifecycle.Session( "example/project", {"id": 42}, OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40), recovering=recovering, @@ -2312,13 +2484,56 @@ def finish_report(monkeypatch, outcome, comments, recovering=False): session.finish(CandidateOutcome( outcome=outcome, phase="final-sweep" if validated else "targeted", pull_request=7, - run_ids=[9], repairs_used=0, + run_ids=[9], repairs_used=repairs, )) [report] = [body for body in posted if body.startswith(session.marker)] return report.split("\n-->\n", 1)[1] +@pytest.mark.parametrize("ended", [True, False], ids=["ended-meanwhile", "still-running"]) +def test_finish_tolerates_only_a_cancel_refused_because_the_run_already_ended( + monkeypatch, ended +): + session = lifecycle.Session( + "example/project", {"id": 42}, OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40) + ) + pull = {"number": 7, "head": {"sha": "c" * 40}, "merged_at": None, "draft": True, + "state": "closed", "labels": []} + run = {"id": 9, "event": "pull_request", "head_sha": "c" * 40, "run_attempt": 1, + "created_at": "2026-10-15T00:00:00Z", "status": "in_progress", "conclusion": None} + finished = {**run, "status": "completed", "conclusion": "success"} + current, writes = [run], [] + + def write(_repository, path, method, _payload=None): + writes.append((method, path)) + if path == "actions/runs/9/cancel": + if ended: + current[0] = finished + raise subprocess.CalledProcessError(1, ["gh", "api"]) + return {} + + monkeypatch.setattr(session, "pulls", lambda: [pull]) + monkeypatch.setattr(session, "runs", lambda: list(current)) + monkeypatch.setattr(session, "refresh", lambda _pull: pull) + monkeypatch.setattr(claims, "release_family", lambda *_: None) + monkeypatch.setattr(github, "items", lambda *_args: []) + monkeypatch.setattr(github, "read", lambda _repository, path: ( + current[0] if path == "actions/runs/9" else pytest.fail(path))) + monkeypatch.setattr(github, "write", write) + outcome = CandidateOutcome(outcome="failed", phase="targeted", pull_request=7, run_ids=[9], + repairs_used=0) + + if ended: + assert session.finish(outcome).run_ids == [9] + assert writes == [("POST", "issues/7/comments"), ("POST", "actions/runs/9/cancel"), + ("POST", "issues/7/comments")] + else: + with pytest.raises(subprocess.CalledProcessError): + session.finish(outcome) + assert writes == [("POST", "issues/7/comments"), ("POST", "actions/runs/9/cancel")] + + @pytest.mark.parametrize(("labels", "accepted"), [ (["full-sweep-fail-fast"], True), (["full-sweep-enabled"], False), @@ -2402,6 +2617,14 @@ def test_closing_report_states_why_from_the_latest_own_attempt( ) +def test_closing_report_translates_an_unknown_repair_count(monkeypatch): + report = finish_report(monkeypatch, "uncertain", [], recovering=True, repairs=None) + english, chinese = report.split("中文") + + assert "· Repairs: unknown ·" in english + assert "· 修复次数:未知 ·" in chinese + + def test_every_candidate_pr_body_says_how_to_stop_the_candidate(monkeypatch): body = reporting.render_body(frozen_dsr1_baseline(monkeypatch)) english, chinese = body.split("中文") @@ -2562,6 +2785,21 @@ def test_live_families_preserve_public_identity_after_move(tmp_path, family_conf } +def test_live_families_name_atom_disagg_as_its_published_mooncake_atom(tmp_path, family_configs): + master, runners = family_configs + master["fixture"]["framework"] = "atom-disagg" + configs = tmp_path / "configs" + configs.mkdir() + (configs / "amd-master.yaml").write_text(yaml.safe_dump(master)) + (configs / "runners.yaml").write_text(yaml.safe_dump(runners)) + write_fixture_recipe(tmp_path) + + families = klaud.live_families(tmp_path) + + assert [key[2] for key in families] == ["mooncake-atom"] + + + def test_historical_families_ignore_leftover_nested_results(tmp_path, family_configs): master, runners = family_configs configs = tmp_path / "configs" @@ -2697,6 +2935,110 @@ def test_agentx_deltas_compare_only_the_same_replay_corpus(old, new, cell, note) assert notes == ([f"**Note:** All rows: Δ N/A: {note}."] if note else []) +@pytest.mark.parametrize(("completed", "errors", "note"), [ + pytest.param(100, 0, None, id="no-failed-requests"), + pytest.param(97, 3, "**Note:** All rows: 3 request errors.", id="failed-requests"), +]) # fmt: skip +def test_a_fixed_sequence_result_reports_its_failed_requests(completed, errors, note): + from infx.results.fixed_sequence import build_result + + raw = { + "max_concurrency": 4, "model_id": "m", "num_prompts": 100, "completed": completed, + "total_token_throughput": 800.0, "output_throughput": 400.0, + "mean_ttft_ms": 100.0, "mean_tpot_ms": 10.0, + "benchmark_outcome": {"status": "passed", "requested": 100, "completed": completed, + "failed": 100 - completed, "max_failure_rate": 0.05}, + } # fmt: skip + env = { + "RUNNER_TYPE": "h100", "FRAMEWORK": "sglang", "PRECISION": "fp8", "SPEC_DECODING": "none", + "ISL": "1024", "OSL": "1024", "DISAGG": "false", "MODEL_PREFIX": "dsr1", "IMAGE": "i:1", + "TP": "8", "EP_SIZE": "1", "DP_ATTENTION": "false", + } # fmt: skip + values = reporting.values(build_result(raw, env)) + point = reporting.Point( + key="f" * 64, label="1024/1024 c4 TP8 EP1 ffffff", conc=4, scenario="fixed-seq-len", + values=values, result="passed", + ) # fmt: skip + + text = reporting.point_table([point]) + + assert values.request_errors == errors + assert values.output_tps_gpu == 50.0 + notes = [line.rstrip() for line in text.splitlines() if "request error" in line] + assert notes == ([note] if note else []) + + +def test_baseline_notes_name_concurrency_rows_like_their_table(): + points = [ + agentx_point("cc", output_tps_gpu=10.0, request_errors=2), + agentx_point("cc", output_tps_gpu=10.0).model_copy(update={ + "key": "e" * 64, "label": "AgentX c8 TP8 EP1 eeeeee", "conc": 8}), + ] # fmt: skip + baseline = reporting.Baseline( + family=f"configs/nvidia-master.yaml:{KEY}", date="2026-09-19", image="example/image:1", + goal=reporting.Prose(en="Update the image.", zh="更新镜像。"), sources=[], points=points, + ) # fmt: skip + + text = reporting.baseline_table(baseline) + + assert "| 4 | N/A | 10 | N/A | N/A |" in text + assert text.endswith("**Note:** c4: 2 request errors.") + + +@pytest.mark.parametrize(("points", "change"), [ + pytest.param([], "**变更:**更新镜像。 \n", id="queued-without-results"), + pytest.param([agentx_point("cc", output_tps_gpu=10.0, request_errors=0)], + "**变更:**更新镜像。实测数值及异常说明见上表。 \n", id="with-results"), +]) # fmt: skip +def test_attempt_translation_points_to_a_table_only_when_one_is_shown(points, change): + record = reporting.Attempt( + kind="initial", number=0, head="c" * 40, image="example/image:2", run_id=7, + run_attempt=1, status="running", + change=reporting.Prose(en="Update the image.", zh="更新镜像。"), + next=reporting.Prose(en="Wait.", zh="等待。"), benchmarks_expected=1, + benchmarks_passed=0, evals_expected=0, evals_passed=0, points=points, + ) # fmt: skip + + english, chinese = reporting.render_attempt(record, None, "example/project").split("
") + + assert ("| Concurrency |" in english) == bool(points) + assert change in chinese + + +def test_chunked_attempt_tells_both_languages_where_results_continue(monkeypatch): + head = "c" * 40 + session = SimpleNamespace( + repository="example/project", parent={"id": 42}, + candidate=OwnedCandidate(id=CANDIDATE_ID, family=FAMILY, base="a" * 40), + pulls=lambda: [{"number": 7}], refresh=lambda _pull: {"body": ""}, + runs=lambda: [{"id": 9, "head_sha": head, "run_attempt": 1, "status": "in_progress", + "conclusion": None}], + ) # fmt: skip + points = [ + agentx_point("cc", output_tps_gpu=10.0, request_errors=0).model_copy(update={ + "key": f"{conc:064x}", "label": f"AgentX c{conc} TP8 EP1 {conc:06x}", "conc": conc}) + for conc in range(1, 22) + ] # fmt: skip + record = reporting.Attempt( + kind="initial", number=0, head=head, image="example/image:2", run_id=9, run_attempt=1, + status="running", change=reporting.Prose(en="Update the image.", zh="更新镜像。"), + next=reporting.Prose(en="Wait.", zh="等待。"), benchmarks_expected=21, + benchmarks_passed=0, evals_expected=0, evals_passed=0, points=points, + ) # fmt: skip + posted = [] + monkeypatch.setattr(github, "items", lambda *_args, **_kwargs: []) + monkeypatch.setattr( + github, "write", lambda _repo, _path, _method, payload: posted.append(payload["body"]) + ) + + reporting.publish(session, record) + + [main] = [body for body in posted if ":run-9-1\n" in body.split("-->", 1)[0]] + english, chinese = main.split("
") + assert "Results continue in numbered report comments." in english + assert "结果见后续编号的报告评论。" in chinese + + def eval_entry(tp): return { "model-prefix": "dsr1", "runner": "cluster:fixture-a", "framework": "sglang", diff --git a/inferencex-e2e/infx/tests/launch/fake_slurm.py b/inferencex-e2e/infx/tests/launch/fake_slurm.py index f132c6d6b4..ad49e8f722 100644 --- a/inferencex-e2e/infx/tests/launch/fake_slurm.py +++ b/inferencex-e2e/infx/tests/launch/fake_slurm.py @@ -52,7 +52,8 @@ """, "sacct": r""" state="${FAKE_STATE:-COMPLETED|0:0}" -if [[ " $* " == *" 4242 "* && "$(cat "$FAKE_LOG_DIR/batch-rc" 2>/dev/null)" != 0 ]]; then state="FAILED|1:0"; fi +rc="$(cat "$FAKE_LOG_DIR/batch-rc" 2>/dev/null)" +if [[ " $* " == *" 4242 "* && -n "$rc" && "$rc" != 0 ]]; then state="FAILED|$rc:0"; fi echo "$state" """, "scontrol": "exit 1", @@ -136,7 +137,7 @@ if os.environ.get("FAKE_ACTIVE"): pathlib.Path(os.environ["FAKE_ACTIVE"]).touch() if "--json" in argv: - print(json.dumps({"status": "submitted", "slurm_job_id": "42", "output_dir": str(output)})) + print(os.environ.get("FAKE_MANIFEST") or json.dumps({"status": "submitted", "slurm_job_id": "42", "output_dir": str(output)})) else: print("✅ Job 42 submitted!") sys.exit(int(os.environ.get("FAKE_SRTCTL_RC", "0"))) diff --git a/inferencex-e2e/infx/tests/launch/test_backend_pluggability.py b/inferencex-e2e/infx/tests/launch/test_backend_pluggability.py index 6923e152b3..d27271eac1 100644 --- a/inferencex-e2e/infx/tests/launch/test_backend_pluggability.py +++ b/inferencex-e2e/infx/tests/launch/test_backend_pluggability.py @@ -140,3 +140,18 @@ def test_cleanup_hands_the_backend_the_runners_settings_when_it_has_a_record( [(settings, cleaned)] = FakeBackend.cleaned assert (getattr(settings, "namespace", None), cleaned) == (namespace, runner) + + +def test_a_runner_without_a_cluster_record_fails_the_run_with_one_error_line( + fake, workspace, tmp_path, monkeypatch, capsys +): + config = tmp_path / "runners.yaml" + config.write_text(yaml.safe_dump(fake)) + monkeypatch.setenv("RUNNER_NAME", "unlisted_00") + monkeypatch.setenv("GITHUB_WORKSPACE", str(workspace)) + + assert main(["--runner-config", str(config), "run"]) == 1 + + err = capsys.readouterr().err + assert "ERROR: runner 'unlisted_00' must be in exactly one cluster label, found []" in err + assert "Traceback" not in err diff --git a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py index e82f1b37dc..6c85895c50 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py @@ -25,7 +25,9 @@ def test_fixed_sequence_results_use_bounded_point_names(tmp_path): point = logs / "cfg_isl1024_osl128" (point / "nested").mkdir(parents=True) (point / "results_concurrency_4_gpus_8.json").write_text("agg") + (point / "results_concurrency_4_gpus_8.pytorch.json").write_text("agg sidecar") (point / "nested" / "results_concurrency_16_gpus_12_ctx_2_gen_3.json").write_text("disagg") + (point / "nested" / "results_concurrency_16_gpus_12_ctx_2_gen_3.pytorch.json").write_text("[]") (logs / "unrelated").mkdir() (logs / "unrelated" / "results_concurrency_1_gpus_1.json").write_text("ignored") @@ -75,7 +77,8 @@ def test_bundle_skips_empty_logs_and_archives_contents(tmp_path): assert "./worker.log" in tar.getnames() -def test_cleanup_outputs_retries_then_removes_nfs_files(tmp_path, monkeypatch): +@pytest.mark.parametrize(("busy", "removed"), [(2, True), (5, False)], ids=["frees", "stays-busy"]) +def test_cleanup_outputs_retries_then_removes_nfs_files(tmp_path, monkeypatch, busy, removed): (tmp_path / "outputs" / "job").mkdir(parents=True) (tmp_path / "keep" / ".nfs0001").parent.mkdir() (tmp_path / "keep" / ".nfs0001").write_text("") @@ -84,15 +87,15 @@ def test_cleanup_outputs_retries_then_removes_nfs_files(tmp_path, monkeypatch): def flaky_rmtree(path): calls.append(path) - if len(calls) < 3: + if len(calls) <= busy: raise OSError("Device or resource busy") real_rmtree(path) monkeypatch.setattr(collect.shutil, "rmtree", flaky_rmtree) sleeps = [] cleanup_outputs(tmp_path, sleep=sleeps.append) - assert len(calls) == 3 and sleeps == [10, 10] - assert not (tmp_path / "outputs").exists() + assert len(calls) == min(busy + 1, 5) and sleeps == [10] * min(busy, 4) + assert (tmp_path / "outputs").exists() is not removed assert sorted(p.name for p in (tmp_path / "keep").iterdir()) == ["log"] diff --git a/inferencex-e2e/infx/tests/launch/test_launch_policy.py b/inferencex-e2e/infx/tests/launch/test_launch_policy.py index 940e1443b2..e12be1756f 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_policy.py @@ -13,7 +13,7 @@ from infx.launch.drivers.srt.power import PowerPolicyError, decide_power from infx.launch.lifecycle import Lifecycle from infx.launch.policy import LaunchPath, Match, salloc_time_limit -from infx.launch.request import LaunchRequest, MultiNodeRequest +from infx.launch.request import LaunchRequest, MultiNodeRequest, RequestError DCGM, AGENTX, ADAPTER, ERR = "dcgm", "agentx", "adapter", "error" MULTI, NATIVE = LaunchPath.SRT_MULTI, LaunchPath.SRT_NATIVE @@ -168,3 +168,26 @@ def test_cluster_settings_beat_the_host_but_never_the_points_settings(chosen, ex env = policy.runtime_env(cluster, request, {"LATER": "later", "SHARED": "later"}) assert {name: env[name] for name in HOST} == expected + + +def test_a_role_without_additional_settings_arrives_as_null_and_sets_no_names(): + record = {**RECORD, "env": {"CLUSTER": "cluster", "SHARED": "cluster"}} + cluster = load_inventory({"labels": {"cluster:c": ["c_0"]}, "clusters": {"c": record}}).clusters["c"] + request = _request( + PREFILL_ADDITIONAL_SETTINGS='["SHARED=point"]', DECODE_ADDITIONAL_SETTINGS="null", + **{**HOST, "SHARED": "point"}, + ) + + env = policy.runtime_env(cluster, request, {"LATER": "later", "SHARED": "later"}) + + assert {name: env[name] for name in HOST} == {"CLUSTER": "cluster", "LATER": "later", "SHARED": "point"} + + +@pytest.mark.parametrize(("settings", "kind"), [("5", "int"), ("true", "bool"), ('{"SHARED=point": 1}', "dict")]) +def test_additional_settings_that_are_not_a_list_are_rejected(settings, kind): + record = {**RECORD, "env": {"CLUSTER": "cluster", "SHARED": "cluster"}} + cluster = load_inventory({"labels": {"cluster:c": ["c_0"]}, "clusters": {"c": record}}).clusters["c"] + request = _request(DECODE_ADDITIONAL_SETTINGS=settings, **HOST) + + with pytest.raises(RequestError, match=f"^DECODE_ADDITIONAL_SETTINGS must be a JSON list .*, got {kind}$"): + policy.runtime_env(cluster, request, {"LATER": "later", "SHARED": "later"}) diff --git a/inferencex-e2e/infx/tests/launch/test_script_driver.py b/inferencex-e2e/infx/tests/launch/test_script_driver.py index 3dc007f62c..cefbef8631 100644 --- a/inferencex-e2e/infx/tests/launch/test_script_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_script_driver.py @@ -27,7 +27,14 @@ with open(os.environ["FAKE_CALLS"], "a") as log: log.write(json.dumps([name, *args]) + "\n") if name == "salloc": + if os.environ.get("FAKE_QUEUED_MARKER"): + open(os.environ["FAKE_QUEUED_MARKER"], "w").close() + import time + time.sleep(60) print("salloc: Granted job allocation 42", file=sys.stderr) +elif name == "squeue": + if os.environ.get("FAKE_STUCK_JOB"): + print(os.environ["FAKE_STUCK_JOB"]) elif name == "unsquashfs": sys.exit(1) elif name == "srun": @@ -255,6 +262,7 @@ def test_collectors_serve_the_checkpoint_srt_jobs_would(fakes, workspace, tmp_pa @pytest.mark.parametrize(("overrides", "message"), [ ({"OUT_YAML": "/tmp/speedbench-reference-al.yaml"}, "lies outside the container workspace"), + ({"OUT_YAML": "/workspace/../etc/reference.yaml"}, "lies outside the container workspace"), ({"GPU_COUNT": ""}, "GPU_COUNT"), ]) # fmt: skip def test_a_bad_input_fails_before_any_slurm_call(fakes, workspace, tmp_path, capsys, overrides, message): @@ -303,6 +311,17 @@ def test_salloc_exclude_escape_hatch_extends_the_cluster_exclusions(fakes, works assert option(allocation, "--exclude") == "bad-node,node[1,3]" +def test_the_echoed_container_step_masks_secrets_in_the_container_env(fakes, workspace, tmp_path, capsys): + inventory = inventory_for(tmp_path) + inventory["clusters"]["fixture"]["env"]["MIRROR_TOKEN"] = "tok-cluster-only" + + assert launch(load_inventory(inventory).clusters["fixture"], request_for(workspace)) == 0 + + assert "MIRROR_TOKEN=tok-cluster-only" in option(container_step(fakes()), "--export").split(",") + err = capsys.readouterr().err + assert "MIRROR_TOKEN=***" in err and "tok-cluster-only" not in err + + def test_sigterm_during_the_collector_cancels_the_allocation_and_exits_143( fakes, workspace, tmp_path ): @@ -324,3 +343,29 @@ def test_sigterm_during_the_collector_cancels_the_allocation_and_exits_143( assert launcher.wait(timeout=60) == 143 assert fakes()[-1] == ["scancel", "42"] + + +def test_sigterm_while_salloc_is_queued_cancels_the_pending_request(fakes, workspace, tmp_path): + config = tmp_path / "runners.yaml" + config.write_text(yaml.safe_dump(inventory_for(tmp_path))) + queued = tmp_path / "salloc-queued" + launcher = subprocess.Popen( + [sys.executable, "-m", "infx.launch", "--runner-config", str(config), "run"], + env={**os.environ, **launch_env(workspace), "PYTHONPATH": str(ROOT), + "FAKE_QUEUED_MARKER": str(queued), "FAKE_STUCK_JOB": "99", "USER": "runner"}, + cwd=workspace, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) # fmt: skip + try: + deadline = time.monotonic() + 60 + while not queued.exists() and launcher.poll() is None and time.monotonic() < deadline: + time.sleep(0.1) + assert queued.exists(), "salloc never started" + + launcher.send_signal(signal.SIGTERM) + + assert launcher.wait(timeout=30) == 143 + finally: + launcher.kill() + launcher.wait() + assert ["scancel", "--user=runner", "--name=fixture_00"] in fakes() + assert [call for call in fakes() if call[0] == "squeue"] == [] diff --git a/inferencex-e2e/infx/tests/launch/test_slurm_cli.py b/inferencex-e2e/infx/tests/launch/test_slurm_cli.py index 60c1ab76e7..943875f3ab 100644 --- a/inferencex-e2e/infx/tests/launch/test_slurm_cli.py +++ b/inferencex-e2e/infx/tests/launch/test_slurm_cli.py @@ -11,6 +11,7 @@ from infx.clusters import load_inventory from infx.launch.backends import slurm from infx.launch.backends.base import Job, JobState +from infx.launch.backends.slurm import cli from infx.launch.backends.slurm.cli import ( ContainerSpec, Resources, @@ -79,7 +80,9 @@ def test_salloc_without_a_grant_fails(fake_bin, body): @pytest.mark.parametrize("record,state,exit_code,nodes", [ ("COMPLETED|0:0|gpu[01-02]", JobState.SUCCEEDED, 0, "gpu[01-02]"), ("FAILED|1:0|gpu03", JobState.FAILED, 1, "gpu03"), - ("COMPLETED|0:15", JobState.FAILED, 0, None), + ("COMPLETED|0:15", JobState.FAILED, 143, None), + ("FAILED|0:9|gpu03", JobState.FAILED, 137, "gpu03"), + ("OUT_OF_MEMORY|0:125|gpu03", JobState.FAILED, 0, "gpu03"), ("CANCELLED by 1001|0:0|None assigned", JobState.CANCELLED, 0, None), ]) def test_final_status_maps_allocation_accounting(fake_bin, tmp_path, record, state, exit_code, nodes): @@ -128,6 +131,14 @@ def test_queue_state_reads_only_the_named_job(fake_bin): assert queue_state(Job("7")) is None +def test_queue_state_retries_a_failed_squeue_instead_of_reporting_the_job_gone(fake_bin, tmp_path, monkeypatch): + failed = tmp_path / "failed" + fake_bin("squeue", f"if [ ! -e {failed} ]; then touch {failed}; " + "echo 'slurm_load_jobs error: Socket timed out' >&2; exit 1; fi; echo '42|RUNNING'") + monkeypatch.setattr(cli, "SQUEUE_RETRY_S", 0) + assert queue_state(Job("42")) == "RUNNING" + + def test_waiting_for_a_log_fails_when_the_job_dies_before_it(fake_bin, tmp_path): log = tmp_path / "scontrol.log" fake_bin("squeue", "exit 0") @@ -137,6 +148,15 @@ def test_waiting_for_a_log_fails_when_the_job_dies_before_it(fake_bin, tmp_path) assert calls(log) == [["show", "job", "77"]] +def test_waiting_for_a_log_accepts_one_created_just_before_the_job_left_the_queue(fake_bin, tmp_path, capsys): + batch_log, scontrol = tmp_path / "batch.log", tmp_path / "scontrol.log" + fake_bin("squeue", f"echo started > {batch_log}") + fake_bin("scontrol", recorder(scontrol)) + wait_for_log(Job("77"), batch_log, wait_s=0) + assert calls(scontrol) == [] + assert "ERROR" not in capsys.readouterr().err + + def slurm_backend() -> slurm.SlurmBackend: """A Slurm backend for runner ``c_0`` of a minimal cluster.""" record = {"gpus-per-node": 8, "arch": "x86_64", "scheduler": "slurm", @@ -158,6 +178,32 @@ def test_backend_cancel_waits_until_a_listed_job_leaves_the_queue(fake_bin, tmp_ assert queue_state(Job("4242")) is None +def test_backend_cancel_scancels_even_when_squeue_fails(fake_bin, tmp_path, monkeypatch): + log = tmp_path / "scancel.log" + fake_bin("squeue", "exit 1") + fake_bin("scancel", recorder(log)) + monkeypatch.setattr(cli, "SQUEUE_RETRY_S", 0) + backend = slurm_backend() + + backend.cancel(backend.attach("4242")) + + assert calls(log) == [["4242"]] + + +def test_backend_cancel_skips_a_job_squeue_reports_as_purged(fake_bin, tmp_path, monkeypatch): + queries, scancel = tmp_path / "squeue.log", tmp_path / "scancel.log" + fake_bin("squeue", f"{recorder(queries)}\n" + "echo 'slurm_load_jobs error: Invalid job id specified' >&2; exit 1") + fake_bin("scancel", recorder(scancel)) + monkeypatch.setattr(cli, "SQUEUE_RETRY_S", 0) + backend = slurm_backend() + + backend.cancel(backend.attach("4242")) + + assert len(calls(queries)) == 1 + assert calls(scancel) == [] + + def test_workflow_cleanup_cancels_the_runners_jobs_and_those_srtctl_submitted(fake_bin, tmp_path, monkeypatch): scancel, squeue = tmp_path / "scancel.log", tmp_path / "squeue.log" fake_bin("scancel", recorder(scancel)) @@ -173,6 +219,21 @@ def test_workflow_cleanup_cancels_the_runners_jobs_and_those_srtctl_submitted(fa assert "--name=b300-dsxe_03,inferencex-b300-dsxe_03" in query +def test_workflow_cleanup_stops_waiting_for_a_job_that_never_leaves_the_queue(fake_bin, monkeypatch, capsys): + fake_bin("scancel", "exit 0") + fake_bin("squeue", "echo 4242") + monkeypatch.setattr(cli, "CANCEL_NAMED_WAIT_S", 0) + + def stuck(seconds: float) -> None: + raise AssertionError("cleanup kept waiting past its deadline") + + monkeypatch.setattr(cli.time, "sleep", stuck) + + slurm.SlurmBackend.cleanup(None, "b300-dsxe_03") + + assert "WARNING: jobs 4242 still in the queue" in capsys.readouterr().err + + def test_a_container_step_killed_by_a_signal_reports_the_shell_exit_code(): step = subprocess.Popen(["bash", "-c", "kill -KILL $$"]) step.wait() @@ -182,6 +243,39 @@ def test_a_container_step_killed_by_a_signal_reports_the_shell_exit_code(): assert (status.state, status.exit_code) == (JobState.FAILED, 137) +@pytest.mark.parametrize(("allocation", "kind", "retriable"), [ + ("NODE_FAIL|0:0|gpu01", "JobNodeFail", True), + ("RUNNING|0:0|gpu01", "JobFailed", False), +]) # fmt: skip +def test_a_container_step_on_a_lost_allocation_is_recorded_as_retriable(fake_bin, allocation, kind, retriable): + fake_bin("sacct", f"echo '{allocation}'") + step = subprocess.Popen(["bash", "-c", "exit 1"]) + step.wait() + backend = slurm_backend() + + backend.state(slurm.SlurmJob("42", step=step)) + + error = backend.life.event.finish(1).error + assert (error.type, error.retriable) == (kind, retriable) + + +def test_a_running_allocation_records_no_end_from_the_controllers_projection(fake_bin): + fake_bin("sacct", "echo 'sacct: error: Problem talking to the database' >&2; exit 1") + # A running job's EndTime is its start plus the time limit, not when it ended. + fake_bin("scontrol", "echo 'JobId=42 JobState=RUNNING ExitCode=0:0 NodeList=gpu01 " + "SubmitTime=1791547200 StartTime=1791547260 EndTime=1791633660'") + step = subprocess.Popen(["bash", "-c", "exit 0"]) + step.wait() + backend = slurm_backend() + + backend.state(slurm.SlurmJob("42", step=step)) + + record = backend.life.event.finish(0) + assert (record.slurm_started_at, record.slurm_ended_at, record.queue_s) == ( + datetime(2026, 10, 9, 12, 1, tzinfo=UTC), None, 60.0, + ) + + def test_srun_rejects_env_value_that_would_split_export(): with pytest.raises(ValueError, match="UCX_NET_DEVICES"): srun_argv(None, ["true"], container=ContainerSpec(image="i", env={"UCX_NET_DEVICES": "a,b"})) diff --git a/inferencex-e2e/infx/tests/launch/test_squash_images.py b/inferencex-e2e/infx/tests/launch/test_squash_images.py index 1103521352..90564abf6c 100644 --- a/inferencex-e2e/infx/tests/launch/test_squash_images.py +++ b/inferencex-e2e/infx/tests/launch/test_squash_images.py @@ -141,15 +141,31 @@ def test_import_producing_invalid_squash_fails_without_replacing(tools, tmp_path assert sorted(p.name for p in (tmp_path / "squash").iterdir()) == [f"{SQUASH_NAME}.lock"] +def test_submit_host_import_without_unsquashfs_keeps_the_existing_squash(tools, tmp_path, monkeypatch): + _, logs = tools + (tmp_path / "bin" / "unsquashfs").unlink() + monkeypatch.setenv("PATH", str(tmp_path / "bin")) + squash = tmp_path / "squash" / SQUASH_NAME + squash.parent.mkdir() + squash.write_text(VALID) + with pytest.raises(ImageError, match="unsquashfs") as error: + ensure_image(IMAGE, policy(tmp_path, "submit-host"), job=None) + assert error.value.retriable is False + assert squash.read_text() == VALID + assert calls(logs["enroot"]) == [] + + @pytest.mark.parametrize(("mode", "visibility", "job", "message", "steps"), [ ("pre-staged", "shared", None, rf"pre-staged image .* missing or invalid at .*{SQUASH_NAME}", 0), ("pre-staged", "node-local", Job("7"), "stage it there", 1), ("compute", "node-local", None, "requires a job allocation", 0), + ("all-nodes", "shared", None, "needs a job allocation", 0), ]) # fmt: skip def test_an_image_that_cannot_be_staged_fails_at_once(tools, tmp_path, mode, visibility, job, message, steps): _, logs = tools - with pytest.raises(ImageError, match=message): + with pytest.raises(ImageError, match=message) as error: ensure_image(IMAGE, policy(tmp_path, mode, visibility), job=job) + assert error.value.retriable is False assert len(calls(logs["srun"])) == steps assert calls(logs["enroot"]) == [] diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index 8b061ac977..01a1d4216d 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -514,6 +514,21 @@ def test_submission_failure_code_propagates_and_cancels_the_job(harness, shape): assert (error["stage"], error["type"], error["exit_code"]) == ("submit", "SubmitFailed", 7) +@pytest.mark.parametrize("shape", ["single", "multi"]) +def test_a_manifest_that_is_not_a_json_object_fails_as_a_launch_error(harness, shape): + if shape == "single": + env = single_node_env(harness, "mi355x-amds", FAKE_MANIFEST='["42"]') + else: + env = lane_env(harness, "b300-dsxe", MODEL_PREFIX="dsr1", PRECISION="fp4", FRAMEWORK="dynamo-trt", + MODEL="deepseek-r1-fp4", FAKE_MANIFEST='["42"]') # fmt: skip + result = launch(env, harness.config, harness.workspace) + assert result.returncode == 1, result.stderr[-4000:] + assert "Traceback" not in result.stderr + error = job_event(harness.workspace)["error"] + assert error["type"] == "LaunchError" + assert error["message"].startswith("srtctl did not report one submitted job") + + @pytest.mark.parametrize("shape", ["single", "multi"]) def test_sigterm_while_streaming_cancels_the_job_and_exits_143(harness, shape): active, tailing = harness.tmp / "active", harness.tmp / "tailing" @@ -595,6 +610,15 @@ def test_a_failed_batch_wrapped_point_reports_the_wrapped_failure_once(harness): assert (result.stdout + result.stderr).count("::error") == 1 +def test_a_batch_wrapped_point_exits_with_the_wrapped_launchs_code(harness): + env = batch_env(harness, FAKE_SRTCTL_RC="7") + result = launch(env, harness.config, harness.workspace) + assert (harness.logs / "batch-rc").read_text() == "7" + assert result.returncode == 7, result.stderr[-4000:] + error = job_event(harness.workspace)["error"] + assert (error["type"], error["exit_code"]) == ("SubmitFailed", 7) + + def test_tilert_uses_upstream_submission_and_prepared_weights(harness): env = lane_env( harness, "mi355x-amds", MODEL_PREFIX="glm5.3", PRECISION="fp8", FRAMEWORK="tilert", @@ -707,6 +731,36 @@ def test_a_setup_failure_without_a_bad_archive_is_not_retried(harness): assert (error["stage"], error["type"], error["evidence"]) == ("prepare", "SetupFailed", "srt-setup.log") +def test_a_setup_step_killed_by_a_signal_exits_128_plus_the_signal(harness): + fakes = Path(harness.env["PATH"].split(os.pathsep)[0]) + (fakes / "make").write_text("#!/bin/bash\nkill -KILL $$\n") + env = lane_env( + harness, "h200-dgxc", MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", + MODEL="deepseek-ai/DeepSeek-R1-0528", + ) # fmt: skip + result = launch(env, harness.config, harness.workspace) + assert result.returncode == 137, result.stderr[-4000:] + error = job_event(harness.workspace)["error"] + assert (error["type"], error["exit_code"]) == ("SetupFailed", 137) + + +def test_a_failed_uv_download_fails_the_uv_install(harness): + fakes = Path(harness.env["PATH"].split(os.pathsep)[0]) + (fakes / "uv").unlink() + (fakes / "curl").write_text("#!/bin/bash\necho 'curl: (6) Could not resolve host' >&2\nexit 6\n") + (fakes / "curl").chmod(0o755) + env = lane_env( + harness, "h200-dgxc", MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", + MODEL="deepseek-ai/DeepSeek-R1-0528", + ) # fmt: skip + result = launch(env, harness.config, harness.workspace) + assert result.returncode == 1 + error = job_event(harness.workspace)["error"] + assert (error["stage"], error["type"], error["message"]) == ( + "prepare", "LaunchError", "installing uv failed (exit 6)", + ) # fmt: skip + + @pytest.mark.parametrize(("cluster_id", "env", "message"), [ ("h200-dgxc", dict(MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-vllm"), "Unsupported framework"), ("b200-nscale", dict(MODEL_PREFIX="dsv4", PRECISION="fp4", FRAMEWORK="dynamo-trt"), "only dynamo-vllm"), @@ -725,14 +779,21 @@ def test_unsupported_multinode_requests_fail_before_any_setup(harness, cluster_i @pytest.mark.parametrize(("shape", "overrides", "missing"), [ ("single", dict(IS_AGENTIC="1", SPEC_DECODING="mtp", THINKING_MODE=""), "THINKING_MODE"), + ("single", dict(IS_AGENTIC="1", DURATION="600", INFERENCEX_RESULTS_PYTHON=None), + "INFERENCEX_RESULTS_PYTHON"), ("multi", dict(SPEC_DECODING=None), "SPEC_DECODING"), ("multi", dict(SRT_RECIPE=None), "SRT_RECIPE"), + ("power", dict(INFERENCEX_RESULTS_PYTHON=None), "INFERENCEX_RESULTS_PYTHON"), ("batch", dict(SRT_RECIPE=None), "SRT_RECIPE"), ]) # fmt: skip def test_a_missing_input_is_named_before_any_setup(harness, shape, overrides, missing): if shape == "multi": env = lane_env(harness, "h200-dgxc", MODEL_PREFIX="dsr1", PRECISION="fp8", FRAMEWORK="dynamo-sglang", MODEL="deepseek-ai/DeepSeek-R1-0528") # fmt: skip + elif shape == "power": + env = lane_env(harness, "h200-dgxc", MODEL_PREFIX="glm5.2", PRECISION="fp8", + FRAMEWORK="dynamo-sglang", MODEL="zai-org/GLM-5.2-FP8", IS_AGENTIC="1", + CONC="4", FAKE_RESULTS="agentic", POWER="1") # fmt: skip else: env = single_node_env(harness, "b300-dsxe" if shape == "batch" else "h100-cw") if shape == "batch": diff --git a/inferencex-e2e/infx/tests/matrix/test_changed.py b/inferencex-e2e/infx/tests/matrix/test_changed.py index 77efa8bab3..14a09273ed 100644 --- a/inferencex-e2e/infx/tests/matrix/test_changed.py +++ b/inferencex-e2e/infx/tests/matrix/test_changed.py @@ -5,6 +5,7 @@ import subprocess from pathlib import Path +import pytest import yaml from infx.matrix import changed @@ -78,3 +79,43 @@ def test_keys_with_changed_recipes_or_points_are_listed(tmp_path, monkeypatch, c changed.main(["--base", "base"]) assert capsys.readouterr().out.splitlines()[-1] == "3 of 4 config keys changed since base." + + +def test_rows_without_fingerprints_are_compared_individually(tmp_path): + def matrix(*tps: int) -> changed.Matrix: + rows = [ + {"exp-name": f"key_tp{tp}", "tp": tp, "ep": 1, "conc": [4], "srt-recipe": "gone.yaml"} + for tp in tps + ] + return {"key": changed._points(rows, tmp_path)} + + base, head = matrix(4, 8), matrix(4, 2) + + assert [len(errors) for _, errors in (base["key"], head["key"])] == [2, 2] + assert [ + (change["base-points"], change["head-points"], + [point["tp"] for point in change["added"]], [point["tp"] for point in change["removed"]], + [point["recipe-fingerprint"] for point in change["added"] + change["removed"]]) + for change in changed.compare(base, head) + ] == [(2, 2, [2], [8], [None, None])] # fmt: skip + + +def test_points_sharing_a_name_and_concurrency_keep_generation_order(tmp_path): + tps = [8, 4, 2, 1, 16, 32, 64, 128] + rows = [{"exp-name": "dsr1_8k1k", "tp": tp, "ep": 1, "conc": [32]} for tp in tps] + head = {"key": changed._points(rows, tmp_path)} + + [change] = changed.compare({}, head) + + assert [point["tp"] for point in change["added"]] == tps + + +def test_a_head_config_error_wins_over_a_base_that_cannot_generate(tmp_path, monkeypatch): + (tmp_path / "configs").mkdir() + (tmp_path / "configs/nvidia-master.yaml").write_text("key: [unclosed\n") + (tmp_path / "configs/amd-master.yaml").write_text("{}\n") + (tmp_path / "configs/runners.yaml").write_text("labels: {}\n") + monkeypatch.chdir(tmp_path) + + with pytest.raises(yaml.YAMLError, match="while parsing a flow sequence"): + changed.main(["--base", "missing-ref"]) diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index b3b07209e9..03c7a16995 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -119,6 +119,8 @@ def write(recipe: dict, name: str = "test.yaml") -> str: ({"prefill": {"nodes": 2}, "decode": {"nodes": 4}}, 6), ({"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}, 2), ({"prefill": {"nodes": 2}, "decode": {"workers": 1}}, None), + ({"prefill": {"nodes": 2}, "decode": {"nodes": None}}, None), + ({"prefill": {"nodes": 2}, "decode": None}, None), ]) def test_multinode_node_count_reads_schema_two_roles(tmp_path, multinode_recipe, roles, expected): import infx.matrix.generate as generate @@ -130,11 +132,34 @@ def test_multinode_node_count_reads_schema_two_roles(tmp_path, multinode_recipe, assert generate.recipe_node_count(tmp_path, srt_recipe) == expected +@pytest.mark.parametrize("sections", [ + {"services": None}, + {"infra": None}, + {"frontend": None}, + {"benchmark": None}, + {"services": [{"type": "etcd", "placement": None}, {"type": "nats"}]}, +]) +def test_recipe_node_count_treats_null_sections_as_absent(tmp_path, multinode_recipe, sections): + import infx.matrix.generate as generate + srt_recipe = multinode_recipe( + {"schema": 2, "roles": {"prefill": {"nodes": 1}, "decode": {"nodes": 2}}, **sections} + ) + assert generate.recipe_node_count(tmp_path, srt_recipe) == 3 + + +def test_recipe_node_count_rejects_an_empty_recipe(tmp_path, multinode_recipe): + import infx.matrix.generate as generate + srt_recipe = multinode_recipe(None) + with pytest.raises(ValueError, match="must declare schema: 2: .*test.yaml"): + generate.recipe_node_count(tmp_path, srt_recipe) + + @pytest.mark.parametrize("selector, expected", [ ("base", 3), ("override_wide", 5), ("override_colocated", 2), - ("zip_override_sweep[0]", None), + ("zip_override_sweep[0]", 2), + ("zip_override_sweep[1]", 3), ]) def test_recipe_node_count_resolves_override_selectors(tmp_path, multinode_recipe, selector, expected): srt_recipe = multinode_recipe({ @@ -1235,55 +1260,61 @@ def test_matrix_entry_structure(self, sample_single_node_config, sample_runner_c } == {(2, 2, 2)} def test_filter_by_model_prefix(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): - full_sweep_args_single_node.model_prefix = ["dsr1"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) > 0 + entry = sample_single_node_config["dsr1-fp8-mi300x-sglang"] + sample_single_node_config["dsr1flash-fp8-mi300x-sglang"] = {**entry, "model-prefix": "dsr1flash"} - full_sweep_args_single_node.model_prefix = ["nonexistent"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) == 0 + selected = {} + for prefixes in (["dsr1"], ["dsr1flash"], ["dsr1", "dsr1flash"]): + full_sweep_args_single_node.model_prefix = prefixes + result = generate_full_sweep( + full_sweep_args_single_node, + sample_single_node_config, + sample_runner_config + ) + selected[tuple(prefixes)] = sorted({row["model-prefix"] for row in result}) - def test_filter_by_precision(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): - full_sweep_args_single_node.precision = ["fp8"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) > 0 + assert selected == { + ("dsr1",): ["dsr1"], + ("dsr1flash",): ["dsr1flash"], + ("dsr1", "dsr1flash"): ["dsr1", "dsr1flash"], + } - full_sweep_args_single_node.precision = ["fp4"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) == 0 + @pytest.mark.parametrize("field", ["precision", "framework"]) + def test_filter_by_config_field( + self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node, field, + ): + entry = sample_single_node_config["dsr1-fp8-mi300x-sglang"] + sample_single_node_config["dsr1-fp4-mi300x-vllm"] = { + **entry, "precision": "fp4", "framework": "vllm", + } - def test_filter_by_framework(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): - full_sweep_args_single_node.framework = ["sglang"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) > 0 + selected = {} + for value in ("fp8", "fp4") if field == "precision" else ("sglang", "vllm"): + setattr(full_sweep_args_single_node, field, [value]) + result = generate_full_sweep( + full_sweep_args_single_node, sample_single_node_config, sample_runner_config, + ) + selected[value] = {row["precision"] for row in result} - full_sweep_args_single_node.framework = ["vllm"] - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_config, - sample_runner_config - ) - assert len(result) == 0 + if field == "precision": + assert selected == {"fp8": {"fp8"}, "fp4": {"fp4"}} + else: + assert selected == {"sglang": {"fp8"}, "vllm": {"fp4"}} + + @pytest.mark.parametrize(("filters", "message"), [ + ({"model_prefix": ["dsr1", "deepseek"]}, "No config has model prefix: deepseek"), + ({"precision": ["FP8"]}, "No config has precision: FP8"), + ({"framework": ["trtllm"]}, "No config has framework: trtllm"), + ]) + def test_filter_value_no_config_has_is_rejected( + self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node, + filters, message, + ): + vars(full_sweep_args_single_node).update(filters) + with pytest.raises(ValueError, match=f"^{message}$"): + generate_full_sweep( + full_sweep_args_single_node, sample_single_node_config, sample_runner_config, + ) def test_filter_by_runner_type(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): full_sweep_args_single_node.runner_type = ["mi300x"] @@ -2266,6 +2297,16 @@ def test_trim_conc_updates_multinode_dispatch_concurrency(self): assert result[0]['eval-conc'] == 4 assert result[0]['run-eval'] is True + def test_trim_conc_merges_concurrencies_whose_recipe_variants_differ(self): + # A single-node bundle selects one variant per CONC, so each point has its own fingerprint. + shape = {'tp': 8, 'model': 'm', 'srt-recipe': 'agentic.yaml'} + rows = [ + {**shape, 'conc': 8, 'recipe-fingerprint': 'b' * 64}, + {**shape, 'conc': 2, 'recipe-fingerprint': 'a' * 64}, + ] + + assert trim_conc(rows) == [{**shape, 'conc': 2, 'recipe-fingerprint': 'a' * 64}] + def test_trim_conc_never_merges_a_standalone_eval_into_another_suite(self): shape = {'tp': 8, 'model': 'm'} vendor = {**shape, 'conc': 16, 'run-eval': True, @@ -2280,6 +2321,68 @@ def test_trim_conc_never_merges_a_standalone_eval_into_another_suite(self): (row['eval-framework'], row['conc'], bool(row.get('eval-only'))) for row in result ) == [('kimi-vendor', 1, False), ('lm-eval', 16, True)] + @pytest.mark.parametrize('multinode', [False, True]) + def test_trimmed_all_evals_keep_gsm8k_beside_the_vendor_eval(self, multinode): + shape = { + 'scenario-type': 'agentic-coding', 'model-prefix': 'minimaxm3', 'model': 'm', + 'runner': 'r', 'framework': 'vllm', 'precision': 'fp4', 'spec-decoding': 'none', + 'dp-attn': False, 'image': 'img', 'kv-offloading': 'none', + } + if multinode: + shape |= {'prefill': {'tp': 8}, 'decode': {'tp': 8}} + rows = [{**shape, 'conc': [4, 8]}] + else: + shape |= {'tp': 4} + rows = [{**shape, 'conc': 4}, {**shape, 'conc': 8}] + + result = generate_sweep_configs.select_matrix_evals(rows, mode='all', trim=True) + + expected_conc = [4] if multinode else 4 + assert sorted((row['eval-framework'], row['conc']) for row in result) == [ + ('lm-eval', expected_conc), ('minimax-vendor', expected_conc), + ] + + def test_trimming_does_not_give_two_recipes_the_same_app_eval(self): + # Each recipe's eval starts at its own highest concurrency; trimming moves both to 2. + shape = { + 'model': 'm', 'model-prefix': 'm', 'runner': 'r', 'framework': 'f', + 'precision': 'fp4', 'spec-decoding': 'none', 'disagg': True, 'isl': 8192, + 'osl': 1024, 'prefill': {'tp': 8, 'num-worker': 1}, + 'decode': {'tp': 8, 'num-worker': 2}, + } + rows = [ + {**shape, 'srt-recipe': 'narrow.yaml', 'conc': [2, 64]}, + {**shape, 'srt-recipe': 'wide.yaml', 'conc': [2, 128]}, + ] + + result = generate_sweep_configs.select_matrix_evals(rows, mode='default', trim=True) + + assert [ + (row['srt-recipe'], row['conc'], row['run-eval'], row.get('eval-conc')) + for row in result + ] == [('narrow.yaml', [2], True, 2), ('wide.yaml', [2], False, None)] + + def test_trimmed_agentic_eval_without_a_framework_still_claims_its_app_eval(self): + # dense.yaml's eval moves from conc 64 to its conc-16 row; sparse.yaml evaluates conc 16. + shape = { + 'scenario-type': 'agentic-coding', 'model': 'm', 'model-prefix': 'm', + 'runner': 'r', 'framework': 'f', 'precision': 'fp4', 'spec-decoding': 'none', + 'disagg': True, 'kv-offloading': 'none', 'prefill': {'tp': 8, 'num-worker': 1}, + 'decode': {'tp': 8, 'num-worker': 1}, + } + rows = [ + {**shape, 'srt-recipe': 'dense.yaml', 'conc': [16]}, + {**shape, 'srt-recipe': 'dense.yaml', 'conc': [64]}, + {**shape, 'srt-recipe': 'sparse.yaml', 'conc': [16]}, + ] + + result = generate_sweep_configs.select_matrix_evals(rows, mode='default', trim=True) + + assert [ + (row['srt-recipe'], row['conc'], row['run-eval'], row.get('eval-framework')) + for row in result + ] == [('dense.yaml', [16], True, 'lm-eval'), ('sparse.yaml', [16], False, None)] + @pytest.mark.parametrize('entrypoint', ['cli', 'api']) def test_smoke_keeps_canonical_eval_instead_of_throughput_minimum( self, monkeypatch, sample_single_node_config, sample_runner_config, entrypoint, @@ -2312,6 +2415,19 @@ def test_multinode_smoke_preserves_representative_eval_concurrency(self): ([4], False, False), ([64], True, True)] assert result[1]['eval-conc'] == 64 + def test_smoke_does_not_turn_a_standalone_eval_into_an_idle_row(self): + from infx.matrix.generate import smoke_entries + shape = {'tp': 4, 'model': 'm'} + vendor = {**shape, 'conc': 4, 'run-eval': True, 'eval-framework': 'minimax-vendor'} + gsm8k = {**shape, 'conc': 8, 'run-eval': True, 'eval-only': True, + 'eval-framework': 'lm-eval'} + + result = smoke_entries([vendor, gsm8k]) + + assert [ + (row['conc'], row.get('eval-only', False), row['run-eval']) for row in result + ] == [(4, False, False), (4, True, True), (8, True, True)] + def test_all_evals_batches_each_multinode_concurrency( self, monkeypatch, @@ -2549,18 +2665,22 @@ def test_point_order_and_input_preservation( assert [(e["runner"], e["conc"]) for e in entries] == expected assert config == original - @pytest.mark.parametrize(("full_filters", "exact_filters", "expected"), [ - ({}, {}, [3, 6, 10]), - ({"min_conc": 5, "max_conc": 9}, {"conc": [6, 9]}, [6]), - ({"max_conc": 2}, {"conc": [2]}, []), - ({"min_conc": 11}, {"conc": [11]}, []), + @pytest.mark.parametrize(("full_filters", "exact_filters", "expected", "rejected"), [ + ({}, {}, [3, 6, 10], None), + ({"min_conc": 5, "max_conc": 9}, {"conc": [6]}, [6], None), + ({"max_conc": 2}, {"conc": [2]}, [], "2"), + ({"min_conc": 11}, {"conc": [6, 11]}, [], "11"), ]) def test_range_boundaries( self, agentic_config, sample_runner_config, generate_agentic_sweep, - agentic_mode, full_filters, exact_filters, expected, + agentic_mode, full_filters, exact_filters, expected, rejected, ): config, benchmark = agentic_config benchmark.update({"conc-start": 3, "conc-end": 10}) + if agentic_mode == "test-config" and rejected is not None: + with pytest.raises(ValueError, match=f"not generated by the selected configs: {rejected}$"): + generate_agentic_sweep(config, sample_runner_config, **exact_filters) + return filters = full_filters if agentic_mode == "full-sweep" else exact_filters entries = generate_agentic_sweep(config, sample_runner_config, **filters) points = [entry["conc"] for entry in entries] diff --git a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py index 3ad666dbb3..8909b1389a 100644 --- a/inferencex-e2e/infx/tests/matrix/test_process_changelog.py +++ b/inferencex-e2e/infx/tests/matrix/test_process_changelog.py @@ -679,6 +679,40 @@ def test_generation_failure_does_not_publish_a_partial_matrix(planning_repo, cha assert '\"single_node\":' not in captured.out +@pytest.mark.parametrize("key", ["single", "multi"]) +def test_plan_rejects_a_sequence_length_no_sweep_job_runs(planning_repo, changelog_run, key): + root, master, _ = planning_repo + master[key]["scenarios"]["fixed-seq-len"][0].update(isl=1024, osl=8192) + (root / "configs/nvidia-master.yaml").write_text(yaml.safe_dump(master, sort_keys=False)) + + with pytest.raises(ValueError, match=r"isl=1024 osl=8192 .*1k1k, 8k1k"): + changelog_run([{"config-keys": [key], "scenario-type": ["fixed-seq-len"], "no-evals": True}]) + + +def test_a_master_config_key_defined_twice_is_rejected(planning_repo, changelog_run): + root, master, _ = planning_repo + shadow = {"single": {**master["single"], "image": "example/image:shadow"}} + with (root / "configs/nvidia-master.yaml").open("a") as master_file: + master_file.write(yaml.safe_dump(shadow, sort_keys=False)) + + with pytest.raises(yaml.YAMLError, match="found duplicate key 'single'"): + changelog_run([{"scenario-type": ["fixed-seq-len"], "no-evals": True}]) + + +def test_a_changelog_addition_with_a_repeated_field_is_rejected(planning_repo, monkeypatch): + added = ( + "- config-keys: [single]\n config-keys: [multi]\n description: [Change]\n" + " pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1\n" + ) + monkeypatch.setattr(process_changelog, "get_added_lines", lambda *_: added) + monkeypatch.setattr(sys, "argv", ["infx.matrix.plan", "--base-ref", "base", + "--head-ref", "head", "--changelog-file", "perf-changelog.yaml"]) + + with pytest.raises(yaml.YAMLError, match="found duplicate key 'config-keys'"): + process_changelog.main() + + + @pytest.mark.parametrize("threshold,expected", [ (None, ["single", "default", "low", "high", "null", "invalid"]), (1, ["single", "default", "low", "high"]), @@ -804,3 +838,56 @@ def git(*args): git("commit", "-qam", "append entry") monkeypatch.chdir(tmp_path / "inferencex-e2e") assert process_changelog.get_added_lines(base, "HEAD", "perf-changelog.yaml") == "new entry" + + +@pytest.fixture +def commit_changelog(tmp_path, monkeypatch): + """Commit ``perf-changelog.yaml`` with the given text in a scratch repo; return the SHA.""" + monkeypatch.chdir(tmp_path) + + def git(*args): + return subprocess.run( + ["git", *args], check=True, capture_output=True, text=True, + ).stdout.strip() + + git("init", "-q") + git("config", "user.name", "Test") + git("config", "user.email", "test@example.com") + + def commit(text): + (tmp_path / "perf-changelog.yaml").write_text(text) + git("add", ".") + git("commit", "-qm", "changelog") + return git("rev-parse", "HEAD") + + return commit + + +def test_added_lines_report_a_failing_git_diff(commit_changelog, tmp_path, monkeypatch): + base = commit_changelog("- old\n") + head = commit_changelog("- old\n- new\n") + stub = tmp_path / "bin/git" + stub.parent.mkdir() + stub.write_text( + "#!/bin/sh\n" + 'if [ "$1" = diff ]; then echo "fatal: diff exploded" >&2; exit 128; fi\n' + f'exec {shutil.which("git")} "$@"\n' + ) + stub.chmod(0o755) + monkeypatch.setenv("PATH", f"{stub.parent}{os.pathsep}{os.environ['PATH']}") + + with pytest.raises(RuntimeError, match="fatal: diff exploded"): + process_changelog.get_added_lines(base, head, "perf-changelog.yaml") + + +def test_added_lines_reject_deleting_a_line_that_starts_with_dashes(commit_changelog): + base = commit_changelog("- a\n---\n- b\n") + head = commit_changelog("- a\n- b\n- c\n") + with pytest.raises(ValueError, match="Deletions are not allowed.*Found deleted line: ---"): + process_changelog.get_added_lines(base, head, "perf-changelog.yaml") + + +def test_added_lines_keep_an_added_line_that_starts_with_pluses(commit_changelog): + base = commit_changelog("- a\n") + head = commit_changelog("- a\n++ b\n") + assert process_changelog.get_added_lines(base, head, "perf-changelog.yaml") == "++ b" diff --git a/inferencex-e2e/infx/tests/matrix/test_revision.py b/inferencex-e2e/infx/tests/matrix/test_revision.py index 06fbe681ac..c17d1ff8a4 100644 --- a/inferencex-e2e/infx/tests/matrix/test_revision.py +++ b/inferencex-e2e/infx/tests/matrix/test_revision.py @@ -102,6 +102,43 @@ def advance_after_listing(command, **kwargs): assert not snapshot.exists() +@pytest.mark.parametrize("ref,expected", [("older", ("older", 2)), ("newer", ("newer", 6))]) +def test_batch_generation_runs_each_revisions_own_generator( + generation_repo, ref, expected, monkeypatch +): + def no_process_per_request(*_): + raise AssertionError("a request fell back to its own process") + + monkeypatch.setattr(revision.Revision, "generate", no_process_per_request) + with revision.snapshot(ref) as inputs: + batches = inputs.generate_each([(["fixture"], ["--no-evals"], None)] * 3, 2) + + assert [[(row["model"], row["conc"]) for row in rows] for rows in batches] == [[expected]] * 3 + + +def test_batch_generation_raises_the_generators_own_failure(generation_repo): + requests = [(["fixture"], ["--no-evals"], None), (["absent"], ["--no-evals"], None)] + with revision.snapshot("newer") as inputs, pytest.raises(subprocess.CalledProcessError) as error: + inputs.generate_each(requests, 2) + + assert "absent" in error.value.stderr + + +@pytest.mark.parametrize("flush", [False, True]) +def test_batch_generation_fails_like_each_run_when_an_import_writes_to_stdout( + generation_repo, flush +): + _, git = generation_repo + source = git("show", "newer:infx/matrix/validation.py") + Path("infx/matrix/validation.py").write_text(f"print('imported', flush={flush})\n{source}\n") + git("add", "infx/matrix/validation.py") + git("commit", "-qm", "noisy import") + + with revision.snapshot("HEAD") as inputs, pytest.raises(json.JSONDecodeError): + inputs.generate_each([(["fixture"], ["--no-evals"], None)] * 3, 1) + + + def test_historical_generation_rejects_missing_inputs(generation_repo): _, git = generation_repo git("rm", "-f", "configs/runners.yaml") @@ -234,6 +271,8 @@ def test_snapshot_runs_pre_root_move_revisions_with_their_own_config_directory(t print(json.dumps({{"planner": NAME, "args": sys.argv[1:], "cwd": os.getcwd(), "root": os.environ["INFERENCEX_REPOSITORY_ROOT"], "env": dict(os.environ)}})) print("planner diagnostics", file=sys.stderr) +if "--kill" in sys.argv: + os.kill(os.getpid(), 9) sys.exit(int(sys.argv[sys.argv.index("--exit") + 1]) if "--exit" in sys.argv else 0) """ @@ -281,6 +320,14 @@ def test_cli_runs_the_checkouts_own_planner_and_forwards_its_output_and_status(t assert result.stderr == "planner diagnostics\n" +def test_cli_exits_128_plus_the_signal_that_killed_the_tool(tmp_path): + checkout = planner_checkout(tmp_path / "checkout", "module") + + result = run_cli(tmp_path, "plan", str(checkout), "--kill") + + assert result.returncode == 137, result.stderr + + def test_cli_runs_the_tool_without_the_callers_credentials(tmp_path): checkout = planner_checkout(tmp_path / "checkout", "module") caller = { diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index 54413d0a28..696a0652d0 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -632,6 +632,34 @@ def test_conc_list_values_must_be_positive(self): }) assert "must be greater than 0" in str(exc_info.value) + def test_conc_list_values_must_be_unique(self): + with pytest.raises(ValidationError, match="must not repeat"): + SingleNodeSearchSpaceEntry(**{ + "tp": 4, + "conc-list": [4, 8, 4], + }) + + @pytest.mark.parametrize("bound", ["conc-start", "conc-end"]) + def test_conc_list_rejects_a_lone_range_bound(self, bound): + with pytest.raises(ValidationError, match="Cannot specify both"): + SingleNodeSearchSpaceEntry(**{ + "tp": 4, + "conc-list": [4], + bound: 1, + }) + + +@pytest.mark.parametrize(("schema", "entry"), [ + (SingleNodeSearchSpaceEntry, {"tp": 0, "conc-list": [4]}), + (AgenticCodingSearchSpaceEntry, {"tp": -2, "kv-offloading": "none", "conc-list": [4]}), + (WorkerConfig, {"num-worker": 1, "tp": 0, "ep": 1, "dp-attn": False}), + (MultiNodeSearchSpaceEntry, + {"worker": {"tp": 0, "ep": 1, "dp-attn": False}, "num-nodes": 1, "conc-list": [4]}), +]) # fmt: skip +def test_tensor_parallelism_must_be_positive(schema, entry): + with pytest.raises(ValidationError, match="greater than 0"): + schema(**entry) + class TestMultiNodeSearchSpaceEntry: diff --git a/inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py b/inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py new file mode 100644 index 0000000000..b55d6e4f8a --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_analyze_benchmark_distributions.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +E2E_ROOT = Path(__file__).resolve().parents[4] + + +def _record(isl: int, osl: int, *, phase: str = "profiling", error: dict | None = None) -> dict: + return { + "metadata": {"benchmark_phase": phase}, + "metrics": { + "input_sequence_length": {"value": isl, "unit": "tokens"}, + "output_sequence_length": {"value": osl, "unit": "tokens"}, + }, + "error": error, + } + + +def test_summary_excludes_warmup_and_failed_requests(tmp_path: Path) -> None: + artifacts = tmp_path / "aiperf_artifacts" + artifacts.mkdir() + rows = [ + _record(9000, 900, phase="warmup"), + _record(100, 10), + _record(5000, 500, error={"type": "HTTPStatusError", "message": "500"}), + _record(300, 30), + ] + (artifacts / "profile_export.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows)) + + subprocess.run( + [sys.executable, "-m", "infx.results.agentic.analyze_benchmark_distributions", + str(artifacts), "-o", str(tmp_path / "out")], + cwd=E2E_ROOT, check=True, capture_output=True, text=True, timeout=30, + ) + + summary = (tmp_path / "out" / "workload_distribution_summary.txt").read_text() + assert "Total requests: 2\n" in summary + assert "ALL REQUESTS ISL:\n n=2 mean=200 " in summary + assert "ALL REQUESTS OSL:\n n=2 mean=20 " in summary diff --git a/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py b/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py index a6da1f2848..9715c0729d 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py @@ -287,6 +287,20 @@ def test_build_power_window_rejects_ambiguous_inputs( assert expected_reason in reasons +@pytest.mark.parametrize( + ("aggregate", "records"), + [(["2023-11-14T22:13:21+00:00"], None), (None, [["not", "an", "object"]])], +) +def test_build_power_window_rejects_non_object_profile_json( + tmp_path: Path, aggregate: list | None, records: list | None +): + from infx.results.agentic.power_adapter import build_power_window + + result_dir = _write_artifacts(tmp_path, aggregate=aggregate, records=records) + + assert build_power_window(result_dir) == (None, ["profile_artifacts_invalid"]) + + _RETIRED_FORK_WINDOW_ENV = ( "SRT_MEASUREMENT_WINDOW_BENCHMARK_TYPE", "SRT_MEASUREMENT_WINDOW_CONCURRENCIES", diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index e162f49685..363ce7ba1a 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -1607,6 +1607,37 @@ def test_processor_normalizes_dynamo_server_metrics(tmp_path: Path): assert any(source["role"] == "prefill" for source in agg["server_metrics"]["sources"]) +@pytest.mark.parametrize(("cached", "router_shared", "expected"), [ + (0.0, 0.55, 0.0), + (25.0, None, 0.25), + (None, 0.55, 0.55), +]) +def test_dynamo_overall_cache_hit_rate_falls_back_to_frontend_then_router( + cached, router_shared, expected, +): + metrics = { + "dynamo_frontend_input_sequence_tokens": { + "type": "counter", "series": [{"stats": {"total": 100.0}}], + }, + } + if cached is not None: + metrics["dynamo_frontend_cached_tokens"] = { + "type": "counter", "series": [{"stats": {"total": cached}}], + } + if router_shared is not None: + metrics["dynamo_component_router_shared_cache_hit_rate"] = { + "type": "gauge", "series": [{"stats": {"avg": router_shared}}], + } + + _, nested, _ = compute_server_metrics( + {"metrics": metrics}, framework="dynamo-vllm", records=[], + ) + + assert nested["adapter"] == "dynamo-vllm" + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(expected) + + + def test_processor_uses_aiperf_theoretical_cache_metric(tmp_path: Path): """Aiperf's exported profile aggregate is the theoretical cache source.""" result_dir = _write_fixture(tmp_path) diff --git a/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py b/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py index 7cdf87142b..a1fd53410a 100644 --- a/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py +++ b/inferencex-e2e/infx/tests/results/power/test_aggregate_power.py @@ -57,17 +57,11 @@ def test_power_replacement_rejects_nonfinite_without_overwriting( assert path.read_bytes() == original -@pytest.mark.parametrize(("raw", "error_type"), [ - ("[]", TypeError), ('[["model","fixture"]]', TypeError), - ("null", AttributeError), ('"fixture"', AttributeError), - ("4", AttributeError), ("false", AttributeError), -]) -def test_power_replacement_rejects_nonobject_json( - tmp_path, patch_validated_power, raw, error_type -): +@pytest.mark.parametrize("raw", ["[]", '[["model","fixture"]]', "null", '"fixture"', "4", "false"]) +def test_power_replacement_rejects_nonobject_json(tmp_path, patch_validated_power, raw): path = tmp_path / "agg.json" path.write_text(raw) - with pytest.raises(error_type): + with pytest.raises(ValueError, match="aggregate is not a JSON object"): patch_validated_power(path, power_valid=False, metrics={}) assert path.read_text() == raw diff --git a/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py b/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py index f3d959881c..ec2198f047 100644 --- a/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py +++ b/inferencex-e2e/infx/tests/results/power/test_aggregate_power_multinode.py @@ -754,3 +754,17 @@ def test_v3_samples_reject_invalid_temperature(tmp_path, temperature): rows, reasons = apm.read_samples(path) assert rows == () assert reasons == ("samples_csv_malformed",) + + +@pytest.mark.parametrize("aggregate", ["[]", "7"]) +def test_non_object_aggregate_is_recorded_as_unwritable(tmp_path, aggregate): + pkg = build_package(tmp_path) + pkg.agg_result.write_text(aggregate) + + assert pkg.run(require_power=False) == 0 + sidecar = pkg.sidecar() + assert sidecar["power_valid"] is False + assert "aggregate_result_unwritable" in sidecar["reasons"] + assert sidecar["metrics"] == {} + assert pkg.agg_result.read_text() == aggregate + assert pkg.run(require_power=True) == 1 diff --git a/inferencex-e2e/infx/tests/results/test_agentic_point_files.py b/inferencex-e2e/infx/tests/results/test_agentic_point_files.py index d546ddaedb..d61c218828 100644 --- a/inferencex-e2e/infx/tests/results/test_agentic_point_files.py +++ b/inferencex-e2e/infx/tests/results/test_agentic_point_files.py @@ -24,3 +24,9 @@ def test_keeps_files_without_run_attempt(tmp_path: Path): first = write(tmp_path, "old_runner", ROW) second = write(tmp_path, "new_runner", {**ROW, "run_attempt": 2}) assert agentic_point_files(tmp_path) == sorted([first, second]) + + +def test_rerun_does_not_supersede_a_point_with_other_spec_decoding(tmp_path: Path): + mtp = write(tmp_path, "mtp_a1_runner", {**ROW, "spec_decoding": "mtp", "run_attempt": 1}) + plain = write(tmp_path, "none_a2_runner", {**ROW, "spec_decoding": "none", "run_attempt": 2}) + assert agentic_point_files(tmp_path) == sorted([mtp, plain]) diff --git a/inferencex-e2e/infx/tests/results/test_collect_eval_results.py b/inferencex-e2e/infx/tests/results/test_collect_eval_results.py index 5fc8d7a50e..393df8c53f 100644 --- a/inferencex-e2e/infx/tests/results/test_collect_eval_results.py +++ b/inferencex-e2e/infx/tests/results/test_collect_eval_results.py @@ -21,6 +21,37 @@ ) +@pytest.mark.parametrize("environ", [ + {"IS_MULTINODE": "false", "TP": "8", "PP_SIZE": "2", "DCP_SIZE": "8"}, + {"IS_MULTINODE": "true", "PREFILL_TP": "4", "PREFILL_PP_SIZE": "2", + "DECODE_TP": "8", "DECODE_DCP_SIZE": "4", "DECODE_PCP_SIZE": "2"}, + {"IS_MULTINODE": "true", "PREFILL_TP": "1", "PREFILL_DWDP_SIZE": "8", "DECODE_TP": "8"}, +]) +def test_collected_rows_match_raw_identity_with_parallelism( + tmp_path: Path, environ: dict[str, str], +) -> None: + from infx.bench.eval import meta + from infx.results.eval_artifacts import inspect_eval_artifacts + + raw = tmp_path / "eval_point" + raw.mkdir() + meta.write(raw / "meta_env.json", meta.build( + {"RUNNER_TYPE": "b200", "FRAMEWORK": "vllm", "PRECISION": "fp8", **environ}, + conc=4, suite="gsm8k", + )) + (raw / "results_2026-01-01T00-00-00.json").write_text(json.dumps({ + "lm_eval_version": "0.4.0", + "results": {"gsm8k": {"exact_match,strict-match": 0.9}}, + "configs": {"gsm8k": {"filter_list": [{"name": "strict-match"}]}}, + })) + (tmp_path / "eval_results_all").mkdir() + (tmp_path / "eval_results_all" / "agg_eval_all.json").write_text( + json.dumps(collect_eval_rows(tmp_path)) + ) + + assert inspect_eval_artifacts(tmp_path)[1] == [] + + @pytest.mark.parametrize("batched", [False, True]) def test_result_selection_accepts_empty_candidates(batched: bool) -> None: assert select_latest_results(iter(()), batched=batched) == [] @@ -137,6 +168,29 @@ def test_invalid_primary_does_not_fall_back_to_valid_secondary(tmp_path: Path, s } +def test_one_sample_stderr_sentinel_is_published_as_null(tmp_path: Path) -> None: + # lm-eval writes the string "N/A" as the stderr of a metric with one sample. + (tmp_path / "meta_env.json").write_text("{}") + (tmp_path / "results.json").write_text(json.dumps({ + "lm_eval_version": "test", + "results": {"gsm8k": { + "exact_match,strict-match": 1.0, "exact_match_stderr,strict-match": "N/A", + "exact_match,flexible-extract": 0.0, "exact_match_stderr,flexible-extract": "N/A", + }}, + "configs": {"gsm8k": {"filter_list": [ + {"name": "strict-match"}, {"name": "flexible-extract"}, + ]}}, + "n-samples": {"gsm8k": {"original": 1319, "effective": 1}}, + })) + + [row] = collect_eval_rows(tmp_path) + + assert (row["score"], row["em_strict"], row["em_flexible"]) == (1.0, 1.0, 0.0) + assert (row["score_se"], row["em_strict_se"], row["em_flexible_se"]) == (None, None, None) + assert row["infrastructure_success"] is True + + + @pytest.mark.parametrize("error,expected", [ (None, None), ("", {"type": "IntegrationError", "message": ""}), @@ -352,6 +406,19 @@ def test_collector_cli_runs_outside_checkout(tmp_path: Path) -> None: assert rows[0]["source"] == str(result) +def test_collect_eval_rows_reads_eval_sets_in_name_order(tmp_path: Path) -> None: + scores = {"tp8_dpa": 0.8, "tp4": 0.4, "tp8": 0.81, "tp2": 0.2, "tp4_dpa": 0.41, "tp1": 0.1} + for name, score in scores.items(): + artifact = tmp_path / f"eval_dsr1_8k1k_{name}_conc64" + artifact.mkdir() + (artifact / "meta_env.json").write_text('{"conc": 64}') + _write_lm_eval_result(artifact / "results.json", score) + + rows = collect_eval_rows(tmp_path) + + assert [row["score"] for row in rows] == [0.1, 0.2, 0.4, 0.41, 0.81, 0.8] + + def test_collect_eval_rows_expands_batched_concurrencies( tmp_path: Path, ) -> None: diff --git a/inferencex-e2e/infx/tests/results/test_compare_results.py b/inferencex-e2e/infx/tests/results/test_compare_results.py new file mode 100644 index 0000000000..3fcbd9a055 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/test_compare_results.py @@ -0,0 +1,114 @@ +"""The PR summary compares each fixed-sequence result with main's latest row of its config.""" + +import json +import re +import sqlite3 +import sys + +import pytest + +import infx.results.compare_results as compare + +SCHEMA = """ +CREATE TABLE configs ( + id INTEGER PRIMARY KEY, hardware TEXT, framework TEXT, model TEXT, precision TEXT, + spec_method TEXT, disagg BOOLEAN, is_multinode BOOLEAN, prefill_tp INTEGER, + prefill_ep INTEGER, prefill_dp_attention BOOLEAN, decode_tp INTEGER, decode_ep INTEGER, + decode_dp_attention BOOLEAN +); +CREATE TABLE workflow_runs (id INTEGER PRIMARY KEY, head_branch TEXT); +CREATE TABLE benchmark_results ( + config_id INTEGER, workflow_run_id INTEGER, metrics TEXT, isl INTEGER, osl INTEGER, + conc INTEGER, date TEXT, error TEXT +); +INSERT INTO workflow_runs VALUES (1, 'main'); +""" + + +class Database: + """An in-memory SQL database behind psycopg2's pyformat parameters.""" + + def __init__(self, configs: list[tuple[str, str, str, float]], framework: str = "sglang"): + self.connection = sqlite3.connect(":memory:") + self.connection.executescript(SCHEMA) + for index, (model, hardware, date, tput) in enumerate(configs, start=1): + self.connection.execute( + "INSERT INTO configs VALUES (?, ?, ?, ?, 'fp4', 'none', 0, 0, 8, 1, 0, 8, 1, 0)", + (index, hardware, framework, model), + ) + self.connection.execute( + "INSERT INTO benchmark_results VALUES (?, 1, ?, 8192, 1024, 4, ?, NULL)", + (index, json.dumps({"tput_per_gpu": tput}), date), + ) + + def cursor(self): + return self + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, query: str, params: dict) -> None: + query = re.sub(r"%\((\w+)\)s", r":\1", query).replace("%%", "%") + self.rows = self.connection.execute(query, params).fetchall() + + def fetchone(self): + return self.rows[0] if self.rows else None + + def close(self) -> None: + self.connection.close() + + +def summary(tmp_path, monkeypatch, capsys, database: Database, **result) -> str: + row = { + "hw": "b200", "infmax_model_prefix": "dsv4", "model": "deepseek-ai/DeepSeek-V4-Pro", + "framework": "sglang", "precision": "fp4", "spec_decoding": "none", "disagg": False, + "is_multinode": False, "tp": 8, "ep": 1, "dp_attention": False, "isl": 8192, + "osl": 1024, "conc": 4, "tput_per_gpu": 110.0, **result, + } # fmt: skip + (tmp_path / "agg.json").write_text(json.dumps([row])) + monkeypatch.setenv("DATABASE_URL", "postgresql://unused") + monkeypatch.setattr(compare.psycopg2, "connect", lambda url: database) + monkeypatch.setattr(sys, "argv", ["compare_results", str(tmp_path)]) + compare.main() + return capsys.readouterr().out + + +def test_a_model_prefix_is_not_compared_with_a_longer_prefix_it_begins( + tmp_path, monkeypatch, capsys +): + database = Database( + [("dsv4", "b200", "2026-10-01", 100.0), ("dsv41flash", "b200", "2026-10-20", 400.0)] + ) + + out = summary(tmp_path, monkeypatch, capsys, database) + + assert r"\text{+10.00}$ (+10.0%)" in out + + +@pytest.mark.parametrize( + ("runner", "hardware"), [("cluster:b200-nscale", "b200"), ("gb300-nv", "gb300")] +) +def test_a_runner_label_finds_the_baseline_of_its_gpu( + tmp_path, monkeypatch, capsys, runner, hardware +): + database = Database([("dsv4", hardware, "2026-10-01", 100.0)]) + + out = summary(tmp_path, monkeypatch, capsys, database, hw=runner) + + assert r"\text{+10.00}$ (+10.0%)" in out + + +def test_sglang_disagg_finds_the_baseline_ingestion_stored_as_mori_sglang( + tmp_path, monkeypatch, capsys +): + database = Database([("dsr1", "mi355x", "2026-10-01", 100.0)], framework="mori-sglang") + + out = summary( + tmp_path, monkeypatch, capsys, database, + hw="cluster:mi355x-amds", infmax_model_prefix="dsr1", framework="sglang-disagg", + ) # fmt: skip + + assert r"\text{+10.00}$ (+10.0%)" in out diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index da3d073d34..7e1751e72b 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -8,7 +8,7 @@ import pytest import yaml -from infx.srt_slurm.single_node import runtime_arguments, select_recipe, submission_fields +from infx.srt_slurm.single_node import main, runtime_arguments, select_recipe, submission_fields from infx.srt_slurm.synthetic_acceptance import Planned, plan_commands ROOT = Path(__file__).resolve().parents[3] @@ -114,6 +114,32 @@ def test_ambiguous_native_variants_are_rejected(point): runtime_arguments(str(path), env) +def test_a_variant_missing_a_required_field_is_reported_not_raised(point): + path, recipe, env = point + broken = {"roles": {"agg": {"args": {"tensor-parallel-size": None}}}} + path.write_text(yaml.safe_dump({"base": recipe, "override_good": {}, "override_broken": broken})) + assert select_recipe(str(path), env)[0] == f"{path}:override_good" + with pytest.raises(ValueError, match="override_broken: .*'tensor-parallel-size'"): + select_recipe(f"{path}:override_broken", env) + + +def test_prepare_refuses_an_arguments_file_named_like_the_bound_recipe( + point, tmp_path, monkeypatch, capsys +): + path, _, env = point + for name, value in env.items(): + monkeypatch.setenv(name, value) + output = tmp_path / "job" / "recipe.yaml" + output.parent.mkdir() + argv = ["single_node", "prepare", f"{path}:base", str(output), "--fabric", "{}"] + monkeypatch.setattr(sys, "argv", argv) + with pytest.raises(SystemExit) as exit_info: + main() + assert exit_info.value.code == 2 + assert "would overwrite the bound recipe" in capsys.readouterr().err + assert not output.exists() + + @pytest.mark.parametrize("algorithm,spec_decoding", [("EAGLE", "mtp"), ("DFLASH", "draft_model")]) def test_speculative_binding_uses_real_verification_and_the_chat_template( point, tmp_path, algorithm, spec_decoding @@ -266,6 +292,7 @@ def test_sglang_decode_context_parallel_size_must_match_the_point(point, flag): ({"status": "error"}, None), ({"status": "submitted", "slurm_job_id": "42;43", "output_dir": "/shared/42"}, None), ({"status": "submitted", "slurm_job_id": "42", "output_dir": "relative"}, None), + (["42"], None), ]) def test_submission_manifest(tmp_path, record, expected): path = tmp_path / "submission.json" diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index 0e5c429cee..e687fc922c 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -12,6 +12,7 @@ import yaml from infx.srt_slurm.synthetic_acceptance import ( + acceptance_length, build_overrides, plan_commands, selected_recipes, @@ -202,6 +203,42 @@ def test_adaptive_dspark_skips_golden_acceptance_on_throughput(golden_dir: Path) assert spec["rejection_sample_method"] == "synthetic" +def test_adaptive_verification_records_no_golden_acceptance(golden_dir: Path) -> None: + adaptive = { + "args": { + "speculative-config": json.dumps( + {"method": "dspark", "num_speculative_tokens": 5, "enable_adaptive_verification": True} + ) + } + } + recipe = {"roles": {"prefill": adaptive, "decode": adaptive}} + env = {**ENV, "SPEC_DECODING": "dspark"} + + assert acceptance_length(recipe, "vllm-disagg", env, golden_dir=golden_dir) is None + assert build_overrides(recipe, "vllm-disagg", env, golden_dir=golden_dir) == [] + + +def test_adaptive_generation_records_no_golden_acceptance_while_prefill_simulates( + golden_dir: Path, +) -> None: + decode = vllm_recipe("dspark", enable_adaptive_verification=True)["roles"]["agg"] + prefill = vllm_recipe("dspark")["roles"]["agg"] + recipe = {"roles": {"prefill": prefill, "decode": decode}} + env = {**ENV, "SPEC_DECODING": "dspark"} + + assert acceptance_length(recipe, "vllm-disagg", env, golden_dir=golden_dir) is None + result = apply_native( + recipe, build_overrides(recipe, "vllm-disagg", env, golden_dir=golden_dir) + ) + assert result["roles"]["decode"] == decode + assert json.loads(result["roles"]["prefill"]["args"]["speculative-config"]) == { + "method": "dspark", + "num_speculative_tokens": 3, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_length": 2.7, + } + + @pytest.mark.parametrize("sampling", [None, "unknown"]) def test_kimi_curve_requires_an_explicit_supported_sampler( golden_dir: Path, sampling: str | None @@ -609,7 +646,12 @@ def test_caller_json_is_merged_before_golden_selection(tmp_path: Path, golden_di } -def test_cli_forwards_options_and_submission_failure(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("ending", "returncode"), [("sys.exit(7)", 7), ("os.kill(os.getpid(), signal.SIGKILL)", 137)] +) +def test_cli_forwards_options_and_submission_failure( + tmp_path: Path, ending: str, returncode: int +) -> None: recipe = tmp_path / "recipe.yaml" recipe.write_text( yaml.safe_dump( @@ -618,7 +660,8 @@ def test_cli_forwards_options_and_submission_failure(tmp_path: Path) -> None: ) binary = tmp_path / "srtctl" binary.write_text( - f"#!{sys.executable}\nimport json,sys\nprint(json.dumps(sys.argv[1:]))\nsys.exit(7)\n" + f"#!{sys.executable}\nimport json,os,signal,sys\n" + f"print(json.dumps(sys.argv[1:]), flush=True)\n{ending}\n" ) binary.chmod(0o755) record = tmp_path / "acceptance.json" @@ -639,7 +682,7 @@ def test_cli_forwards_options_and_submission_failure(tmp_path: Path) -> None: capture_output=True, text=True, ) - assert result.returncode == 7, result.stderr + assert result.returncode == returncode, result.stderr assert json.loads(record.read_text()) == {"golden_acceptance_length": None} argv = json.loads(result.stdout) assert argv[:5] == ["apply", "-f", str(recipe), "--tags", "a b"] diff --git a/inferencex-e2e/infx/tests/test_github.py b/inferencex-e2e/infx/tests/test_github.py index 9dd05935ab..b2c4245974 100644 --- a/inferencex-e2e/infx/tests/test_github.py +++ b/inferencex-e2e/infx/tests/test_github.py @@ -14,6 +14,8 @@ ("gh: Not Found (HTTP 404)\n", 404), ("gh: Forbidden (HTTP 403)", 403), ("gh: Bad Gateway (HTTP 502)", 502), + ('gh: Not Found (HTTP 404)\ngh: This API operation needs the "admin:org" scope. To request ' + "it, run: gh auth refresh -h github.com -s admin:org\n", 404), ("network timeout mentioning 404 without HTTP status", None), ]) def test_authenticated_api_errors_expose_status_without_guessing(monkeypatch, stderr, status): diff --git a/inferencex-e2e/infx/tests/test_module_entrypoints.py b/inferencex-e2e/infx/tests/test_module_entrypoints.py index 93edc5b367..ceca5fbe02 100644 --- a/inferencex-e2e/infx/tests/test_module_entrypoints.py +++ b/inferencex-e2e/infx/tests/test_module_entrypoints.py @@ -44,6 +44,20 @@ def test_collector_preserves_nested_json_and_empty_inputs(invoke, tmp_path, payl assert json.loads((tmp_path / "agg_test.json").read_text()) == expected +def test_collector_orders_rows_by_artifact_path(invoke, tmp_path): + for name in ("tp8_conc64", "tp4_conc4", "tp8_conc4", "tp2_conc16", "tp4_conc64", "tp1_conc1"): + artifact = tmp_path / "inputs" / f"bmk_dsr1_1k1k_{name}" + artifact.mkdir(parents=True) + (artifact / f"dsr1_1k1k_{name}.json").write_text(json.dumps({"point": name})) + + result = invoke("infx.results.collect_results", "inputs", "test") + + assert result.returncode == 0, result.stderr + assert [row["point"] for row in json.loads((tmp_path / "agg_test.json").read_text())] == [ + "tp1_conc1", "tp2_conc16", "tp4_conc4", "tp4_conc64", "tp8_conc4", "tp8_conc64", + ] + + @pytest.mark.parametrize("published", [None, b'[{"previous": true}]\n']) def test_collector_does_not_publish_partial_output_on_invalid_json(invoke, tmp_path, published): inputs = tmp_path / "inputs" diff --git a/inferencex-e2e/infx/tests/test_yamlio.py b/inferencex-e2e/infx/tests/test_yamlio.py new file mode 100644 index 0000000000..5cc685a80f --- /dev/null +++ b/inferencex-e2e/infx/tests/test_yamlio.py @@ -0,0 +1,95 @@ +import pytest +import yaml + +from infx import yamlio + + +def test_stream_errors_name_the_file_with_pyyamls_wording(tmp_path): + path = tmp_path / "bad.yaml" + path.write_text("a: [1\n") + + with path.open() as stream, pytest.raises(yaml.YAMLError) as error: + yamlio.safe_load(stream) + + assert str(error.value) == ( + f'while parsing a flow sequence\n in "{path}", line 1, column 4\n' + f"expected ',' or ']', but got ''\n in \"{path}\", line 2, column 1" + ) + + +def test_a_text_that_fails_to_parse_raises_on_every_load(): + for _ in range(2): + with pytest.raises(yaml.YAMLError, match="expected ',' or ']'"): + yamlio.safe_load("a: [1\n") + + +def test_custom_loader_constructors_apply(): + with pytest.raises(yaml.YAMLError, match="found duplicate key 'a'"): + yamlio.load("x: 1\ny: {a: 1, a: 2}\n", yamlio.UniqueKeyLoader) + + +def test_one_text_loads_through_each_loader_separately(): + text = "a: 1\na: 2\n" + + assert yamlio.safe_load(text) == {"a": 2} + with pytest.raises(yaml.YAMLError, match="found duplicate key 'a'"): + yamlio.load(text, yamlio.UniqueKeyLoader) + + +def test_safe_unique_key_loader_merges_and_rejects_repeated_own_keys(): + merged = "base: &b {x: 1, y: 1}\nk: {<<: *b, y: 2}\n" + repeated = "base: &b {x: 1}\nk: {<<: *b, y: 2, y: 3}\n" + + assert yamlio.load(merged, yamlio.UniqueKeySafeLoader) == { + "base": {"x": 1, "y": 1}, + "k": {"x": 1, "y": 2}, + } + with pytest.raises(yaml.YAMLError, match="found duplicate key 'y'"): + yamlio.load(repeated, yamlio.UniqueKeySafeLoader) + + +@pytest.mark.parametrize("load", [yamlio.load, lambda text, loader: yaml.load(text, Loader=loader)]) +@pytest.mark.parametrize( + ("merging", "expected"), [("{<<: *b}", {"k": 2}), ("!!set {<<: *b}", {"k"})] +) +def test_safe_unique_key_loader_allows_overriding_a_mapping_flattened_elsewhere_first( + load, merging, expected +): + text = f"x:\n b: &b {{<<: &a {{k: 1}}, k: 2}}\ny: {merging}\n" + + assert load(text, yamlio.UniqueKeySafeLoader) == {"x": {"b": {"k": 2}}, "y": expected} + + +def test_safe_unique_key_loader_reports_a_non_mapping_map_as_safeloader_does(): + with pytest.raises(yaml.constructor.ConstructorError, match="expected a mapping node"): + yamlio.load("a: !!map x\n", yamlio.UniqueKeySafeLoader) + + +def test_repeated_loads_return_independent_values(): + text = "roles:\n agg: {gpus: 8}\n" + first = yamlio.safe_load(text) + first["roles"]["agg"]["gpus"] = 4 + + assert yamlio.safe_load(text) == {"roles": {"agg": {"gpus": 8}}} + + +def test_repeated_loads_keep_anchor_sharing(): + text = "base: &env {A: '1'}\nother: *env\n" + yamlio.safe_load(text) + + loaded = yamlio.safe_load(text) + + assert loaded["base"] is loaded["other"] + + +def test_a_deep_alias_chain_loads_as_pyyaml_builds_it(): + chain = "".join(f"k{i}: &a{i} [{f'*a{i - 1}' if i else 'x'}]\n" for i in range(1200)) + text = "first: null\n" + chain + "first: *a1199\n" + yamlio.safe_load(text) + + loaded = yamlio.safe_load(text) + + depth, node = 0, loaded["first"] + while isinstance(node, list): + depth, node = depth + 1, node[0] + assert (depth, node, loaded["first"] is loaded["k1199"]) == (1200, "x", True) diff --git a/inferencex-e2e/infx/tests/workflows/test_ci_priority.py b/inferencex-e2e/infx/tests/workflows/test_ci_priority.py index d0f8b78460..3c0a8541ca 100644 --- a/inferencex-e2e/infx/tests/workflows/test_ci_priority.py +++ b/inferencex-e2e/infx/tests/workflows/test_ci_priority.py @@ -1,11 +1,13 @@ from decimal import Decimal import pytest +import yaml from infx.workflows.ci_priority import ( PriorityContext, annotate_jobs, calculate_priority, + load_policy, queue_token, ) @@ -32,6 +34,19 @@ def policy(): } +@pytest.mark.parametrize(("misspell", "unknown"), [ + (lambda p: p["adjustments"].update({"multinode": p["adjustments"].pop("multi-node")}), + "adjustments.multinode"), + (lambda p: p.update({"patchwork_score": p.pop("patchwork-score")}), "patchwork_score"), +]) +def test_load_policy_rejects_misspelled_keys(policy, tmp_path, misspell, unknown): + misspell(policy) + path = tmp_path / "ci-priority.yaml" + path.write_text(yaml.safe_dump(policy)) + with pytest.raises(ValueError, match=f"^Unknown CI priority policy keys: {unknown}$"): + load_policy(path) + + def test_combined_signals_add_their_configured_weights(policy): baseline = { "runner": "h100", diff --git a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py index d92e82db8f..3ba4626198 100644 --- a/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py +++ b/inferencex-e2e/infx/tests/workflows/test_merge_with_reuse.py @@ -2,14 +2,19 @@ from __future__ import annotations +import json import os import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch import git as gitpython import pytest +import requests from infx.workflows.merge_with_reuse import ( changelog_path, @@ -20,6 +25,7 @@ _latest_statuses, _MergeState, _poll_pr_head, + _make_github, _resolve_token, _retry_delay, find_eligible_run, @@ -100,6 +106,7 @@ def make_check_run( started_at: str = "2024-01-01T00:00:00Z", details_url: str = "", cr_id: int = 1, + check_suite_id: int | None = None, ) -> MagicMock: """Build a mock CheckRun.""" cr = MagicMock() @@ -109,6 +116,7 @@ def make_check_run( cr.started_at = started_at cr.details_url = details_url cr.id = cr_id + cr.check_suite_id = check_suite_id return cr @@ -350,6 +358,95 @@ def test_transient_error_retried_then_succeeds(self): result = wait_for_check("a" * 40, "check-changelog", gh_repo, timeout=60) assert result == 0 + def test_unstarted_rerun_supersedes_completed_failure(self): + old = make_check_run( + name="check-changelog", conclusion="failure", started_at="2024-01-01T00:00:00Z", cr_id=10 + ) + queued = make_check_run( + name="check-changelog", status="queued", conclusion=None, started_at=None, cr_id=20 + ) + passed = make_check_run( + name="check-changelog", started_at="2024-01-01T01:00:00Z", cr_id=20 + ) + first, second = MagicMock(), MagicMock() + first.get_check_runs.return_value = [old, queued] + second.get_check_runs.return_value = [old, passed] + gh_repo = MagicMock() + gh_repo.get_commit.side_effect = [first, second] + + with patch("infx.workflows.merge_with_reuse.time.sleep"): + result = wait_for_check("a" * 40, "check-changelog", gh_repo, timeout=60) + + assert result == 0 + + def test_later_skipped_run_does_not_replace_verdict(self): + """An unrelated label event re-runs the workflow with check-changelog skipped.""" + verdict = make_check_run( + name="check-changelog", started_at="2024-01-01T00:00:05Z", cr_id=10 + ) + skipped = make_check_run( + name="check-changelog", conclusion="skipped", started_at="2024-01-01T00:00:31Z", + cr_id=20, + ) + gh_repo = MagicMock() + gh_repo.get_commit.return_value.get_check_runs.return_value = [verdict, skipped] + + assert wait_for_check("a" * 40, "check-changelog", gh_repo, timeout=60) == 0 + + def test_server_errors_past_client_retries_are_retried(self, monkeypatch): + """PyGithub's own 5xx retries end in requests' RetryError, which must not abort.""" + sha = "a" * 40 + state = {"healthy": False, "check_polls": 0} + + class GitHub(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_GET(self): + base = f"http://127.0.0.1:{self.server.server_port}/repos/owner/repo" + path = self.path.partition("?")[0] + if path == "/repos/owner/repo": + status, body = 200, {"full_name": "owner/repo", "url": base} + elif path == f"/repos/owner/repo/commits/{sha}": + status, body = 200, {"sha": sha, "url": f"{base}/commits/{sha}"} + elif path == f"/repos/owner/repo/commits/{sha}/check-runs": + state["check_polls"] += 1 + status, body = (200, {"total_count": 1, "check_runs": [{ + "id": 1, "name": "check-changelog", "status": "completed", + "conclusion": "success", "started_at": "2024-01-01T00:00:00Z", + "details_url": None, + }]}) if state["healthy"] else (502, {"message": "Server Error"}) + else: + status, body = 404, {"message": "Not Found"} + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = ThreadingHTTPServer(("127.0.0.1", 0), GitHub) + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}) + thread.start() + monkeypatch.setenv("GITHUB_API_URL", f"http://127.0.0.1:{server.server_port}") + # PyGithub throttles with time.sleep too, so only the helper's own sleep heals GitHub. + monkeypatch.setattr( + "infx.workflows.merge_with_reuse.time", + SimpleNamespace( + monotonic=time.monotonic, sleep=lambda _seconds: state.update(healthy=True) + ), + ) + try: + gh_repo = _make_github("token").get_repo("owner/repo") + result = wait_for_check(sha, "check-changelog", gh_repo, timeout=60) + finally: + server.shutdown() + thread.join() + server.server_close() + + assert result == 0 + assert state["check_polls"] > 1 + @@ -541,6 +638,63 @@ def test_transient_error_retried_then_succeeds(self): result = wait_for_checks(pull, sha, gh_repo, timeout=60) assert result == 0 + @pytest.mark.parametrize( + ("second_workflow_id", "expected"), + [(2, 1), (1, 0)], + ids=["other-workflow-failure-blocks", "same-workflow-rerun-supersedes"], + ) + def test_same_job_name_in_two_workflows(self, second_workflow_id, expected): + gh_repo = MagicMock() + commit = MagicMock() + gh_repo.get_commit.return_value = commit + commit.get_check_runs.return_value = [ + make_check_run( + name="setup", + conclusion="failure", + started_at="2024-01-01T00:00:00Z", + cr_id=10, + check_suite_id=100, + ), + make_check_run( + name="setup", + conclusion="success", + started_at="2024-01-01T01:00:00Z", + cr_id=20, + check_suite_id=200, + ), + ] + gh_repo.get_workflow_runs.return_value = [ + SimpleNamespace(check_suite_id=100, workflow_id=1, event="pull_request"), + SimpleNamespace( + check_suite_id=200, workflow_id=second_workflow_id, event="pull_request" + ), + ] + commit.get_combined_status.return_value = SimpleNamespace(statuses=[]) + + result = wait_for_checks(MagicMock(), "a" * 40, gh_repo, timeout=10) + + assert result == expected + + def test_unstarted_rerun_supersedes_completed_failure(self): + old = make_check_run( + name="build", conclusion="failure", started_at="2024-01-01T00:00:00Z", cr_id=10 + ) + queued = make_check_run( + name="build", status="queued", conclusion=None, started_at=None, cr_id=20 + ) + passed = make_check_run(name="build", started_at="2024-01-01T01:00:00Z", cr_id=20) + first, second = MagicMock(), MagicMock() + first.get_check_runs.return_value = [old, queued] + second.get_check_runs.return_value = [old, passed] + for commit in (first, second): + commit.get_combined_status.return_value = SimpleNamespace(statuses=[]) + gh_repo = MagicMock() + gh_repo.get_commit.side_effect = [first, second] + + with patch("infx.workflows.merge_with_reuse.time.sleep"): + result = wait_for_checks(MagicMock(), "a" * 40, gh_repo, timeout=60) + + assert result == 0 @@ -580,14 +734,36 @@ def test_github_429_is_transient(self): ) assert _is_transient_error(exc) is True + @pytest.mark.parametrize( + ("exc_type", "message", "expected"), + [ + ("GithubException", "Resource not accessible by integration", False), + ("RateLimitExceededException", "API rate limit exceeded for user", True), + ], + ) + def test_github_403_is_transient_only_when_rate_limited(self, exc_type, message, expected): + import github + + exc = getattr(github, exc_type)(status=403, data={"message": message}, headers={}) + assert _is_transient_error(exc) is expected + def test_github_404_is_not_transient(self): from github import GithubException exc = GithubException(status=404, data={"message": "Not Found"}, headers={}) assert _is_transient_error(exc) is False - def test_connection_error_is_transient(self): - assert _is_transient_error(ConnectionError("refused")) is True + @pytest.mark.parametrize( + "exc", + [ + ConnectionError("refused"), + requests.exceptions.ConnectionError("refused"), + requests.exceptions.ReadTimeout("read timed out"), + ], + ids=["builtin", "requests-connection", "requests-read-timeout"], + ) + def test_connection_error_is_transient(self, exc): + assert _is_transient_error(exc) is True def test_timeout_error_is_transient(self): assert _is_transient_error(TimeoutError("timed out")) is True diff --git a/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py b/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py index cc1fef602b..37b69704a8 100644 --- a/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py +++ b/inferencex-e2e/infx/tests/workflows/test_prepare_perf_changelog_merge.py @@ -131,6 +131,21 @@ def test_conflict_resolution_rejects_duplicate_remaining_contribution() -> None: ) +def test_conflict_resolution_keeps_pr_entry_repeating_a_historical_entry() -> None: + base = block("same-config", "https://github.com/SemiAnalysisAI/InferenceX/pull/1") + pr = base + b"\n" + block("same-config", "XXX") + main = base + b"\n" + block( + "main-config", + "https://github.com/SemiAnalysisAI/InferenceX/pull/41", + ) + + result = resolve_conflict_bytes(base, pr, main, 42, "SemiAnalysisAI/InferenceX") + + assert result == main + b"\n" + block( + "same-config", "https://github.com/SemiAnalysisAI/InferenceX/pull/42" + ) + + def test_conflict_resolution_separates_multiple_contributions_by_one_blank_line() -> None: base = block("base-config", "https://github.com/SemiAnalysisAI/InferenceX/pull/1") pr = base + b"\n" + block("new-a", "XXX") + b"\n" + block("new-b", "XXX") diff --git a/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py b/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py index c870b05d34..1b2519d4b2 100644 --- a/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py +++ b/inferencex-e2e/infx/tests/workflows/test_validate_perf_changelog.py @@ -43,6 +43,14 @@ def test_parse_changelog_rejects_missing_final_newline() -> None: parse_changelog(raw, "test changelog") +@pytest.mark.parametrize("line_break", ["\x85", "\u2028", "\u2029"]) +def test_parse_changelog_rejects_line_breaks_other_than_newline(line_break: str) -> None: + raw = render([entry("config-a")]) + line_break.encode() + render([entry("config-b")]) + + with pytest.raises(ChangelogValidationError, match="line break"): + parse_changelog(raw, "test changelog") + + def test_parse_changelog_rejects_malformed_nested_entry() -> None: raw = b"""- config-keys: - config-a @@ -74,6 +82,11 @@ def test_parse_changelog_rejects_duplicate_mapping_keys() -> None: parse_changelog(raw, "test changelog") +def test_parse_changelog_reports_a_tagged_non_mapping_entry_as_invalid_yaml() -> None: + with pytest.raises(ChangelogValidationError, match="expected a mapping node, but found scalar"): + parse_changelog(b"- !!map config-a\n", "test changelog") + + def test_compare_entries_allows_appended_pr_entry() -> None: base = [entry("config-a")] added = entry("config-b", "XXX") @@ -174,13 +187,14 @@ def test_raw_append_requires_exact_separator_and_final_newline() -> None: corrections=0, ) - with pytest.raises(ChangelogValidationError, match="end with one newline"): - validate_raw_change( - base, - base + b"\n" + first + b"\n", - additions=1, - corrections=0, - ) + for trailing in (b"\n", b"\n \n", b" \n"): + with pytest.raises(ChangelogValidationError, match="end with one newline"): + validate_raw_change( + base, + base + b"\n" + first + trailing, + additions=1, + corrections=0, + ) def test_raw_correction_rejects_whitespace_only_history_change() -> None: diff --git a/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py b/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py index 532b8ab7a6..9a07b76023 100644 --- a/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py +++ b/inferencex-e2e/infx/tests/workflows/test_validate_reusable_sweep_artifacts.py @@ -409,6 +409,19 @@ def write_agentic_artifacts( (root / f"agentic_{result_name}").mkdir() +def test_eval_validation_matches_mixed_case_raw_meta_to_lowercased_aggregate( + tmp_path: Path, +) -> None: + write_raw_eval_artifact(tmp_path, 32) + meta_path = tmp_path / "eval_result_conc32_h100-dgxc-slurm_00" / "meta_env.json" + meta_path.write_text( + json.dumps({**single_eval_meta(32), "framework": "VLLM", "precision": "FP4"}) + ) + write_eval_aggregate(tmp_path, [single_eval_result(32)]) + + assert validate_eval_artifacts(tmp_path) == [] + + def test_eval_validation_accepts_legacy_results_alongside_debug_artifacts( tmp_path: Path, ) -> None: @@ -947,6 +960,19 @@ def test_agentic_validation_requires_point_and_raw_artifacts( assert "agentic aggregate artifacts contain 1 unexpected row(s)" in errors +def test_agentic_validation_ignores_raw_dir_of_superseded_attempt(tmp_path: Path) -> None: + for attempt in (1, 2): + name = f"agentic_dsv4_conc16_a{attempt}_runner" + point_dir = tmp_path / f"bmk_{name}" + point_dir.mkdir() + (point_dir / "point.json").write_text( + json.dumps({**agentic_result(), "run_attempt": attempt}) + ) + (tmp_path / name).mkdir() + + assert validate_agentic_artifacts(tmp_path) == [] + + def test_agentic_validation_rejects_duplicate_point_identity( tmp_path: Path, ) -> None: diff --git a/inferencex-e2e/infx/workflows/benchmark_schema.py b/inferencex-e2e/infx/workflows/benchmark_schema.py index e1952347ff..38ccb662ee 100644 --- a/inferencex-e2e/infx/workflows/benchmark_schema.py +++ b/inferencex-e2e/infx/workflows/benchmark_schema.py @@ -58,6 +58,11 @@ def _validate_rows( multinode: bool | None = None, agentic: bool | None = None, ) -> None: + """Validate each row in rows against its workflow schema. + + multinode and agentic force the schema choice when set, otherwise each row decides. Raises + ValueError located by path. + """ if not isinstance(rows, list): raise ValueError(f"{path}: expected a list of matrix rows") for index, row in enumerate(rows): @@ -106,6 +111,10 @@ def validate_matrix(matrix: object, *, plan: bool = False) -> None: def main() -> None: + """Validate the matrix JSON on stdin and echo it unchanged to stdout. + + Exits with a usage error if the matrix is invalid. + """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--plan", diff --git a/inferencex-e2e/infx/workflows/calc_success_rate.py b/inferencex-e2e/infx/workflows/calc_success_rate.py index 1a3f415e95..8efbadfdab 100644 --- a/inferencex-e2e/infx/workflows/calc_success_rate.py +++ b/inferencex-e2e/infx/workflows/calc_success_rate.py @@ -16,6 +16,7 @@ def load_hardware_labels() -> list[str]: def build_hardware_match_patterns( hardware_labels: Iterable[str], ) -> dict[str, tuple[re.Pattern[str], ...]]: + """Map each hardware label to whole-word regexes for it and its cluster label.""" return { hardware: tuple( re.compile(rf"(? str | None: + """Return the first hardware label whose patterns match job_name, or None if nothing matches. + + Builds patterns from runners.yaml when match_patterns is omitted. + """ job_lower = job_name.lower() if match_patterns is None: match_patterns = build_hardware_match_patterns(load_hardware_labels()) @@ -39,6 +44,7 @@ def extract_hardware_from_name( def calculate_hardware_success_rates() -> dict[str, dict[str, int]]: + """Count successful and total non-skipped jobs per hardware in the current Actions run.""" hardware_labels = load_hardware_labels() patterns = build_hardware_match_patterns(hardware_labels) run_id = int(os.environ["GITHUB_RUN_ID"]) @@ -82,6 +88,7 @@ def print_success_rates(success_rates: dict[str, dict[str, int]] | None) -> None def main() -> None: + """Print per-hardware success rates for this run and write them to .json.""" run_stats = calculate_hardware_success_rates() print_success_rates(run_stats) diff --git a/inferencex-e2e/infx/workflows/ci_priority.py b/inferencex-e2e/infx/workflows/ci_priority.py index fa588e397b..8f3b5a7220 100755 --- a/inferencex-e2e/infx/workflows/ci_priority.py +++ b/inferencex-e2e/infx/workflows/ci_priority.py @@ -17,6 +17,21 @@ from infx.config import repository_root SCORE_QUANTUM = Decimal("0.001") +POLICY_KEYS = frozenset({"version", "base-score", "adjustments", "patchwork-score"}) +ADJUSTMENT_KEYS = frozenset( + { + "additional-node", + "event", + "multi-node", + "agentic", + "eval-only", + "checklist-complete", + "precision", + "spec-decoding", + "framework-prefix", + "model-prefix", + } +) @dataclass(frozen=True) @@ -27,18 +42,32 @@ class PriorityContext: def _decimal(value: Any) -> Decimal: + """Convert value to an exact Decimal via its string form.""" return Decimal(str(value)) def load_policy(path: str | Path) -> dict[str, Any]: + """Load the priority policy YAML at path. + + Raises ValueError on an unsupported version or unknown keys. + """ with Path(path).open() as policy_file: policy = yaml.safe_load(policy_file) if policy.get("version") != 1: raise ValueError(f"Unsupported CI priority policy version: {policy.get('version')}") + unknown = sorted(set(policy) - POLICY_KEYS) + sorted( + f"adjustments.{key}" for key in set(policy.get("adjustments") or {}) - ADJUSTMENT_KEYS + ) + if unknown: + raise ValueError(f"Unknown CI priority policy keys: {', '.join(unknown)}") return policy def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal: + """Return the adjustment for the first prefix matching value, or 0 if none matches. + + A prefix matches when value equals it or starts with it followed by a hyphen. + """ for prefix, adjustment in adjustments.items(): if value == prefix or value.startswith(f"{prefix}-"): return _decimal(adjustment) @@ -68,6 +97,10 @@ def _entry_from_criteria( policy: dict[str, Any], entry: dict[str, Any], ) -> dict[str, Any]: + """Project entry onto only the fields named in criteria, for priority grouping. + + Raises ValueError if criteria contains names the policy does not support. + """ unknown = criteria - supported_criteria(policy) if unknown: raise ValueError(f"Unknown CI priority criteria: {sorted(unknown)}") @@ -155,10 +188,12 @@ def calculate_priority( def format_priority(score: Decimal) -> str: + """Format score with three decimals.""" return f"{score:.3f}" def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: + """Return a 32-hex-character hash of value, namespace and path for queue grouping.""" canonical = json.dumps(value, sort_keys=True, separators=(",", ":")) material = f"{namespace}:{'/'.join(path)}:{canonical}".encode() return hashlib.sha256(material).hexdigest()[:32] @@ -191,6 +226,10 @@ def annotate_jobs( def _criteria_from_json(raw_criteria: str) -> frozenset[str] | None: + """Parse --criteria-json into a set of names, or None when empty or null. + + Raises ValueError unless it is a JSON array of strings. + """ if not raw_criteria: return None value = json.loads(raw_criteria) @@ -202,6 +241,7 @@ def _criteria_from_json(raw_criteria: str) -> frozenset[str] | None: def main() -> int: + """Annotate the matrix JSON from --input or stdin with priorities, print it and return 0.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--policy", default=str(repository_root() / "configs/ci-priority.yaml")) parser.add_argument("--event-name", default="") diff --git a/inferencex-e2e/infx/workflows/merge_with_reuse.py b/inferencex-e2e/infx/workflows/merge_with_reuse.py index d3514976f6..aaa22cf4b2 100644 --- a/inferencex-e2e/infx/workflows/merge_with_reuse.py +++ b/inferencex-e2e/infx/workflows/merge_with_reuse.py @@ -30,24 +30,50 @@ import sys import time from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from infx.config import git_repository_root, project_root +from .prepare_perf_changelog_merge import ( + canonicalize_appended_links, + resolve_conflict_bytes, +) +from .sweep_runs import ( + artifact_names, + completed_pr_runs, + has_reusable_result_artifacts, + pr_commit_shas, +) +from .validate_perf_changelog import read_git_file + try: import git as gitpython + import requests - from github import Github, GithubException + from github import Auth, Github, GithubException, RateLimitExceededException + + _NETWORK_ERRORS: tuple[type[Exception], ...] = ( + ConnectionError, + TimeoutError, + requests.ConnectionError, + requests.Timeout, + requests.exceptions.RetryError, + ) except ImportError: gitpython = None # type: ignore[assignment] Github = None # type: ignore[assignment, misc] + Auth = None # type: ignore[assignment] + _NETWORK_ERRORS = (ConnectionError, TimeoutError) class GithubException(Exception): # type: ignore[no-redef] # noqa: N818 """Stub when PyGithub is not installed.""" status = 0 data: object = None - headers: dict = {} # type: ignore[type-arg] # noqa: RUF012 + headers: ClassVar[dict] = {} # type: ignore[type-arg] + + class RateLimitExceededException(GithubException): # type: ignore[no-redef] + """Stub when PyGithub is not installed.""" _WORKFLOWS_AVAILABLE = gitpython is not None @@ -56,17 +82,6 @@ class GithubException(Exception): # type: ignore[no-redef] # noqa: N818 from github.PullRequest import PullRequest from github.Repository import Repository -from .prepare_perf_changelog_merge import ( # noqa: E402 - canonicalize_appended_links, - resolve_conflict_bytes, -) -from .sweep_runs import ( # noqa: E402 - artifact_names, - completed_pr_runs, - has_reusable_result_artifacts, - pr_commit_shas, -) -from .validate_perf_changelog import read_git_file # noqa: E402 DEFAULT_REPO = "SemiAnalysisAI/InferenceX" DEFAULT_CHECK_TIMEOUT = 900 @@ -104,14 +119,17 @@ def changelog_path() -> str: def log(msg: str) -> None: + """Print msg as a progress line.""" print(f"{_CYAN_BOLD}→{_RESET} {msg}") def ok(msg: str) -> None: + """Print msg as a success line.""" print(f"{_GREEN_BOLD}✓{_RESET} {msg}") def die(msg: str) -> int: + """Print msg as an error on stderr and return exit code 1.""" print(f"{_RED_BOLD}✗{_RESET} {msg}", file=sys.stderr) return 1 @@ -140,21 +158,25 @@ def _resolve_token() -> str: def _make_github(token: str) -> Github: """Build a PyGithub ``Github`` client, respecting ``GITHUB_API_URL``.""" base_url = os.environ.get("GITHUB_API_URL") or "https://api.github.com" - return Github(login_or_token=token, base_url=base_url) + return Github(auth=Auth.Token(token), base_url=base_url) -_TRANSIENT_HTTP_STATUSES = frozenset({403, 429, 500, 502, 503, 504}) +_TRANSIENT_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504}) def _is_transient_error(exc: Exception) -> bool: """Return True for API errors safe to retry during check polling. - Covers server errors (5xx), rate limits (403/429), and connection-level - failures from requests/urllib3. + Covers server errors (5xx), rate limits (429, and 403 rate-limit + responses), and connection-level failures from requests/urllib3. + PyGithub reports network failures as requests exceptions and raises + RetryError once its own retries run out, so both count as transient. """ + if isinstance(exc, RateLimitExceededException): + return True if isinstance(exc, GithubException): return exc.status in _TRANSIENT_HTTP_STATUSES - return isinstance(exc, (ConnectionError, TimeoutError)) + return isinstance(exc, _NETWORK_ERRORS) def _retry_delay(exc: Exception) -> float: @@ -180,17 +202,21 @@ class GitOps: """ def __init__(self, repo: gitpython.Repo | None = None) -> None: + """Wrap repo, or the repository containing the current directory.""" self.repo = repo or gitpython.Repo(".", search_parent_directories=True) def is_clean(self) -> bool: + """Return True if the working tree has no changes or untracked files.""" return not self.repo.is_dirty(untracked_files=True) def current_ref(self) -> str: + """Return the current branch name, or the commit SHA when HEAD is detached.""" if self.repo.head.is_detached: return self.repo.head.commit.hexsha return self.repo.active_branch.name def rev_parse(self, ref: str = "HEAD") -> str: + """Return the full commit SHA that ref resolves to.""" return self.repo.rev_parse(ref).hexsha def diff_name_only_unmerged(self) -> str: @@ -221,9 +247,11 @@ def show_stage(self, stage: int, path: str) -> bytes: ) def fetch(self, *args: str) -> None: + """Run git fetch with args.""" self.repo.git.fetch(*args) def checkout(self, *args: str) -> None: + """Run git checkout with args.""" self.repo.git.checkout(*args) def merge(self, *args: str) -> int: @@ -235,19 +263,24 @@ def merge(self, *args: str) -> int: return exc.status or 1 def add(self, *paths: str) -> None: + """Stage paths with git add.""" self.repo.git.add(*paths) def commit(self, *args: str) -> None: + """Run git commit with args.""" self.repo.git.commit(*args) def push(self, *args: str) -> None: + """Run git push with args.""" self.repo.git.push(*args) def branch_delete(self, name: str) -> None: + """Force-delete local branch name, ignoring git errors.""" with contextlib.suppress(gitpython.GitCommandError): self.repo.git.branch("-D", name) def merge_abort(self) -> None: + """Abort an in-progress merge, ignoring git errors.""" with contextlib.suppress(gitpython.GitCommandError): self.repo.git.merge("--abort") @@ -279,27 +312,39 @@ def find_eligible_run( return None -def _latest_check_runs(check_runs: list) -> list: - """Dedupe check runs by name, keeping the one with the latest started_at/id. +def _check_run_recency(cr: object) -> tuple: + """Order check runs oldest first. A queued run without started_at counts as newest.""" + started_at = getattr(cr, "started_at", None) + return (started_at is None, str(started_at or ""), getattr(cr, "id", 0)) + + +def _latest_check_runs(check_runs: list, suite_workflows: dict | None = None) -> list: + """Dedupe check runs, keeping the most recent per key. GitHub can return superseded re-runs for the same check name. This mirrors - ``gh pr checks --watch --fail-fast`` which only considers the latest run - per check name. + ``gh pr checks --watch --fail-fast``, which keys Actions check runs on + name, workflow and event, and other check runs on name alone. + *suite_workflows* maps a check suite id to its ``(workflow_id, event)``. """ - by_name: dict[str, object] = {} + suite_workflows = suite_workflows or {} + by_key: dict[tuple, object] = {} for cr in check_runs: - existing = by_name.get(cr.name) + key = (cr.name, suite_workflows.get(getattr(cr, "check_suite_id", None))) + existing = by_key.get(key) if existing is None: - by_name[cr.name] = cr + by_key[key] = cr continue - cr_key = (str(cr.started_at or ""), getattr(cr, "id", 0)) - ex_key = ( - str(getattr(existing, "started_at", "") or ""), - getattr(existing, "id", 0), - ) - if cr_key > ex_key: - by_name[cr.name] = cr - return list(by_name.values()) + if _check_run_recency(cr) > _check_run_recency(existing): + by_key[key] = cr + return list(by_key.values()) + + +def _suite_workflows(gh_repo: Repository, sha: str) -> dict: + """Map each Actions check suite on *sha* to its ``(workflow_id, event)``.""" + return { + run.check_suite_id: (run.workflow_id, run.event) + for run in gh_repo.get_workflow_runs(head_sha=sha) + } def _latest_statuses(statuses: list) -> list: @@ -344,8 +389,9 @@ def wait_for_checks( is treated as pending (the check needs re-evaluation). ``skipped`` and ``neutral`` count as passing. - Check runs are deduped by name (latest ``started_at``/``id`` wins) and - commit statuses by context to ignore superseded re-runs. + Check runs are deduped by name, workflow and event (latest + ``started_at``/``id`` wins) and commit statuses by context to ignore + superseded re-runs. Transient API errors (5xx, rate limits, connection failures) are retried with backoff until the timeout expires. @@ -359,7 +405,9 @@ def wait_for_checks( while time.monotonic() < deadline: try: - check_runs = _latest_check_runs(list(commit.get_check_runs())) + check_runs = _latest_check_runs( + list(commit.get_check_runs()), _suite_workflows(gh_repo, sha) + ) all_completed = True for cr in check_runs: @@ -413,6 +461,8 @@ def wait_for_check( Returns 0 on success, 1 on failure or timeout. Transient API errors are retried with backoff until the timeout expires. + Skipped runs are ignored when another exists, because unrelated label + events re-run the workflow with this job skipped on the same SHA. """ log(f"Waiting for {check_name} on {sha[:8]}") deadline = time.monotonic() + timeout @@ -422,8 +472,9 @@ def wait_for_check( try: check_runs = list(commit.get_check_runs()) matching = [cr for cr in check_runs if cr.name == check_name] + matching = [cr for cr in matching if cr.conclusion != "skipped"] or matching if matching: - matching.sort(key=lambda cr: str(cr.started_at or "")) + matching.sort(key=_check_run_recency) latest = matching[-1] if latest.status == "completed": @@ -494,6 +545,7 @@ class _MergeState: __slots__ = ("local_branch",) def __init__(self) -> None: + """Start with no local branch recorded.""" self.local_branch = "" @@ -525,6 +577,7 @@ def merge_pr( state = _MergeState() def cleanup() -> None: + """Abort any merge, restore the original branch and delete the temporary local branch.""" git_ops.merge_abort() with contextlib.suppress(Exception): git_ops.checkout("--quiet", original_branch) @@ -564,6 +617,10 @@ def _merge_pr_inner( git_ops: GitOps, state: _MergeState, ) -> int: + """Merge PR number pr once its reusable sweep, changelog and checks are in place. + + Returns 0 on success, non-zero on failure. Records the local branch in state for cleanup. + """ gh_repo = gh.get_repo(repo) pull = gh_repo.get_pull(pr) @@ -715,6 +772,7 @@ def _poll_pr_head( def main() -> int: + """Merge the PR named on the command line using env settings and return the exit code.""" if not _WORKFLOWS_AVAILABLE: print(_INSTALL_HINT, file=sys.stderr) return 1 diff --git a/inferencex-e2e/infx/workflows/pareto_coverage.py b/inferencex-e2e/infx/workflows/pareto_coverage.py index 5e94eb0715..ae9aa7d4d3 100644 --- a/inferencex-e2e/infx/workflows/pareto_coverage.py +++ b/inferencex-e2e/infx/workflows/pareto_coverage.py @@ -17,10 +17,15 @@ def _positive_finite(value: Any) -> bool: + """Return True if value is a positive finite int or float.""" return type(value) in (int, float) and value > 0 and math.isfinite(value) def assess_curve(curve: dict[str, Any]) -> dict[str, Any]: + """Compute the Pareto frontier of one curve and return its coverage summary. + + Raises ValueError if the curve lacks a key or a points list. + """ key, points = curve.get("key"), curve.get("points") if not isinstance(key, str) or not key.strip() or not isinstance(points, list): raise ValueError("Each curve needs a nonempty key and a points array") @@ -59,6 +64,10 @@ def assess_curve(curve: dict[str, Any]) -> dict[str, Any]: def assess_coverage(curves: Any) -> list[dict[str, Any]]: + """Assess every curve in curves and return their summaries. + + Raises ValueError on empty input, non-object curves or duplicate keys. + """ if not isinstance(curves, list) or not curves: raise ValueError("Provide every affected curve; an empty list cannot prove coverage") results = [] @@ -75,6 +84,7 @@ def assess_coverage(curves: Any) -> list[dict[str, Any]]: def main() -> None: + """Read curves JSON from stdin and print the coverage report, exiting 1 on invalid input.""" try: results = assess_coverage(json.load(sys.stdin)) print(json.dumps(results, indent=2, allow_nan=False)) diff --git a/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py b/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py index 79ccdc71c0..a42ba71fa6 100644 --- a/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py +++ b/inferencex-e2e/infx/workflows/prepare_perf_changelog_merge.py @@ -152,7 +152,7 @@ def resolve_conflict_bytes( expected_link = f"https://github.com/{repo}/pull/{pr_number}" if additions: - main_signatures = {entry_signature(entry) for entry in main_entries} + main_signatures = {entry_signature(entry) for entry in main_entries[len(base_entries) :]} pr_spans = entry_spans(pr_raw, pr_entries) contribution_blocks: list[bytes] = [] for index, entry in enumerate( diff --git a/inferencex-e2e/infx/workflows/recover_failed_ingest.py b/inferencex-e2e/infx/workflows/recover_failed_ingest.py index d7f53c6ecb..6fbe7ab901 100644 --- a/inferencex-e2e/infx/workflows/recover_failed_ingest.py +++ b/inferencex-e2e/infx/workflows/recover_failed_ingest.py @@ -14,14 +14,13 @@ import yaml -from infx import github +from infx import github, yamlio from infx.config import git_path_at_ref, git_repository_root, project_root from infx.matrix.revision import PLANNER, Revision from .validate_perf_changelog import ( CANONICAL_PR_LINK, ChangelogValidationError, - UniqueKeyLoader, parse_changelog, read_git_file, validate_raw_change, @@ -473,7 +472,7 @@ def build_config( def validate_recovery_workflow(path: Path, pr_number: int) -> None: """Reject recovery workflows that could schedule benchmark compute.""" text = path.read_text() - data = yaml.load(text, Loader=UniqueKeyLoader) # noqa: S506 + data = yamlio.load(text, yamlio.UniqueKeyLoader) if not isinstance(data, dict): raise RecoveryError("recovery workflow root must be a mapping") diff --git a/inferencex-e2e/infx/workflows/require_launcher.py b/inferencex-e2e/infx/workflows/require_launcher.py index 6147512874..72ded041f0 100644 --- a/inferencex-e2e/infx/workflows/require_launcher.py +++ b/inferencex-e2e/infx/workflows/require_launcher.py @@ -15,6 +15,10 @@ def main(argv: list[str] | None = None) -> int: + """Check that the checkout in argv contains the Python launcher. + + Returns 0 if it does, else prints a CI error and returns 1. + """ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("checkout", type=Path, help="repository checkout of the measured revision") checkout = parser.parse_args(argv).checkout diff --git a/inferencex-e2e/infx/workflows/reuse.py b/inferencex-e2e/infx/workflows/reuse.py index 599933cb1a..8fe862aa4b 100644 --- a/inferencex-e2e/infx/workflows/reuse.py +++ b/inferencex-e2e/infx/workflows/reuse.py @@ -119,6 +119,7 @@ def find_reuse_authorization( command: str, allowed_author_associations: set[str], ) -> tuple[bool, int | None]: + """Return whether an allowed reuse command exists on the PR, and the run ID it pins if any.""" comment, pinned_run_id = find_reuse_request( repo, pr_number, @@ -250,6 +251,10 @@ def resolve_reusable_run( def main() -> int: + """Decide from CLI arguments whether the sweep can reuse a prior run. + + Writes the outputs and returns the exit code. Raises RuntimeError if no GitHub token is set. + """ parser = argparse.ArgumentParser() parser.add_argument("--repo", required=True) parser.add_argument("--commit-sha", required=True) diff --git a/inferencex-e2e/infx/workflows/reuse_comment.py b/inferencex-e2e/infx/workflows/reuse_comment.py index f0a7bab6b0..4fff9184e5 100644 --- a/inferencex-e2e/infx/workflows/reuse_comment.py +++ b/inferencex-e2e/infx/workflows/reuse_comment.py @@ -49,6 +49,10 @@ def warn_outside_pr( def acknowledge(repo: str, event: dict[str, Any], token: str) -> int: + """React to a reuse command comment on a PR and report whether it was accepted. + + Returns 0 when accepted or not applicable, 1 when the request is rejected. + """ if event.get("action") not in {"created", "edited"} or not event.get("issue", {}).get( "pull_request" ): @@ -122,6 +126,10 @@ def acknowledge(repo: str, event: dict[str, Any], token: str) -> int: def main() -> int: + """Acknowledge the comment from the GitHub Actions event and return the exit code. + + Raises RuntimeError if no GitHub token is set. + """ token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") if not token: raise RuntimeError("GH_TOKEN or GITHUB_TOKEN is required") diff --git a/inferencex-e2e/infx/workflows/signoff_publish.py b/inferencex-e2e/infx/workflows/signoff_publish.py index 00d6769469..0d74da42c3 100644 --- a/inferencex-e2e/infx/workflows/signoff_publish.py +++ b/inferencex-e2e/infx/workflows/signoff_publish.py @@ -41,6 +41,7 @@ def normalize_verdict(verdict: str) -> str: + """Return verdict with pass emojis in N/A rows replaced by the neutral marker.""" return NA_ROW_WITH_PASS_EMOJI.sub(r"\1➖\2", verdict) # noqa: RUF001 @@ -97,18 +98,25 @@ def verdict_errors(verdict: str) -> list[str]: def marker(signoff_key: str) -> str: + """Return the hidden HTML marker for signoff_key, raising ValueError if the key is invalid.""" if SIGNOFF_KEY.fullmatch(signoff_key) is None: raise ValueError("Invalid sign-off key") return f"" def _is_verdict_for(comment: dict[str, Any], signoff_key: str) -> bool: + """Return True if comment is a verifier bot's verdict for signoff_key.""" author = comment.get("user") or {} body = comment.get("body") or "" return author.get("login") in VERIFIER_AUTHORS and body.startswith(f"{marker(signoff_key)}\n") def verdict_body(verdict: str, head_sha: str, signoff_key: str) -> tuple[str, str]: + """Build the PR comment body and status for a verdict on head_sha. + + Returns (body, status) where status is success, warning or failure. Invalid verdicts become a + failure notice. + """ verdict = normalize_verdict(verdict) checks, errors = parse_verdict(verdict) if errors: @@ -143,6 +151,11 @@ def publish( *, verification_succeeded: bool, ) -> None: + """Post or update the PR verdict comment for signoff_key on head_sha. + + Uses the verdict file only when verification succeeded, and reposts if the old comment was + deleted. + """ verdict = "" if verification_succeeded and verdict_path.exists(): verdict = verdict_path.read_text(encoding="utf-8").strip() @@ -171,6 +184,7 @@ def publish( def main() -> None: + """Publish the sign-off verdict using settings from the environment.""" publish( os.environ["GITHUB_REPOSITORY"], os.environ["GH_TOKEN"], diff --git a/inferencex-e2e/infx/workflows/signoff_resolve.py b/inferencex-e2e/infx/workflows/signoff_resolve.py index 20d322c667..915db906eb 100644 --- a/inferencex-e2e/infx/workflows/signoff_resolve.py +++ b/inferencex-e2e/infx/workflows/signoff_resolve.py @@ -14,6 +14,7 @@ def _positive_id(value: Any) -> int: + """Return value as an int, raising ValueError unless it is a positive decimal ID.""" if isinstance(value, bool) or not re.fullmatch(r"[1-9][0-9]*", str(value)): raise ValueError("Expected a positive GitHub resource ID") return int(value) @@ -22,6 +23,11 @@ def _positive_id(value: Any) -> int: def _reference( event_name: str, event: dict[str, Any], repo: str, token: str ) -> tuple[int, str, str, str, str]: + """Locate the sign-off comment for an event and check it belongs to the PR. + + Returns (PR number, author, comment kind, sign-off key, gh command to fetch the body). Raises + ValueError on a mismatched or invalid reference. + """ if event_name == "workflow_dispatch": inputs = event["inputs"] match = re.fullmatch( @@ -83,6 +89,11 @@ def resolve( scoped_head_sha: str, token: str, ) -> dict[str, str]: + """Decide whether an event is a trusted sign-off request and return the workflow outputs. + + Returns {"proceed": "false"} for ignored events. Raises ValueError for bad input and + RuntimeError if the PR head moved. + """ if event_name not in { "issue_comment", "pull_request_review", @@ -131,6 +142,7 @@ def resolve( def main() -> None: + """Resolve the sign-off from the GitHub Actions environment and write the step outputs.""" event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) outputs = resolve( os.environ["GITHUB_REPOSITORY"], diff --git a/inferencex-e2e/infx/workflows/signoff_scope.py b/inferencex-e2e/infx/workflows/signoff_scope.py index d307294263..59005b2e08 100644 --- a/inferencex-e2e/infx/workflows/signoff_scope.py +++ b/inferencex-e2e/infx/workflows/signoff_scope.py @@ -15,6 +15,11 @@ def required_owners(repo: str, pr: dict[str, Any], token: str) -> list[str]: + """Return the base CODEOWNERS owners of the PR's changed files who must sign off. + + Skips the core team and admin users. Raises RuntimeError if files or CODEOWNERS cannot be read + reliably. + """ files = github.paginate(repo, f"/pulls/{pr['number']}/files", token, "") if len({file["filename"] for file in files}) != pr["changed_files"]: raise RuntimeError("Incomplete changed-file list; cannot determine sign-off scope") @@ -52,6 +57,10 @@ def required_owners(repo: str, pr: dict[str, Any], token: str) -> list[str]: def check_scope(repo: str, number: int, token: str) -> dict[str, str]: + """Return workflow outputs saying whether PR number needs a sign-off. + + Raises RuntimeError if the PR changed during the check. + """ pr = github.api(repo, f"/pulls/{number}", token) required = required_owners(repo, pr, token) current = github.api(repo, f"/pulls/{number}", token) @@ -75,6 +84,10 @@ def check_scope(repo: str, number: int, token: str) -> dict[str, str]: def main() -> None: + """Find the PR from the GitHub event, check its sign-off scope and write the step outputs. + + Raises ValueError if a dispatch pr-number does not match comment_url. + """ event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) number = (event.get("pull_request") or event.get("issue") or {}).get("number") if number is None: diff --git a/inferencex-e2e/infx/workflows/signoff_validate.py b/inferencex-e2e/infx/workflows/signoff_validate.py index 37be3c5813..e9c54d568f 100644 --- a/inferencex-e2e/infx/workflows/signoff_validate.py +++ b/inferencex-e2e/infx/workflows/signoff_validate.py @@ -17,6 +17,10 @@ def main(argv: list[str] | None = None) -> int: + """Validate the verdict file named in argv and print the result. + + Returns 0 if valid, 1 if invalid, 2 on usage or read errors. + """ args = sys.argv[1:] if argv is None else argv if len(args) != 1: print("usage: python -m infx.workflows.signoff_validate ", file=sys.stderr) diff --git a/inferencex-e2e/infx/workflows/stage_results.py b/inferencex-e2e/infx/workflows/stage_results.py index b8c99e3287..680343fdd1 100644 --- a/inferencex-e2e/infx/workflows/stage_results.py +++ b/inferencex-e2e/infx/workflows/stage_results.py @@ -23,6 +23,10 @@ def timestamp(value: str | None) -> float: + """Return the POSIX time of an ISO-8601 value, or infinity if it is invalid. + + Naive times are treated as UTC. + """ try: parsed = datetime.fromisoformat(value or "") return (parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)).timestamp() @@ -31,6 +35,7 @@ def timestamp(value: str | None) -> float: def full_sweep_at(events: list[dict[str, Any]], created_at: str) -> bool: + """Return True if a full-sweep label was on the PR at created_at, replaying label events.""" created = timestamp(created_at) if created == float("inf"): return False @@ -50,10 +55,15 @@ def full_sweep_at(events: list[dict[str, Any]], created_at: str) -> bool: def resolve_run( repo: str, pr_number: int, pull: dict[str, Any], token: str, requested_id: str | None ) -> dict[str, Any]: + """Return the requested_id run, or else the first eligible completed sweep run on the PR. + + Raises RuntimeError naming why no run qualifies. + """ timeline = github.paginate(repo, f"/issues/{pr_number}/timeline", token) commits = sweep_runs.pr_commit_shas(repo, pr_number, token) def rejection(run: dict[str, Any]) -> str: + """Return why run cannot be staged, or an empty string if it can.""" if ( run.get("path") != ".github/workflows/run-sweep.yml" or run.get("event") != "pull_request" @@ -98,10 +108,16 @@ def use_command(body: str) -> re.Match[str] | None: def request(repo: str, event: dict[str, Any], token: str) -> dict[str, str] | None: + """Validate a /stage-results comment event and return the staging outputs. + + Returns None if the comment is not a staging request. Raises RuntimeError after posting a reply + when the request is rejected. + """ pr_number = event["issue"]["number"] actor = event["comment"]["user"]["login"] def reject(reason: str, comment: str) -> NoReturn: + """Post comment on the PR and raise RuntimeError with reason.""" github.api( repo, f"/issues/{pr_number}/comments", token, method="POST", data={"body": comment} ) @@ -159,6 +175,7 @@ def reject(reason: str, comment: str) -> NoReturn: def main() -> None: + """Handle the comment event from the environment and append staging outputs to GITHUB_OUTPUT.""" event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) outputs = request(os.environ["GITHUB_REPOSITORY"], event, os.environ["GH_TOKEN"]) if outputs is None: diff --git a/inferencex-e2e/infx/workflows/stage_results_callback.py b/inferencex-e2e/infx/workflows/stage_results_callback.py index 91ce241f1b..9e4ce8210f 100644 --- a/inferencex-e2e/infx/workflows/stage_results_callback.py +++ b/inferencex-e2e/infx/workflows/stage_results_callback.py @@ -16,6 +16,10 @@ def chart_links(payload: dict[str, Any], run_id: str, run_date: str) -> list[str]: + """Return markdown chart links from payload, or a default link for run_id on run_date. + + Raises RuntimeError if any link is malformed or not on the staging site. + """ links = payload.get("chart-links") if links is None: return [f"{STAGING_SITE}/inference?i_dates={quote(f'{run_date}~r{run_id}')}"] @@ -47,6 +51,10 @@ def chart_links(payload: dict[str, Any], run_id: str, run_date: str) -> list[str def report(repo: str, event: dict[str, Any], token: str, server_url: str) -> None: + """Post or update the PR comment for a staging callback event. + + Raises RuntimeError if the payload is malformed or from an unexpected source. + """ payload = event["client_payload"] if payload.get("source-repository") != "SemiAnalysisAI/InferenceX-app": raise RuntimeError("Unsupported staging callback source") @@ -103,6 +111,7 @@ def report(repo: str, event: dict[str, Any], token: str, server_url: str) -> Non def main() -> None: + """Report the staging callback from the GitHub Actions environment.""" event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text(encoding="utf-8")) report( os.environ["GITHUB_REPOSITORY"], diff --git a/inferencex-e2e/infx/workflows/sweep_runs.py b/inferencex-e2e/infx/workflows/sweep_runs.py index 1c916e41ea..d0d5c61f38 100644 --- a/inferencex-e2e/infx/workflows/sweep_runs.py +++ b/inferencex-e2e/infx/workflows/sweep_runs.py @@ -16,6 +16,7 @@ def completed_pr_runs( repo: str, workflow_id: str, head_branch: str, token: str ) -> list[dict[str, Any]]: + """Return every completed pull_request run of workflow_id on head_branch.""" workflow = urllib.parse.quote(workflow_id, safe="") return github.paginate( repo, diff --git a/inferencex-e2e/infx/workflows/validate_perf_changelog.py b/inferencex-e2e/infx/workflows/validate_perf_changelog.py index 799296efd5..643ae4e0e6 100644 --- a/inferencex-e2e/infx/workflows/validate_perf_changelog.py +++ b/inferencex-e2e/infx/workflows/validate_perf_changelog.py @@ -12,9 +12,8 @@ from typing import Any import yaml -from yaml.constructor import ConstructorError -from yaml.resolver import BaseResolver +from infx import yamlio from infx.config import git_path_at_ref from infx.matrix.validation import ChangelogEntry @@ -29,45 +28,6 @@ class ChangelogValidationError(ValueError): """Raised when the changelog or its git diff violates repository rules.""" -class UniqueKeyLoader(yaml.SafeLoader): - """Safe YAML loader that rejects duplicate mapping keys.""" - - -def construct_unique_mapping( - loader: UniqueKeyLoader, - node: yaml.MappingNode, - deep: bool = False, -) -> dict[Any, Any]: - """Construct a mapping while rejecting keys that would overwrite values.""" - mapping: dict[Any, Any] = {} - for key_node, value_node in node.value: - key = loader.construct_object(key_node, deep=deep) - try: - duplicate = key in mapping - except TypeError as exc: - raise ConstructorError( - "while constructing a mapping", - node.start_mark, - f"found unhashable key {key!r}", - key_node.start_mark, - ) from exc - if duplicate: - raise ConstructorError( - "while constructing a mapping", - node.start_mark, - f"found duplicate key {key!r}", - key_node.start_mark, - ) - mapping[key] = loader.construct_object(value_node, deep=deep) - return mapping - - -UniqueKeyLoader.add_constructor( - BaseResolver.DEFAULT_MAPPING_TAG, - construct_unique_mapping, -) - - def read_git_file(ref: str, path: str) -> bytes: """Read a repository file exactly as stored at a git ref.""" try: @@ -86,7 +46,11 @@ def read_git_file(ref: str, path: str) -> bytes: def parse_changelog(raw: bytes, label: str) -> list[dict[str, Any]]: - """Validate file-level invariants and return raw YAML entry mappings.""" + """Validate file-level invariants and return raw YAML entry mappings. + + Rejects NEL, LS and PS because YAML breaks lines there while the + byte-level append checks only see newlines. + """ if not raw.endswith(b"\n"): raise ChangelogValidationError(f"{label} does not end with a newline") if b"\r" in raw: @@ -100,9 +64,11 @@ def parse_changelog(raw: bytes, label: str) -> list[dict[str, Any]]: text = raw.decode("utf-8") except UnicodeDecodeError as exc: raise ChangelogValidationError(f"{label} is not UTF-8: {exc}") from exc + if any(line_break in text for line_break in "\x85\u2028\u2029"): + raise ChangelogValidationError(f"{label} contains a line break other than newline") try: - data = yaml.load(text, Loader=UniqueKeyLoader) # noqa: S506 + data = yamlio.load(text, yamlio.UniqueKeyLoader) except yaml.YAMLError as exc: raise ChangelogValidationError(f"{label} is not valid YAML: {exc}") from exc @@ -230,7 +196,7 @@ def validate_raw_change( raise ChangelogValidationError( "appended changelog entries must have exactly one empty separator line" ) - if head_raw.endswith(b"\n\n"): + if re.search(rb"\n *\n\Z", head_raw): raise ChangelogValidationError( "the final appended changelog entry must end with one newline" ) diff --git a/inferencex-e2e/infx/yamlio.py b/inferencex-e2e/infx/yamlio.py new file mode 100644 index 0000000000..bd8997aae8 --- /dev/null +++ b/inferencex-e2e/infx/yamlio.py @@ -0,0 +1,146 @@ +"""PyYAML loading that parses each repeated text once per process.""" + +from __future__ import annotations + +import functools +from collections.abc import Iterator +from typing import IO, Any + +import yaml +from yaml.constructor import ConstructorError +from yaml.resolver import BaseResolver + +from infx import fastcopy + +_MERGE_TAG = "tag:yaml.org,2002:merge" +_VALUE_TAG = "tag:yaml.org,2002:value" +_CACHED_TEXT_CHARACTERS = 128 * 1024 + + +@functools.lru_cache(maxsize=512) +def _parse(text: str, loader: type) -> Any: + """Return ``yaml.load(text, Loader=loader)``, parsed once per text and loader.""" + return yaml.load(text, Loader=loader) # noqa: S506 + + +def load(stream: str | IO[str], loader: type = yaml.SafeLoader) -> Any: + """Return what ``yaml.load(stream, Loader=loader)`` returns or raises. + + A ``str`` of at most 128 KiB is parsed once per process and loader, and every call gets a + fresh deep copy. Streams and larger texts are parsed on each call. Constructors added to a + loader after it parsed a text do not apply to that cached text. + """ + if type(stream) is not str or len(stream) > _CACHED_TEXT_CHARACTERS: + return yaml.load(stream, Loader=loader) # noqa: S506 + value = _parse(stream, loader) + try: + return fastcopy.deepcopy(value) + except RecursionError: + return yaml.load(stream, Loader=loader) # noqa: S506 + + +def safe_load(stream: str | IO[str]) -> Any: + """Return what ``yaml.safe_load(stream)`` returns, using the same cache as ``load``.""" + return load(stream, yaml.SafeLoader) + + +class UniqueKeyLoader(yaml.SafeLoader): + """Safe YAML loader that rejects duplicate mapping keys.""" + + +def construct_unique_mapping( + loader: UniqueKeyLoader, + node: yaml.Node, + deep: bool = False, +) -> dict[Any, Any]: + """Construct a mapping while rejecting keys that would overwrite values.""" + if not isinstance(node, yaml.MappingNode): + raise ConstructorError( + None, None, f"expected a mapping node, but found {node.id}", node.start_mark + ) + mapping: dict[Any, Any] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError as exc: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found unhashable key {key!r}", + key_node.start_mark, + ) from exc + if duplicate: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +class UniqueKeySafeLoader(yaml.SafeLoader): + """SafeLoader, merge keys included, that rejects a key repeated within one mapping.""" + + +def _reject_repeated_keys(loader: UniqueKeySafeLoader, node: yaml.MappingNode) -> None: + """Check ``node``'s own keys, and those of the mappings it merges, before PyYAML's + ``flatten_mapping`` rewrites any of them in place. Checked nodes stay referenced on the + loader so a later document cannot reuse their ids.""" + checked = loader.__dict__.setdefault("_infx_checked_mappings", {}) + if id(node) in checked: + return + checked[id(node)] = node + seen = set() + for key_node, value_node in node.value: + if key_node.tag == _MERGE_TAG: + sources = ( + value_node.value if isinstance(value_node, yaml.SequenceNode) else [value_node] + ) + for source in sources: + if isinstance(source, yaml.MappingNode): + _reject_repeated_keys(loader, source) + continue + if key_node.tag == _VALUE_TAG: + key = loader.construct_scalar(key_node) + else: + key = loader.construct_object(key_node) + try: + duplicate = key in seen + except TypeError: + continue + if duplicate: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key!r}", + key_node.start_mark, + ) + seen.add(key) + + +def construct_unique_safe_mapping( + loader: UniqueKeySafeLoader, node: yaml.Node +) -> Iterator[dict[Any, Any]]: + """``SafeConstructor.construct_yaml_map`` after checking the mapping's own keys.""" + data: dict[Any, Any] = {} + yield data + if isinstance(node, yaml.MappingNode): + _reject_repeated_keys(loader, node) + data.update(loader.construct_mapping(node)) + + +def construct_unique_safe_set(loader: UniqueKeySafeLoader, node: yaml.Node) -> Iterator[set[Any]]: + """``SafeConstructor.construct_yaml_set`` after checking the mapping's own keys.""" + data: set[Any] = set() + yield data + if isinstance(node, yaml.MappingNode): + _reject_repeated_keys(loader, node) + data.update(loader.construct_mapping(node)) + + +UniqueKeyLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_mapping) +UniqueKeySafeLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_unique_safe_mapping) +UniqueKeySafeLoader.add_constructor("tag:yaml.org,2002:set", construct_unique_safe_set) diff --git a/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md b/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md index 1a1f7d9205..73409e40c5 100644 --- a/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md +++ b/inferencex-e2e/utils/runner_setup/RUNNER_SETUP.md @@ -92,7 +92,7 @@ Required permissions (all of these endpoints require **admin access to the repos 4. Configure the runners: ```bash - ./InferenceX/utils/runner_setup/setup.sh \ + ./InferenceX/inferencex-e2e/utils/runner_setup/setup.sh \ \ \ \ @@ -104,7 +104,7 @@ Required permissions (all of these endpoints require **admin access to the repos For example, configure 18 runners (`b300-dsxe_00` … `b300-dsxe_17`) on the B300 DSXE cluster: ```bash - ./InferenceX/utils/runner_setup/setup.sh \ + ./InferenceX/inferencex-e2e/utils/runner_setup/setup.sh \ AOPHAHI... \ https://github.com/actions/runner/releases/download/v2.335.1/actions-runner-linux-x64-2.335.1.tar.gz \ 0 17 \ @@ -119,7 +119,7 @@ Required permissions (all of these endpoints require **admin access to the repos 5. Start the runners: ```bash - ./InferenceX/utils/runner_setup/start_runners.sh 0 17 ~/gharunners + ./InferenceX/inferencex-e2e/utils/runner_setup/start_runners.sh 0 17 ~/gharunners ``` This (re)creates a tmux session (default name: `github-actions`) with one tiled pane @@ -151,9 +151,7 @@ key off that name: runner names listed under each `labels` entry in [`configs/runners.yaml`](../../configs/runners.yaml). New runners do **not** receive sweep jobs until they are added there, and the entries must match the - registered names exactly, including zero-padding. Some older fleets predate the - padded convention, such as `gb200-nv_0`. Because `setup.sh` always zero-pads, new - entries should use the padded form. + registered names exactly, including zero-padding. `setup.sh` always zero-pads. ## Labels / `ADDITIONAL_RUNNER_TAGS`