diff --git a/.github/actions/classify-priority/action.yml b/.github/actions/classify-priority/action.yml index 9d0bc0ccb7..4fee106105 100644 --- a/.github/actions/classify-priority/action.yml +++ b/.github/actions/classify-priority/action.yml @@ -16,11 +16,15 @@ inputs: description: "full returns every criterion; patchwork returns only the patchwork verdict" required: false default: full + pr-number: + description: Pull request whose description decides checklist-complete; empty never returns it + required: false + default: "" anthropic-api-key: description: Anthropic API key for the classifier required: true github-token: - description: GitHub token for claude-code-action + description: GitHub token for claude-code-action and reading the PR description required: true outputs: @@ -74,9 +78,24 @@ runs: uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml \ python -P -m infx.workflows.ci_priority \ --policy "$E2E_ROOT/configs/ci-priority.yaml" --list-criteria) - schema=$(jq -c '{type:"object",properties:{criteria:{type:"array",items:{type:"string",enum:.},uniqueItems:true},reason:{type:"string"}},required:["criteria","reason"]}' <<<"$criteria") + schema=$(jq -c 'map(select(. != "checklist-complete")) | {type:"object",properties:{criteria:{type:"array",items:{type:"string",enum:.},uniqueItems:true},reason:{type:"string"}},required:["criteria","reason"]}' <<<"$criteria") echo "json=$schema" >> "$GITHUB_OUTPUT" + - name: Check PR checklist + id: checklist + if: steps.range.outputs.head != '' && inputs.pr-number != '' + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + PR_NUMBER: ${{ inputs.pr-number }} + E2E_ROOT: ${{ github.action_path }}/../../../inferencex-e2e + run: | + complete=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.body // ""' | + env PYTHONPATH="$E2E_ROOT" uv run --no-project --exclude-newer PT12H --python 3.12 \ + python -P -m infx.workflows.pr_checklist) + echo "complete=$complete" >> "$GITHUB_OUTPUT" + - name: Install Claude Code 2.1.282 id: claude_cli if: steps.range.outputs.head != '' @@ -117,7 +136,6 @@ runs: - mtp, draft_model: speculative decoding method - sglang, vllm, dynamo-sglang, dynamo-vllm: runtime framework - model criterion: the config model-prefix itself (minimaxm31flash, not minimaxm3) - - checklist-complete: PR checklist is satisfied - patchwork: modified upstream engine or runtime source Patchwork includes patch files, git apply, source mutation, @@ -133,6 +151,7 @@ runs: env: OUTPUT: ${{ steps.classify.outputs.structured_output || '{}' }} SCOPE: ${{ inputs.scope }} + CHECKLIST_COMPLETE: ${{ steps.checklist.outputs.complete }} run: | if jq -e 'type == "object" and (.criteria | type == "array")' \ >/dev/null <<<"$OUTPUT"; then @@ -140,6 +159,10 @@ runs: else criteria='["patchwork"]' fi + criteria=$(jq -c 'map(select(. != "checklist-complete"))' <<<"$criteria") + if [ "$CHECKLIST_COMPLETE" = "true" ]; then + criteria=$(jq -c '. + ["checklist-complete"]' <<<"$criteria") + fi if [ "$SCOPE" = "patchwork" ]; then if jq -e 'index("patchwork")' >/dev/null <<<"$criteria"; then criteria='["patchwork"]' diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c2420dd183..b8aa3955c5 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -186,10 +186,14 @@ Labels authorize same-repository sweeps on draft or ready PRs. Marking ready does not dispatch a sweep. Changing a recognized primary or modifier label cancels the active sweep; another starts only if a primary label remains. To retrigger an already-labeled PR, remove and reapply its primary label. -Merge conflicts block sweeps. Queue priority comes from the PR diff and -review state, not priority labels. Trusted external sweeps classify the -approved PR diff the same way. Manual `e2e-tests.yml` and `profile.yml` -dispatches of a revision not yet on `main` classify its diff from the merge +Merge conflicts block sweeps. Queue priority comes from the PR diff and the +PR description's checklist, not priority labels. A PR gets a small bonus when +every item under its `Checklist` heading is checked, except conditional items +("If ...", "Before merging ...") that do not apply yet. The checklist is read +when the sweep starts, so later edits count from the next sweep. Trusted +external sweeps classify the approved PR diff the same way. Manual +`e2e-tests.yml` and `profile.yml` dispatches of a revision not yet on `main` +classify its diff from the merge base only for patchwork, so engine-patching branches still run last. Klaud runs and revisions already on `main` keep unclassified scoring. diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 4f237e82b1..ef22d0a78e 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -198,6 +198,9 @@ jobs: get-jobs: name: get-jobs runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read # Read the external PR description for checklist-complete. outputs: benchmark-ref: ${{ steps.checkout-ref.outputs.commit || steps.checkout-default.outputs.commit }} tooling-ref: ${{ steps.tooling.outputs.result }} @@ -260,6 +263,7 @@ jobs: base-ref: ${{ inputs.changelog-base-ref }} head-ref: ${{ inputs.changelog-head-ref || 'HEAD' }} scope: ${{ inputs.changelog-head-ref != '' && 'full' || 'patchwork' }} + pr-number: ${{ inputs.source-pr-number }} anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] github-token: ${{ github.token }} - id: get-jobs diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 9b0f07d1ff..82fa7c3d52 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -225,6 +225,7 @@ jobs: with: base-ref: origin/${{ github.base_ref }} head-ref: ${{ github.event.pull_request.head.sha }} + pr-number: ${{ github.event.pull_request.number }} anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} # zizmor: ignore[secrets-outside-env] github-token: ${{ github.token }} diff --git a/inferencex-e2e/infx/tests/workflows/test_pr_checklist.py b/inferencex-e2e/infx/tests/workflows/test_pr_checklist.py new file mode 100644 index 0000000000..f54a449afb --- /dev/null +++ b/inferencex-e2e/infx/tests/workflows/test_pr_checklist.py @@ -0,0 +1,44 @@ +import pytest + +from infx.workflows.pr_checklist import checklist_complete + +TEMPLATE = """## Summary + +- [ ] Unrelated task outside the checklist + +
+PR checklist and change type + +## Type of Change + +- [ ] Bug fix + +## Checklist + +- [{tested}] I have tested my changes locally +- [{changelog}] **If `python -m infx.matrix.changed` lists affected config keys, I have appended one entry** +- [ ] **Before merging via reuse, an authorized maintainer has commented `/use `** + +
+ +- [ ] Trailing task after the section +""" + + +@pytest.mark.parametrize( + ("tested", "changelog", "expected"), + [ + ("x", "x", True), + ("X", " ", True), + (" ", "x", False), + ], +) +def test_only_conditional_items_may_stay_unchecked(tested, changelog, expected): + body = TEMPLATE.format(tested=tested, changelog=changelog) + + assert checklist_complete(body) is expected + + +def test_missing_or_empty_checklist_is_incomplete(): + assert checklist_complete("## Summary\n\n- [x] Done\n") is False + assert checklist_complete("## Checklist\n\nNothing here.\n\n## Notes\n- [x] Done\n") is False diff --git a/inferencex-e2e/infx/workflows/pr_checklist.py b/inferencex-e2e/infx/workflows/pr_checklist.py new file mode 100644 index 0000000000..039e1342e3 --- /dev/null +++ b/inferencex-e2e/infx/workflows/pr_checklist.py @@ -0,0 +1,42 @@ +"""Decide whether a PR description's author checklist is complete. + +The checklist is the task list under the ``## Checklist`` heading of the PR template. +Every item must be checked except conditional ones ("If ...", "Before merging ..."), +which may stay unchecked until they apply. +""" + +from __future__ import annotations + +import re +import sys + +CHECKLIST_HEADING = re.compile(r"^#{1,6}\s+Checklist\s*$") +ANY_HEADING = re.compile(r"^#{1,6}\s") +SECTION_END = re.compile(r"^\s*") +TASK_ITEM = re.compile(r"^- \[([ xX])\]\s+(.*)$") +CONDITIONAL = re.compile(r"^(If|Before merging)\b") + + +def checklist_complete(body: str) -> bool: + """Return True when the Checklist section exists and every unconditional item is checked.""" + items: list[tuple[bool, str]] = [] + in_section = False + for line in body.splitlines(): + if CHECKLIST_HEADING.match(line): + in_section = True + continue + if in_section and (ANY_HEADING.match(line) or SECTION_END.match(line)): + break + if in_section and (match := TASK_ITEM.match(line)): + items.append((match.group(1) != " ", match.group(2).lstrip("*").strip())) + return bool(items) and all(checked or CONDITIONAL.match(text) for checked, text in items) + + +def main() -> int: + """Read a PR description from stdin and print true or false.""" + print("true" if checklist_complete(sys.stdin.read()) else "false") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())