From 432bfb94055c1f9b4b98ccd439c5f089605ad913 Mon Sep 17 00:00:00 2001 From: Aviad Rozenhek Date: Tue, 10 Nov 2026 01:45:44 +0200 Subject: [PATCH 1/2] ci: skip unaffected jobs and add CI-esta reports --- .github/labels.yml | 3 + .github/test-lanes.json | 40 ++++ .github/workflows/benchmark.yml | 33 ++- .github/workflows/observe-ci.yml | 36 +++ .github/workflows/release.yml | 32 ++- .github/workflows/test.yml | 70 +++++- .gitignore | 3 + ci/tests/test_lane_routing.py | 370 +++++++++++++++++++++++++++++++ requirements-mypy.txt | 2 + 9 files changed, 583 insertions(+), 6 deletions(-) create mode 100644 .github/test-lanes.json create mode 100644 .github/workflows/observe-ci.yml create mode 100644 ci/tests/test_lane_routing.py diff --git a/.github/labels.yml b/.github/labels.yml index 7534b02a..2d186c53 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -72,3 +72,6 @@ - color: fbca04 description: "Unit tests, linting, CI, etc." name: testing +- color: fbca04 + description: "Run every CI lane on this PR (applies from the next push or reopen)" + name: "test:full" diff --git a/.github/test-lanes.json b/.github/test-lanes.json new file mode 100644 index 00000000..8cba8548 --- /dev/null +++ b/.github/test-lanes.json @@ -0,0 +1,40 @@ +{ + "graph": { + "module_roots": ["src", "."], + "test_roots": ["tests", "ci/tests"], + "global_inputs": ["pyproject.toml", "tox.ini"], + "pytest_policy": { + "pyproject.toml": "8c6a5fd2043787c01e67a463fc6bb2494764f55c2d2c5d358ebe9f46dad0829c", + "tox.ini": "751ee0be550f8a9de9dea8d6f74b0178cc17c099cc18086106bb9b397016076c" + } + }, + "global_inputs": ["pyproject.toml", "tox.ini", ".github/test-lanes.json"], + "lanes": { + "tests": { + "roots": ["tests"], + "inputs": ["src/**", "scripts/**", ".github/workflows/test.yml#test"] + }, + "benchmark": { + "inputs": [ + "tests/test_benchmarks.py", + "src/**", + "scripts/generate-translation-binaries.sh", + ".github/workflows/benchmark.yml#benchmarks" + ] + }, + "package": { + "inputs": [ + "src/**", + "README.md", + "LICENCE", + ".gitignore", + "scripts/generate-translation-binaries.sh", + ".github/workflows/release.yml#build-package" + ] + }, + "routing": { + "roots": ["ci/tests"], + "inputs": [".github/workflows/test.yml#routing"] + } + } +} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index daab4531..7b97f906 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,6 +1,11 @@ name: Benchmark -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: + schedule: + - cron: "17 4 * * 1" permissions: {} @@ -9,7 +14,27 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: 1 jobs: + select-tests: + runs-on: ubuntu-latest + outputs: + lanes: ${{ steps.plan.outputs.lanes }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - id: plan + uses: gigaverse-app/pytest-selective/actions/plan-lanes@286d0aeb75991d35f2ffafc7dda0517b7250b5c3 # v1.4.0 + with: + config: .github/test-lanes.json + benchmarks: + needs: select-tests + # Preserve full manual/scheduled checkpoints; ordinary changes use the plan. + # A failed planner never skips the benchmark lane. + if: >- + ${{ always() && !cancelled() && (needs.select-tests.result != 'success' || + fromJSON(needs.select-tests.outputs.lanes).benchmark) }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -33,7 +58,13 @@ jobs: - name: Install dependencies run: pip install -e ".[tests]" + # Forks cannot upload to the upstream CodSpeed integration. + - name: Run benchmark tests on forks + if: github.event.repository.fork == true + run: pytest tests/test_benchmarks.py --benchmark-disable + - name: Run benchmarks + if: github.event.repository.fork == false uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0 with: mode: simulation diff --git a/.github/workflows/observe-ci.yml b/.github/workflows/observe-ci.yml new file mode 100644 index 00000000..fc206f9f --- /dev/null +++ b/.github/workflows/observe-ci.yml @@ -0,0 +1,36 @@ +name: CI report +on: # zizmor: ignore[dangerous-triggers] -- read-only reporter; never executes source PR code + workflow_run: + workflows: [Test] + types: [completed] + workflow_dispatch: + inputs: + run-id: + description: Completed test workflow run to report + required: true + type: string + head-sha: + description: Exact head SHA of the source run + required: true + type: string + run-attempt: + description: Source run attempt + type: string + default: "1" +permissions: + actions: read + checks: read + contents: read + pull-requests: read +jobs: + report: + # The trusted reporter reads source workflow data without executing PR code. + uses: gigaverse-app/ci-esta/.github/workflows/report.yml@b69484796b301a90c73bd7c7f26f8e542c8bb8a3 # v1.14.0 + with: + run-id: ${{ github.event.workflow_run.id || fromJSON(inputs.run-id) }} + run-attempt: + ${{ github.event.workflow_run.run_attempt || fromJSON(inputs.run-attempt) }} + head-sha: ${{ github.event.workflow_run.head_sha || inputs.head-sha }} + required-gates-json: '[{"kind":"actions_job","job_name":"Test successful"}]' + analyze-critical-path: true + post-sticky: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14b97229..973e6f73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,9 @@ env: FORCE_COLOR: 1 jobs: - # Always build & lint package. + # Build & lint the package on every push, tag, release and dispatch. + # On pull requests, skip the build when the package lane's inputs are unchanged. + # The plan runs inside this job, so the publishing jobs keep their single dependency. build-package: name: Build & verify package runs-on: ubuntu-latest @@ -28,15 +30,41 @@ jobs: fetch-depth: 0 persist-credentials: false + - id: plan + if: github.event_name == 'pull_request' + # A failed planner builds the package; only a successful plan can skip it. + continue-on-error: true + uses: gigaverse-app/pytest-selective/actions/plan-lanes@286d0aeb75991d35f2ffafc7dda0517b7250b5c3 # v1.4.0 + with: + config: .github/test-lanes.json + + - id: admit + name: Package lane admission + env: + EVENT_NAME: ${{ github.event_name }} + PLAN_OUTCOME: ${{ steps.plan.outcome }} + LANES: ${{ steps.plan.outputs.lanes }} + run: | + package_lane="$(python3 -c 'import json, os; print(json.loads(os.environ["LANES"] or "{}").get("package"))')" + if [ "$EVENT_NAME" = pull_request ] && [ "$PLAN_OUTCOME" = success ] && [ "$package_lane" = False ]; then + echo "build=false" >> "$GITHUB_OUTPUT" + echo "Package lane unaffected; skipping the build." + else + echo "build=true" >> "$GITHUB_OUTPUT" + fi + - name: Install gettext + if: steps.admit.outputs.build == 'true' run: | sudo apt install gettext - name: Generate translation binaries + if: steps.admit.outputs.build == 'true' run: | scripts/generate-translation-binaries.sh - - uses: hynek/build-and-inspect-python-package@67230f7b8410ae0052b39c1efc89d89fff43ff10 # v3.0.2 + - if: steps.admit.outputs.build == 'true' + uses: hynek/build-and-inspect-python-package@67230f7b8410ae0052b39c1efc89d89fff43ff10 # v3.0.2 # Upload to Test PyPI on every commit on main. release-test-pypi: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 96321c4b..c01b9db4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,12 @@ name: Test -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: + schedule: + # Weekly full checkpoint: scheduled runs execute every lane. + - cron: "17 4 * * 1" permissions: {} @@ -9,7 +15,26 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: 1 jobs: + select-tests: + runs-on: ubuntu-latest + outputs: + lanes: ${{ steps.plan.outputs.lanes }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - id: plan + uses: gigaverse-app/pytest-selective/actions/plan-lanes@286d0aeb75991d35f2ffafc7dda0517b7250b5c3 # v1.4.0 + with: + config: .github/test-lanes.json + test: + needs: select-tests + # A failed planner runs the matrix; only a successful plan can skip it. + if: >- + ${{ always() && !cancelled() && (needs.select-tests.result != 'success' || + fromJSON(needs.select-tests.outputs.lanes).tests) }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -69,13 +94,52 @@ jobs: flags: ${{ matrix.os }} name: ${{ matrix.os }} Python ${{ matrix.python-version }} + routing: + needs: select-tests + if: >- + ${{ always() && !cancelled() && (needs.select-tests.result != 'success' || + fromJSON(needs.select-tests.outputs.lanes).routing) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + - name: Lane routing regressions + run: | + uv run --no-project --python 3.14 --with "pytest>=9" --with "pytest-selective==1.4.0" \ + python -m pytest ci/tests + success: if: always() - needs: test + needs: [select-tests, test, routing] runs-on: ubuntu-latest name: Test successful steps: - name: Success env: + PLAN_RESULT: ${{ needs.select-tests.result }} + LANES: ${{ needs.select-tests.outputs.lanes }} TEST_RESULT: ${{ needs.test.result }} - run: test "$TEST_RESULT" = "success" + ROUTING_RESULT: ${{ needs.routing.result }} + # Accept a skipped lane only when a successful plan explicitly returned false for it. + run: | + python3 - <<'PY' + import json, os, sys + + if os.environ["PLAN_RESULT"] != "success": + sys.exit(f"Lane planner result: {os.environ['PLAN_RESULT']}") + lanes = json.loads(os.environ["LANES"]) + # Every required lane must have an explicit boolean decision. + if not isinstance(lanes, dict) or any(type(lanes.get(lane)) is not bool for lane in ("tests", "routing")): + sys.exit("Invalid lane plan") + failed = False + for lane, result in (("tests", os.environ["TEST_RESULT"]), ("routing", os.environ["ROUTING_RESULT"])): + accepted = result == "success" or (result == "skipped" and lanes.get(lane) is False) + print(f"{lane}: lane={lanes.get(lane)} result={result} accepted={accepted}") + failed |= not accepted + sys.exit(1 if failed else 0) + PY diff --git a/.gitignore b/.gitignore index e6f08499..7b91f022 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ dmypy.json # hatch-vcs src/*/_version.py + +# pytest-selective graph cache +.test-impact/ diff --git a/ci/tests/test_lane_routing.py b/ci/tests/test_lane_routing.py new file mode 100644 index 00000000..69c7303b --- /dev/null +++ b/ci/tests/test_lane_routing.py @@ -0,0 +1,370 @@ +"""Route real Git changes through this repository's CI lane configuration.""" + +# These are the inputs whose behaviour these tests pin; the lane planner reads them +# to select the routing lane when they change. +# gv:test-dependency: .github/test-lanes.json +# gv:test-dependency: .github/workflows/test.yml +# gv:test-dependency: .github/workflows/benchmark.yml +# gv:test-dependency: .github/workflows/release.yml +# gv:test-dependency: requirements-mypy.txt + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import textwrap +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +import yaml +from pytest_selective.ci import LaneConfig, plan + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = LaneConfig.model_validate_json( + (ROOT / ".github/test-lanes.json").read_text(encoding="utf-8") +) +NONE = dict.fromkeys(CONFIG.lanes, False) +EVERY = dict.fromkeys(CONFIG.lanes, True) +PLAN_LANES = "gigaverse-app/pytest-selective/actions/plan-lanes@" +PLAN_LANES_SHA = "286d0aeb75991d35f2ffafc7dda0517b7250b5c3" +LANE_IF = ( + "${{{{ always() && !cancelled() && (needs.select-tests.result != 'success' || " + "fromJSON(needs.select-tests.outputs.lanes).{lane}) }}}}" +) + +Mutation = Callable[[Path], None] + + +def git(repo: Path, *args: str) -> str: + """Run Git in ``repo`` with a fixed identity and return its output.""" + return subprocess.run( + [ + "git", + "-c", + "user.name=Lane routing", + "-c", + "user.email=routing@example.invalid", + "-c", + "commit.gpgsign=false", + *args, + ], + cwd=repo, + check=True, + capture_output=True, + text=True, + ).stdout + + +def lanes(*names: str) -> dict[str, bool]: + """The expected plan: exactly ``names`` run.""" + assert set(names) <= set(CONFIG.lanes), names + return {name: name in names for name in CONFIG.lanes} + + +def append(path: str) -> Mutation: + """Append a line to ``path``, creating it when absent.""" + + def mutate(repo: Path) -> None: + target = repo / path + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("a", encoding="utf-8", newline="\n") as stream: + stream.write("\n# routing probe\n") + + return mutate + + +def delete(path: str) -> Mutation: + """Remove ``path`` from the head commit.""" + + def mutate(repo: Path) -> None: + git(repo, "rm", "-q", path) + + return mutate + + +def edit_job(workflow: str, job: str) -> Mutation: + """Change one job of ``workflow`` semantically, leaving every other job intact.""" + + def mutate(repo: Path) -> None: + target = repo / ".github/workflows" / workflow + text = target.read_text(encoding="utf-8") + header = f"\n {job}:\n" + assert text.count(header) == 1, (workflow, job) + target.write_text( + text.replace(header, f"{header} timeout-minutes: 77\n"), + encoding="utf-8", + newline="\n", + ) + + return mutate + + +def edit_workflow_settings(workflow: str) -> Mutation: + """Change a setting outside ``jobs:`` that every job of ``workflow`` inherits.""" + + def mutate(repo: Path) -> None: + target = repo / ".github/workflows" / workflow + text = target.read_text(encoding="utf-8") + assert text.count("\njobs:\n") == 1, workflow + target.write_text( + text.replace("\njobs:\n", "\nconcurrency: routing-probe\n\njobs:\n"), + encoding="utf-8", + newline="\n", + ) + + return mutate + + +@pytest.fixture(scope="module") +def template(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A Git repository whose single commit holds this checkout's files.""" + repo = tmp_path_factory.mktemp("template") + git(repo, "init", "-q", "-b", "main") + listed = git(ROOT, "ls-files", "-z", "--cached", "--others", "--exclude-standard") + for name in filter(None, listed.split("\0")): + source = ROOT / name + if source.is_file(): + target = repo / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "base") + return repo + + +def change(template: Path, tmp_path: Path, *mutations: Mutation) -> tuple[Path, str]: + """Clone the template, commit ``mutations`` and return the clone and its base.""" + repo = tmp_path / "repo" + git(tmp_path, "clone", "-q", str(template), str(repo)) + base = git(repo, "rev-parse", "HEAD").strip() + for mutate in mutations: + mutate(repo) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "change") + return repo, base + + +DOCUMENTATION = [ + "docs/time.md", + "docs/requirements.txt", + "docs/css/code_select.css", + "mkdocs.yml", + ".readthedocs.yml", + "RELEASING.md", + ".github/CONTRIBUTING.md", + ".github/PULL_REQUEST_TEMPLATE.md", +] + + +@pytest.mark.parametrize( + "mutation, expected", + [ + # Prose, the docs site and its own requirements run no lane. + *[(append(path), NONE) for path in DOCUMENTATION], + # Development metadata validated by the always-run Lint and Mypy jobs. + # Mypy checks these tests with the pinned planner, so its pins run them. + (append("requirements-mypy.txt"), lanes("routing")), + (append(".pre-commit-config.yaml"), NONE), + (append(".github/labels.yml"), NONE), + (append(".github/renovate.json"), NONE), + (edit_job("lint.yml", "lint"), NONE), + (edit_job("docs.yml", "build"), NONE), + # README.md and LICENCE are embedded in the built distributions. + (append("README.md"), lanes("package")), + (append("LICENCE"), lanes("package")), + (append(".gitignore"), lanes("package")), + # Library code, catalogues and the scripts the jobs and tests execute. + (append("src/humanize/number.py"), lanes("tests", "benchmark", "package")), + (append("src/humanize/py.typed"), lanes("tests", "benchmark", "package")), + ( + append("src/humanize/locale/de_DE/LC_MESSAGES/humanize.po"), + lanes("tests", "benchmark", "package"), + ), + ( + append("scripts/generate-translation-binaries.sh"), + lanes("tests", "benchmark", "package"), + ), + # test_i18n.py runs this script in a subprocess. + (append("scripts/update-translations.sh"), lanes("tests")), + # Tests: edited, added and deleted. + (append("tests/test_time.py"), lanes("tests")), + (append("tests/test_new.py"), lanes("tests")), + (delete("tests/test_lists.py"), lanes("tests")), + (append("tests/test_benchmarks.py"), lanes("tests", "benchmark")), + (append("ci/tests/test_lane_routing.py"), lanes("routing")), + # Environment, collection policy and the lane configuration run every lane. + (append("pyproject.toml"), EVERY), + (append("tox.ini"), EVERY), + (append(".github/test-lanes.json"), EVERY), + # Execution workflows select the lanes of the jobs that changed. + (edit_job("test.yml", "test"), lanes("tests", "routing")), + (edit_job("test.yml", "select-tests"), lanes("routing")), + (edit_job("test.yml", "routing"), lanes("routing")), + (edit_job("test.yml", "success"), lanes("routing")), + (edit_workflow_settings("test.yml"), lanes("tests", "routing")), + (edit_job("benchmark.yml", "benchmarks"), lanes("benchmark", "routing")), + (edit_job("benchmark.yml", "select-tests"), lanes("routing")), + (edit_job("release.yml", "build-package"), lanes("package", "routing")), + (edit_job("release.yml", "release-pypi"), lanes("routing")), + ], +) +def test_change_selects_exactly_its_lanes( + template: Path, tmp_path: Path, mutation: Mutation, expected: dict[str, bool] +) -> None: + """Each change runs exactly the lanes whose inputs it touches.""" + repo, base = change(template, tmp_path, mutation) + result = plan(repo, CONFIG, base, "HEAD") + assert result.lanes == expected, result.lane_reasons + + +def test_documentation_only_commit_runs_no_lane(template: Path, tmp_path: Path) -> None: + """One commit touching every documentation input still runs nothing.""" + repo, base = change(template, tmp_path, *map(append, DOCUMENTATION)) + assert plan(repo, CONFIG, base, "HEAD").lanes == NONE + + +def test_full_checkpoint_and_missing_history_run_every_lane( + template: Path, tmp_path: Path +) -> None: + """Explicit full runs and an unknown base never skip a lane.""" + repo, base = change(template, tmp_path, append("docs/time.md")) + assert plan(repo, CONFIG, base, "HEAD", full=True).lanes == EVERY + missing = "0" * 40 + assert plan(repo, CONFIG, missing, "HEAD").lanes == EVERY + + +def workflow(name: str) -> dict[str, Any]: + """The parsed workflow ``name``.""" + document = yaml.safe_load( + (ROOT / ".github/workflows" / name).read_text(encoding="utf-8") + ) + assert isinstance(document, dict) + return document + + +def squash(text: str) -> str: + """``text`` with whitespace runs collapsed, for comparing folded expressions.""" + return " ".join(text.split()) + + +def assert_pinned_planner(job: dict[str, Any]) -> None: + """The job uses the published planner major and repository configuration.""" + step = next( + step for step in job["steps"] if step.get("uses", "").startswith(PLAN_LANES) + ) + assert step["with"]["config"] == ".github/test-lanes.json" + assert step["uses"] == PLAN_LANES + PLAN_LANES_SHA + + +def test_every_lane_gates_a_job() -> None: + """Each lane is consumed by the job that executes it, behind the planner.""" + test_jobs = workflow("test.yml")["jobs"] + for lane in ("tests", "routing"): + job = test_jobs[lane if lane == "routing" else "test"] + assert job["needs"] == "select-tests" + assert squash(job["if"]) == squash(LANE_IF.format(lane=lane)) + benchmark = workflow("benchmark.yml")["jobs"]["benchmarks"] + assert benchmark["needs"] == "select-tests" + assert squash(benchmark["if"]) == squash( + "${{ always() && !cancelled() && (needs.select-tests.result != 'success' || " + "fromJSON(needs.select-tests.outputs.lanes).benchmark) }}" + ) + build = workflow("release.yml")["jobs"]["build-package"] + assert "needs" not in build, "publishing jobs must keep their single dependency" + steps = build["steps"] + plan_step = next(step for step in steps if step.get("id") == "plan") + assert plan_step["if"] == "github.event_name == 'pull_request'" + assert plan_step["continue-on-error"] is True + admitted = steps[steps.index(plan_step) + 2 :] + assert admitted + for step in admitted: + assert step["if"] == "steps.admit.outputs.build == 'true'", step + for jobs in (test_jobs, workflow("benchmark.yml")["jobs"]): + planner = jobs["select-tests"] + checkout = planner["steps"][0] + assert checkout["with"]["fetch-depth"] == 0 + assert_pinned_planner(planner) + assert_pinned_planner(build) + # The full checkpoint that dependency selection relies on. + # PyYAML reads the bare ``on`` key as the boolean True. + triggers = yaml.safe_load( + (ROOT / ".github/workflows/test.yml").read_text(encoding="utf-8") + )[True] + assert triggers["schedule"] + + +def test_routing_job_installs_the_planner_release() -> None: + """Routing, Mypy and the planner use the measured pytest-selective release.""" + for name in ("test.yml", "benchmark.yml", "release.yml"): + text = (ROOT / ".github/workflows" / name).read_text(encoding="utf-8") + assert PLAN_LANES + PLAN_LANES_SHA in text + run = workflow("test.yml")["jobs"]["routing"]["steps"][-1]["run"] + assert '"pytest-selective==1.4.0"' in run + mypy = (ROOT / "requirements-mypy.txt").read_text(encoding="utf-8") + assert "pytest-selective==1.4.0" in mypy + + +def heredoc(run: str) -> str: + """The Python program embedded in a ``python3 - <<'PY'`` step.""" + body = run.split("<<'PY'\n", 1)[1].rsplit("PY", 1)[0] + return textwrap.dedent(body) + + +@pytest.mark.parametrize( + "plan_result, lanes_json, test_result, routing_result, passes", + [ + ("success", '{"tests": false, "routing": false}', "skipped", "skipped", True), + ("success", '{"tests": true, "routing": false}', "success", "skipped", True), + ("success", '{"tests": true, "routing": true}', "success", "success", True), + # A skip is accepted only when the plan explicitly said the lane is unaffected. + ("success", '{"tests": true, "routing": false}', "skipped", "skipped", False), + ("success", '{"tests": false, "routing": true}', "skipped", "skipped", False), + ("success", '{"tests": true, "routing": false}', "failure", "skipped", False), + ("success", '{"tests": true, "routing": true}', "cancelled", "success", False), + # A failed planner runs every job, and still blocks the gate. + ("failure", "", "success", "success", False), + ("success", "{}", "success", "success", False), + ("success", '{"tests": "true", "routing": true}', "success", "success", False), + ], +) +def test_success_gate_accepts_only_planned_skips( + plan_result: str, + lanes_json: str, + test_result: str, + routing_result: str, + passes: bool, +) -> None: + """The required ``Test successful`` check stays meaningful on the empty route.""" + gate = workflow("test.yml")["jobs"]["success"] + assert gate["if"] == "always()" + assert gate["needs"] == ["select-tests", "test", "routing"] + step = gate["steps"][-1] + assert set(step["env"]) == {"PLAN_RESULT", "LANES", "TEST_RESULT", "ROUTING_RESULT"} + env = os.environ | { + "PLAN_RESULT": plan_result, + "LANES": lanes_json, + "TEST_RESULT": test_result, + "ROUTING_RESULT": routing_result, + } + completed = subprocess.run( + [sys.executable, "-c", heredoc(step["run"])], env=env, check=False + ) + assert (completed.returncode == 0) is passes + + +def test_fork_benchmarks_execute_without_publishing() -> None: + """Forks run benchmarks locally; only upstream publishes to CodSpeed.""" + steps = workflow("benchmark.yml")["jobs"]["benchmarks"]["steps"] + published = next( + step for step in steps if step.get("uses", "").startswith("CodSpeedHQ/action@") + ) + assert published["if"] == "github.event.repository.fork == false" + local = next( + step for step in steps if step.get("name") == "Run benchmark tests on forks" + ) + assert local["if"] == "github.event.repository.fork == true" + assert local["run"].strip() == "pytest tests/test_benchmarks.py --benchmark-disable" diff --git a/requirements-mypy.txt b/requirements-mypy.txt index aee860e8..9ff0c087 100644 --- a/requirements-mypy.txt +++ b/requirements-mypy.txt @@ -1,4 +1,6 @@ mypy==2.4.0 pytest +pytest-selective==1.4.0 types-freezegun +types-pyyaml types-setuptools From b6dbebbda791e61f1116c08fe6d82e108d2eb479 Mon Sep 17 00:00:00 2001 From: Aviad Rozenhek Date: Tue, 10 Nov 2026 01:50:31 +0200 Subject: [PATCH 2/2] style(ci): keep observer formatting stable across hooks --- .github/workflows/observe-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/observe-ci.yml b/.github/workflows/observe-ci.yml index fc206f9f..d34e7ed3 100644 --- a/.github/workflows/observe-ci.yml +++ b/.github/workflows/observe-ci.yml @@ -28,7 +28,7 @@ jobs: uses: gigaverse-app/ci-esta/.github/workflows/report.yml@b69484796b301a90c73bd7c7f26f8e542c8bb8a3 # v1.14.0 with: run-id: ${{ github.event.workflow_run.id || fromJSON(inputs.run-id) }} - run-attempt: + run-attempt: >- ${{ github.event.workflow_run.run_attempt || fromJSON(inputs.run-attempt) }} head-sha: ${{ github.event.workflow_run.head_sha || inputs.head-sha }} required-gates-json: '[{"kind":"actions_job","job_name":"Test successful"}]'