ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:

permissions:
  contents: read

jobs:
  python:
    name: Python lint · type · test · coverage
    runs-on: ubuntu-latest
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-python@v5
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: "3.12"
          cache: pip
      - name: Install
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"
      - name: Lint (ruff)
        run: ruff check tooltrace scripts tests
      - name: Format check (ruff format)
        run: ruff format --check tooltrace scripts tests
      # `scripts/` too, since a CI gate that only type-checks the package
      # let `Path.read_text(newline=...)` -- valid on 3.13, a TypeError on
      # the 3.12 this job runs -- reach the frontend job and fail there at
      # run time instead of here as a type error.
      - name: Type check (mypy)
        run: mypy tooltrace scripts
      - name: Test + coverage
        run: pytest --cov=tooltrace --cov-report=term-missing --cov-fail-under=80
      - name: Validate schemas + task packs
        run: |
          python -m json.tool schemas/task.schema.json > /dev/null
          python -m json.tool schemas/trace.schema.json > /dev/null
          python -m json.tool schemas/result.schema.json > /dev/null
          python -m json.tool schemas/bundle-manifest.schema.json > /dev/null
          tooltrace validate --path tooltrace/tasks/packs --json
      - name: Built wheel is self-contained
        # The wheel shipped without schemas/, so every task load in an installed
        # package raised "task.schema.json not found". A source-tree test cannot
        # see that: this builds a wheel and runs it away from the checkout.
        run: python scripts/wheel_check.py
      - name: Deterministic sample evaluation
        run: python scripts/make_sample_results.py
      - name: CLI smoke test
        run: python scripts/cli_smoke.py
      - name: Sandbox cleanup check
        run: python scripts/sandbox_check.py
      - name: Sandbox escape attempts (adversarial)
        # sandbox_check.py verifies a helper rejects bad paths. This attacks the
        # path an agent actually uses -- ToolExecutor -> tools -- and fails the
        # build if anything the sandbox claims to block gets through.
        run: python scripts/sandbox_escape_check.py
      - name: Secret scan (publication gate)
        run: python scripts/secret_scan.py
      - name: Pinned actions match their version comments
        # The CodeQL pin carried a `@v3` comment on the line above it while
        # the SHA was v4.37.9.
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: python scripts/check_action_pins.py
      - name: Documented uses of our own action resolve and install
        # docs/recipes.md said `@v0.3.0`, the one tag this project has ever cut,
        # which predates action.yml and so contains none. The recipe failed
        # before a task ran, and every test of action.yml passed anyway.
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: python scripts/check_action_refs.py
      - name: Landscape table matches the fetched competitor data
        # The table states its own method ("fetched via the GitHub API")
        # but was typed by hand. In a sibling repo that drifted into
        # publishing that a live competitor's repo was gone.
        run: python scripts/generate_landscape.py --check
      - name: No new unreachable public symbols
        run: python scripts/check_reachability.py --check
      - name: Docs cite modules that exist
        # docs/differentiators.md cited four modules that do not exist, including
        # one describing a capability that never has. The link check only
        # validates markdown links, not backticked paths.
        run: python scripts/check_doc_code_refs.py
      - name: Docs link check
        run: python scripts/check_docs_links.py
      - name: Generate static web data
        run: python scripts/generate_web_data.py
      - name: Generate SBOM (CycloneDX)
        run: |
          # `cyclonedx-py requirements` needs a requirements.txt, and this
          # project does not have one -- it declares dependencies in
          # pyproject. That invocation therefore failed on every run and the
          # `||` silently fell through to the generator, which at the time
          # dumped the entire interpreter. The generator now resolves the
          # declared dependency closure, so call it directly rather than
          # pretending a tool ran that never could.
          python scripts/generate_sbom.py
          python -c "import json,sys; b=json.load(open('sbom.json')); sys.exit(0 if b.get('serialNumber') and b['metadata'].get('timestamp') and b['components'] else 'SBOM missing serialNumber, timestamp or components')"
      # actions/upload-artifact@v7
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: sbom
          path: sbom.json
      # actions/upload-artifact@v7
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: results-and-webdata
          path: |
            results/
            web/public/data/

  frontend:
    name: Frontend lint · typecheck · test · build · e2e · a11y
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: web
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-node@v7
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
        with:
          node-version: 22
          cache: npm
          cache-dependency-path: web/package-lock.json
      - run: npm ci
      - run: npm run lint
      - run: npm run build
      - run: npm test
      # This repo gated pip-audit but nothing on the npm side, while
      # local-ai-hardware-bench did the reverse. Each had what the other
      # lacked; both surfaces are gated now.
      - name: npm audit (fail on high+)
        run: npm audit --audit-level=high
      - name: Generate validated static data for e2e
        working-directory: .
        run: |
          python -m pip install --upgrade pip
          pip install -e .
          python scripts/make_sample_results.py
          python scripts/generate_web_data.py
      - name: E2E + accessibility smoke (installed Chrome)
        run: npm run test:e2e
      # Gated on accessibility, best-practices and SEO; performance is
      # reported and not gated, because it is a timing measurement on a
      # shared runner and a gate that fails randomly gets switched off.
      - name: Lighthouse (a11y / best-practices / SEO >= 95)
        working-directory: .
        run: python scripts/lighthouse_check.py

  python-matrix:
    name: Tests (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [windows-latest, macos-latest]
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-python@v5
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: "3.12"
      - run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"
      - name: Test suite (cross-platform)
        run: pytest -q -m "not slow"

  # Deliberately a separate job rather than a python-version dimension on
  # `python-matrix`: adding one would rename its checks from
  # "Tests (windows-latest)" to "Tests (windows-latest, 3.11)", and branch
  # protection matches contexts byte-for-byte, so every pull request would
  # hang on a check that can never report. This repository has already been
  # in that state once.
  python-versions:
    name: Tests (python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # 3.12 is covered by the main `python` job; these are the other
        # versions pyproject's classifiers claim to support.
        python-version: ["3.11", "3.13", "3.14"]
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-python@v5
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: ${{ matrix.python-version }}
      - run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"
      - name: Test suite
        run: pytest -q -m "not slow"


  # ubuntu-latest runners have Docker; the script skips cleanly elsewhere.
  # Kept as its own job so a Docker outage cannot redden the main gate, and
  # deliberately NOT added to branch protection for the same reason.
  sandbox-docker:
    name: Sandbox conformance (Docker)
    runs-on: ubuntu-latest
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-python@v5
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: "3.12"
      - run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]"
      - name: Local sandbox conformance
        run: python scripts/sandbox_check.py
      - name: Docker sandbox conformance
        run: python scripts/docker_sandbox_check.py
      - name: Image builds and the installed wheel loads its packs
        # The image installs a *built wheel* into a clean container with no
        # repository beside it. That is the same check scripts/wheel_check.py
        # performs, enforced by the artifact people actually run -- and the
        # defect it guards against went unnoticed for three releases because
        # every install anyone tried was editable.
        run: |
          docker build -t tooltrace-bench:ci .
          docker run --rm tooltrace-bench:ci tasks --json > /dev/null
          docker run --rm tooltrace-bench:ci doctor --json > /dev/null

  # `tests/test_github_action_is_usable.py` parses action.yml and checks its
  # shape: inputs documented, outputs wired to a step, no third-party actions.
  # It cannot check that the thing runs, and nothing else did either -- the
  # composite body, the argument assembly and the threshold logic had never
  # executed in any job, in this repository or anywhere else, while the README
  # and docs told people to depend on it.
  action:
    name: The published action actually runs
    runs-on: ubuntu-latest
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-python@v5
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: "3.12"
      # install: false below, so this is the copy under test.
      - run: pip install -e .

      - name: Run it the way the docs tell people to
        id: bench
        uses: ./
        with:
          agent: scripted
          runs: "1"
          limit: "3"
          shuffle: "true"
          seed: "0"
          install: "false"

      - name: The outputs must be real values, not empty strings
        shell: bash
        env:
          RATE: ${{ steps.bench.outputs.success-rate }}
          TASKS: ${{ steps.bench.outputs.tasks-run }}
          SUBSET: ${{ steps.bench.outputs.is-subset }}
          SUMMARY: ${{ steps.bench.outputs.summary-path }}
        run: |
          set -euo pipefail
          echo "rate=$RATE tasks=$TASKS subset=$SUBSET summary=$SUMMARY"
          [ -n "$RATE" ]
          [ "$TASKS" = "3" ]
          [ "$SUBSET" = "true" ]
          [ -s "$SUMMARY" ]
          python -c "import json,sys; json.load(open(sys.argv[1]))" "$SUMMARY"

      - name: A floor the run cannot meet must fail the step
        id: floor
        continue-on-error: true
        uses: ./
        with:
          agent: scripted
          runs: "1"
          limit: "2"
          install: "false"
          # Above 1.0 on purpose: this asserts the gate fires, without
          # depending on some task failing to make it fire.
          min-success-rate: "1.1"

      - name: ...and that failure must be reported, not swallowed
        shell: bash
        env:
          OUTCOME: ${{ steps.floor.outcome }}
        run: |
          set -euo pipefail
          [ "$OUTCOME" = "failure" ] || { echo "the threshold did not fail the step"; exit 1; }

  audit:
    name: Dependency audit (blocking)
    runs-on: ubuntu-latest
    steps:
      # actions/checkout@v7
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
      # actions/setup-python@v5
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: "3.12"
      - run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]" pip-audit
      - name: Audit Python dependencies (gate)
        run: pip-audit --skip-editable