diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 006c1c08..f1b3f403 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -8,6 +8,9 @@ on: branches: - master +permissions: + contents: read + jobs: pre-commit: @@ -18,11 +21,16 @@ jobs: with: python-version: "3.12" - uses: pre-commit/action@v3.0.1 + env: + SKIP: coverage # Run all normal formatting hooks in CI, but skip the local coverage hook because the coverage job handles it separately. tests: name: Python ${{matrix.python-version}} | ${{matrix.sim}} runs-on: ubuntu-24.04 + permissions: + contents: read + checks: write env: SIM: ${{matrix.sim}} @@ -42,6 +50,7 @@ jobs: - sim: icarus python-version: '3.13' + coverage: true - sim: verilator sim-version: 5.044 @@ -50,6 +59,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + fetch-depth: ${{ matrix.coverage && '0' || '1' }} - name: Set up miniforge ${{matrix.python-version}} uses: conda-incubator/setup-miniconda@v3 with: @@ -71,7 +82,13 @@ jobs: - name: Install Python dependencies shell: bash -l {0} run: | - pip install pyvisa pyvisa-sim pytest coverage coveralls pytest-cov cocotb>=2 cocotb-bus + pip install pyvisa pyvisa-sim pytest cocotb>=2 cocotb-bus + + - name: Install coverage dependency + if: matrix.coverage + shell: bash -l {0} + run: | + pip install coverage diff-cover - name: Install basil shell: bash -l {0} @@ -79,13 +96,49 @@ jobs: pip install -e . - name: Test + if: ${{ !matrix.coverage }} shell: bash -l {0} run: | - pytest ${{matrix.pytest-marker}} --cov=basil tests/test_*.py examples/*/*/test_*.py + pytest ${{matrix.pytest-marker}} tests/test_*.py examples/*/*/test_*.py - - name: Upload to codecov + - name: Test with coverage + if: matrix.coverage + shell: bash -l {0} + run: | + coverage run --source=basil -m pytest ${{matrix.pytest-marker}} \ + tests/test_*.py examples/*/*/test_*.py + coverage xml -o coverage.xml + coverage report --format=total | awk '{print "### Test coverage: " $1 "%"}' >> "$GITHUB_STEP_SUMMARY" + + - name: Create differential coverage report + if: matrix.coverage && github.event_name == 'pull_request' + env: + BASE_REF: ${{github.base_ref}} shell: bash -l {0} run: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} + diff-cover coverage.xml \ + --compare-branch="origin/${BASE_REF}" \ + --format=markdown:diff-cover.md + cat diff-cover.md >> "$GITHUB_STEP_SUMMARY" + + - name: Publish differential coverage check + if: >- + matrix.coverage && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + env: + GH_TOKEN: ${{github.token}} + HEAD_SHA: ${{github.event.pull_request.head.sha}} + run: | + gh api --method POST \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "repos/${GITHUB_REPOSITORY}/check-runs" \ + --raw-field name="Coverage (informational)" \ + --raw-field head_sha="$HEAD_SHA" \ + --raw-field status=completed \ + --raw-field conclusion=neutral \ + --raw-field "output[title]=Differential coverage" \ + --field "output[summary]=@diff-cover.md" \ + --silent diff --git a/.gitignore b/.gitignore index 68bb5bfb..dfbb7b14 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ Makefile *.out examples/lx9/device/src/SiTCP +# Test coverage +.coverage +coverage.xml + # PyDev files .project .pydevproject diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32c39dfc..1f81974f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,21 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace + - repo: local + hooks: + - id: coverage + name: Coverage + entry: >- + bash -c 'SIM=icarus coverage run + --source=basil -m pytest + -q --disable-warnings + tests/test_*.py examples/*/*/test_*.py + && coverage xml -q -o coverage.xml + && diff-cover coverage.xml + --compare-branch=origin/master + --show-uncovered' + language: system + pass_filenames: false + verbose: true + files: ^(basil/|tests/|examples/).*\.py$ + stages: [pre-commit] diff --git a/pyproject.toml b/pyproject.toml index d05e1e4a..d531dd86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,12 +38,11 @@ hw = ["pyserial", "PyVISA"] test = [ "coverage", - "coveralls", "cocotb>=2.0.1", "cocotb-bus>=0.3.0", "cocotb-test>=0.2.6", + "diff-cover", "pytest>=9.0.3", - "pytest-cov", "PyVISA", "pyvisa-sim", ]