From 7dc080e405b38d6fecbfa08329f4d70a78ddd0cc Mon Sep 17 00:00:00 2001 From: kcaisley Date: Wed, 19 Aug 2026 18:30:49 +0200 Subject: [PATCH 1/6] fix(ci): replace codecov with basic github coverage check --- .github/workflows/regression-tests.yml | 20 ++++++++++++++------ pyproject.toml | 1 - 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 006c1c08..cd75db58 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -42,6 +42,7 @@ jobs: - sim: icarus python-version: '3.13' + coverage: true - sim: verilator sim-version: 5.044 @@ -71,7 +72,13 @@ jobs: - name: Install Python dependencies shell: bash -l {0} run: | - pip install pyvisa pyvisa-sim pytest coverage coveralls pytest-cov cocotb>=2 cocotb-bus + pip install pyvisa pyvisa-sim pytest cocotb>=2 cocotb-bus + + - name: Install coverage dependency + if: matrix.coverage + shell: bash -l {0} + run: | + pip install pytest-cov - name: Install basil shell: bash -l {0} @@ -79,13 +86,14 @@ jobs: pip install -e . - name: Test + if: ${{ !matrix.coverage }} shell: bash -l {0} run: | - pytest ${{matrix.pytest-marker}} --cov=basil tests/test_*.py examples/*/*/test_*.py + pytest ${{matrix.pytest-marker}} tests/test_*.py examples/*/*/test_*.py - - name: Upload to codecov + - name: Test with coverage + if: matrix.coverage shell: bash -l {0} run: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} + pytest ${{matrix.pytest-marker}} --cov=basil tests/test_*.py examples/*/*/test_*.py + coverage report --format=total | awk '{print "### Test coverage: " $1 "%"}' >> "$GITHUB_STEP_SUMMARY" diff --git a/pyproject.toml b/pyproject.toml index d05e1e4a..11fcc191 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ hw = ["pyserial", "PyVISA"] test = [ "coverage", - "coveralls", "cocotb>=2.0.1", "cocotb-bus>=0.3.0", "cocotb-test>=0.2.6", From 50fa7b3965a413f43f76df2d0e72195c1bb055e8 Mon Sep 17 00:00:00 2001 From: kcaisley Date: Thu, 20 Aug 2026 14:42:51 +0200 Subject: [PATCH 2/6] feat(ci): add diff-cover library to analyze the coverage .xml file already existing --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 11fcc191..fe0af47e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ test = [ "cocotb>=2.0.1", "cocotb-bus>=0.3.0", "cocotb-test>=0.2.6", + "diff-cover", "pytest>=9.0.3", "pytest-cov", "PyVISA", From 404e10126dcb0276c29bc7abafcac208f7ca02ac Mon Sep 17 00:00:00 2001 From: kcaisley Date: Thu, 20 Aug 2026 14:43:01 +0200 Subject: [PATCH 3/6] feat(ci): add pre-commit local hook, and github actions remote hook to view diff-cover result --- .github/workflows/regression-tests.yml | 56 +++++++++++++++++++++++++- .gitignore | 4 ++ .pre-commit-config.yaml | 17 ++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index cd75db58..92a8d6d2 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -8,6 +8,9 @@ on: branches: - master +permissions: + contents: read + jobs: pre-commit: @@ -18,11 +21,16 @@ jobs: with: python-version: "3.12" - uses: pre-commit/action@v3.0.1 + env: + SKIP: diff-coverage # Run all normal formatting hooks in CI, but skip the local diff-coverage hook because the coverage job handles it separately. tests: name: Python ${{matrix.python-version}} | ${{matrix.sim}} runs-on: ubuntu-24.04 + permissions: + contents: read + checks: write env: SIM: ${{matrix.sim}} @@ -78,7 +86,7 @@ jobs: if: matrix.coverage shell: bash -l {0} run: | - pip install pytest-cov + pip install pytest-cov diff-cover - name: Install basil shell: bash -l {0} @@ -95,5 +103,49 @@ jobs: if: matrix.coverage shell: bash -l {0} run: | - pytest ${{matrix.pytest-marker}} --cov=basil tests/test_*.py examples/*/*/test_*.py + pytest ${{matrix.pytest-marker}} \ + --cov=basil \ + --cov-report=xml:coverage.xml \ + tests/test_*.py examples/*/*/test_*.py coverage report --format=total | awk '{print "### Test coverage: " $1 "%"}' >> "$GITHUB_STEP_SUMMARY" + + - name: Fetch pull request base + if: matrix.coverage && github.event_name == 'pull_request' + env: + BASE_REF: ${{github.base_ref}} + run: | + git fetch --no-tags --depth=1 origin \ + "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" + + - name: Create differential coverage report + if: matrix.coverage && github.event_name == 'pull_request' + env: + BASE_REF: ${{github.base_ref}} + shell: bash -l {0} + run: | + diff-cover coverage.xml \ + --compare-branch="origin/${BASE_REF}" \ + --format=markdown:diff-cover.md + cat diff-cover.md >> "$GITHUB_STEP_SUMMARY" + + - name: Publish differential coverage check + if: >- + matrix.coverage && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + env: + GH_TOKEN: ${{github.token}} + HEAD_SHA: ${{github.event.pull_request.head.sha}} + run: | + gh api --method POST \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "repos/${GITHUB_REPOSITORY}/check-runs" \ + --raw-field name="Coverage (informational)" \ + --raw-field head_sha="$HEAD_SHA" \ + --raw-field status=completed \ + --raw-field conclusion=neutral \ + --raw-field "output[title]=Differential coverage" \ + --field "output[summary]=@diff-cover.md" \ + --silent diff --git a/.gitignore b/.gitignore index 68bb5bfb..dfbb7b14 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ Makefile *.out examples/lx9/device/src/SiTCP +# Test coverage +.coverage +coverage.xml + # PyDev files .project .pydevproject diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32c39dfc..3d719f3d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,20 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace + - repo: local + hooks: + - id: diff-coverage + name: Differential test coverage + entry: >- + bash -c 'SIM=icarus pytest + --cov=basil + --cov-report=xml:coverage.xml + tests/test_*.py examples/*/*/test_*.py + && diff-cover coverage.xml + --compare-branch=origin/master + --show-uncovered' + language: system + pass_filenames: false + verbose: true + files: ^(basil/|tests/|examples/).*\.py$ + stages: [pre-commit] From 90567dae9e6cc8e8a316c1a98d7dc29ed36d69be Mon Sep 17 00:00:00 2001 From: kcaisley Date: Thu, 20 Aug 2026 14:46:59 +0200 Subject: [PATCH 4/6] fix(ci): remove previously installed pytest-cov since it's coverage points out is unnecessary in simple cases Read online here: [https://coverage.readthedocs.io/en/7.15.4/#quick-start](https://coverage.readthedocs.io/en/7.15.4/#quick-start) --- .github/workflows/regression-tests.yml | 7 +++---- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 92a8d6d2..118fa982 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -86,7 +86,7 @@ jobs: if: matrix.coverage shell: bash -l {0} run: | - pip install pytest-cov diff-cover + pip install coverage diff-cover - name: Install basil shell: bash -l {0} @@ -103,10 +103,9 @@ jobs: if: matrix.coverage shell: bash -l {0} run: | - pytest ${{matrix.pytest-marker}} \ - --cov=basil \ - --cov-report=xml:coverage.xml \ + coverage run --source=basil -m pytest ${{matrix.pytest-marker}} \ tests/test_*.py examples/*/*/test_*.py + coverage xml -o coverage.xml coverage report --format=total | awk '{print "### Test coverage: " $1 "%"}' >> "$GITHUB_STEP_SUMMARY" - name: Fetch pull request base diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d719f3d..61b9c87a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,10 +20,10 @@ repos: - id: diff-coverage name: Differential test coverage entry: >- - bash -c 'SIM=icarus pytest - --cov=basil - --cov-report=xml:coverage.xml + bash -c 'SIM=icarus coverage run + --source=basil -m pytest tests/test_*.py examples/*/*/test_*.py + && coverage xml -o coverage.xml && diff-cover coverage.xml --compare-branch=origin/master --show-uncovered' diff --git a/pyproject.toml b/pyproject.toml index fe0af47e..d531dd86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,6 @@ test = [ "cocotb-test>=0.2.6", "diff-cover", "pytest>=9.0.3", - "pytest-cov", "PyVISA", "pyvisa-sim", ] From 4df14f1da23aa0c9815adfb931e41f057f57e395 Mon Sep 17 00:00:00 2001 From: kcaisley Date: Thu, 20 Aug 2026 15:08:44 +0200 Subject: [PATCH 5/6] Update pre-commit hook name --- .github/workflows/regression-tests.yml | 2 +- .pre-commit-config.yaml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 118fa982..0289c7d7 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -22,7 +22,7 @@ jobs: python-version: "3.12" - uses: pre-commit/action@v3.0.1 env: - SKIP: diff-coverage # Run all normal formatting hooks in CI, but skip the local diff-coverage hook because the coverage job handles it separately. + SKIP: coverage # Run all normal formatting hooks in CI, but skip the local coverage hook because the coverage job handles it separately. tests: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61b9c87a..1f81974f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,13 +17,14 @@ repos: - id: trailing-whitespace - repo: local hooks: - - id: diff-coverage - name: Differential test coverage + - id: coverage + name: Coverage entry: >- bash -c 'SIM=icarus coverage run --source=basil -m pytest + -q --disable-warnings tests/test_*.py examples/*/*/test_*.py - && coverage xml -o coverage.xml + && coverage xml -q -o coverage.xml && diff-cover coverage.xml --compare-branch=origin/master --show-uncovered' From cf3e5969640594978b9b19e2e18a983eb29bfcc1 Mon Sep 17 00:00:00 2001 From: kcaisley Date: Thu, 20 Aug 2026 15:18:26 +0200 Subject: [PATCH 6/6] fix(ci): fetch history for differential coverage --- .github/workflows/regression-tests.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 0289c7d7..f1b3f403 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -59,6 +59,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + fetch-depth: ${{ matrix.coverage && '0' || '1' }} - name: Set up miniforge ${{matrix.python-version}} uses: conda-incubator/setup-miniconda@v3 with: @@ -108,14 +110,6 @@ jobs: coverage xml -o coverage.xml coverage report --format=total | awk '{print "### Test coverage: " $1 "%"}' >> "$GITHUB_STEP_SUMMARY" - - name: Fetch pull request base - if: matrix.coverage && github.event_name == 'pull_request' - env: - BASE_REF: ${{github.base_ref}} - run: | - git fetch --no-tags --depth=1 origin \ - "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" - - name: Create differential coverage report if: matrix.coverage && github.event_name == 'pull_request' env: