Merge pull request #28 from Point72/tkp/speed

Speed up CI and update from template

Author: timkpaine

.copier-answers.yaml

@@ -1,8 +1,9 @@
 # Changes here will be overwritten by Copier
-_commit: 84508d6
+_commit: be75208
 _src_path: https://github.com/python-project-templates/base.git
 add_docs: false
 add_extension: rust
+add_pyodide: false
 add_wiki: true
 email: OpenSource@Point72.com
 github: Point72

.github/dependabot.yaml

@@ -4,6 +4,10 @@ updates:
     directory: "/"
     schedule:
       interval: "monthly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
     labels:
       - "part: github_actions"
     cooldown:
@@ -15,6 +19,10 @@ updates:
       interval: "monthly"
     cooldown:
       default-days: 7
+    groups:
+      python:
+        patterns:
+          - "*"
     labels:
       - "lang: python"
       - "part: dependencies"
@@ -25,7 +33,10 @@ updates:
       interval: "monthly"
     cooldown:
       default-days: 7
+    groups:
+      rust:
+        patterns:
+          - "*"
     labels:
       - "lang: rust"
       - "part: dependencies"
-

.github/workflows/build.yaml

@@ -23,6 +23,7 @@ permissions:
 
 jobs:
   build:
+    name: build (${{ matrix.name }})
     runs-on: ${{ matrix.os }}
     permissions:
       contents: read
@@ -31,7 +32,28 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - name: ubuntu-latest
+            dist_path: dist
+            os: ubuntu-latest
+            python: "3.11"
+            target: native
+          - name: ubuntu-24.04-arm
+            dist_path: dist
+            os: ubuntu-24.04-arm
+            python: "3.11"
+            target: native
+          - name: macos-latest
+            dist_path: dist
+            os: macos-latest
+            python: "3.11"
+            target: native
+          - name: windows-latest
+            dist_path: dist
+            os: windows-latest
+            python: "3.11"
+            target: native
+          
 
     steps:
       - name: Checkout
@@ -42,7 +64,7 @@ jobs:
       - name: Setup Python
         uses: actions-ext/python/setup@ec67f82b3bd863948308a41ecb9e70547cbb19c7
         with:
-          version: "3.11"
+          version: ${{ matrix.python }}
 
       - name: Setup Rust
         uses: actions-ext/rust/setup@0f0b7c9ab3cdb6e9a1f79e2147974d6762ddafea
@@ -57,54 +79,62 @@ jobs:
 
       - name: Install dependencies
         run: make develop
+        if: matrix.target == 'native'
 
       - name: Lint
         run: make lint
+        if: matrix.target == 'native'
 
       - name: Checks
         run: make checks
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.name == 'ubuntu-latest'
 
       # On Linux `make dist-rs` rebuilds this same release profile, and
       # cibuildwheel then rebuilds the tree again from scratch in the manylinux
       # container, so building here is a third compile and a third copy on disk.
       - name: Build
         run: make build
-        if: matrix.os != 'ubuntu-latest'
+        if: matrix.target == 'native' && runner.os != 'Linux'
 
+      # Coverage instrumentation forces a rebuild that shares nothing with the
+      # cached ordinary build, so only the platform that uploads it pays for it.
       - name: Test
         run: make coverage
+        if: matrix.name == 'ubuntu-latest'
+
+      - name: Test
+        run: make test
+        if: matrix.target == 'native' && matrix.name != 'ubuntu-latest'
+
+      - name: Build and test Pyodide wheel
+        run: make test-pyodide
+        if: matrix.target == 'pyodide'
 
       - name: Upload test results (Python)
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
-          name: test-results-${{ matrix.os }}
+          name: test-results-${{ matrix.name }}
           path: '**/junit.xml'
-        if: always()
+        if: always() && matrix.target == 'native'
 
       - name: Publish test results
         uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3  # v2.24.0
         with:
           files: '**/junit.xml'
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.name == 'ubuntu-latest'
 
       - name: Upload coverage
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8  # v4.2.0
-        with:
-          platforms: all
-        if: runner.os == 'Linux' && runner.arch == 'X64'
+        if: matrix.name == 'ubuntu-latest'
 
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/share/swift
           df -h /
         shell: bash
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.target == 'native' && runner.os == 'Linux'
 
       # The wheel is compiled inside the manylinux container, which cannot see
       # the Swatinem/rust-cache the host steps use. pyproject points the
@@ -119,25 +149,40 @@ jobs:
           path: |
             /tmp/cibw-cargo-home
             /tmp/cibw-cargo-target
-          key: cibw-cargo-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'rust/Cargo.lock') }}
+          key: cibw-cargo-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Cargo.lock', 'rust/Cargo.lock') }}
           restore-keys: |
-            cibw-cargo-${{ runner.os }}-
-        if: matrix.os == 'ubuntu-latest'
+            cibw-cargo-${{ runner.os }}-${{ runner.arch }}-
+        if: matrix.target == 'native' && runner.os == 'Linux'
 
-      - name: Build distributions (Linux)
+      - name: Build distributions (Linux x86_64)
         run: |
           rm -rf dist
           make dist-rs
           make dist-py-sdist
-          rm -rf rust/target
           make dist-py-wheel
           make dist-check
           # The container writes these as root; hand them back so the cache save
           # can read them.
           sudo chown -R "$(id -u):$(id -g)" /tmp/cibw-cargo-home /tmp/cibw-cargo-target
           du -sh /tmp/cibw-cargo-home /tmp/cibw-cargo-target
         shell: bash
-        if: matrix.os == 'ubuntu-latest'
+        env:
+          CIBW_ARCHS_LINUX: native
+        if: matrix.name == 'ubuntu-latest'
+
+      - name: Build distributions (Linux ARM64)
+        run: |
+          rm -rf dist
+          make dist-py-wheel
+          make dist-check
+          # The container writes these as root; hand them back so the cache save
+          # can read them.
+          sudo chown -R "$(id -u):$(id -g)" /tmp/cibw-cargo-home /tmp/cibw-cargo-target
+          du -sh /tmp/cibw-cargo-home /tmp/cibw-cargo-target
+        shell: bash
+        env:
+          CIBW_ARCHS_LINUX: native
+        if: matrix.name == 'ubuntu-24.04-arm'
 
       - name: Build distributions (macOS/Windows)
         run: |
@@ -147,22 +192,22 @@ jobs:
         shell: bash
         env:
           CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0
-        if: matrix.os != 'ubuntu-latest'
+        if: matrix.target == 'native' && runner.os != 'Linux'
 
       - name: Test wheel
         uses: actions-ext/python/test-wheel@ec67f82b3bd863948308a41ecb9e70547cbb19c7
         with:
           module: nepenthe
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.target == 'native' && runner.os == 'Linux'
 
       - name: Test source distribution
         uses: actions-ext/python/test-sdist@ec67f82b3bd863948308a41ecb9e70547cbb19c7
         with:
           module: nepenthe
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.name == 'ubuntu-latest'
 
       - name: Upload distributions
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
-          name: dist-${{ matrix.os }}
-          path: dist
+          name: dist-${{ matrix.name }}
+          path: ${{ matrix.dist_path }}

.gitignore

@@ -13,6 +13,10 @@ __pycache__/
 *.exp
 *.lib
 
+# vcpkg
+vcpkg/
+vcpkg_installed/
+
 # Rust
 target
 target-capi
@@ -142,6 +146,7 @@ nepenthe/labextension
 
 # Emscripten SDK (locally installed)
 emsdk
+.pyodide_build/
 
 # Mac
 .DS_Store

Cargo.toml

@@ -17,4 +17,6 @@ tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
 
 [profile.release]
 panic = 'abort'
-lto = true
+# Thin LTO keeps most of the optimization but links in parallel; fat LTO over
+# the rattler dependency tree dominated every wheel build.
+lto = "thin"

Makefile

@@ -12,9 +12,7 @@ develop: develop-rs develop-py  ## setup project for development
 
 .PHONY: requirements-py requirements-rs requirements
 requirements-py:  ## install prerequisite python build requirements
-	python -m pip install --upgrade pip toml
-	python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print("\n".join(c["build-system"]["requires"]))'`
-	python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print(" ".join(c["project"]["optional-dependencies"]["develop"]))'`
+	uv pip install -r pyproject.toml --extra develop
 
 requirements-rs:  ## install prerequisite rust build requirements
 	make -C rust requirements
@@ -112,6 +110,7 @@ coverage-rs:  ## run rust tests and collect test coverage
 
 .PHONY: test coverage tests
 test: test-py test-rs  ## run all tests
+
 coverage: coverage-py coverage-rs  ## run all tests and collect test coverage
 
 # alias

pyproject.toml

@@ -65,6 +65,10 @@ current_version = "0.1.1"
 commit = true
 tag = true
 commit_args = "-s"
+pre_commit_hooks = [
+    "cargo update --workspace",
+    "git add Cargo.lock",
+]
 
 [[tool.bumpversion.files]]
 filename = "pyproject.toml"
@@ -94,6 +98,10 @@ rustup target add aarch64-unknown-linux-gnu
 rustup target add x86_64-unknown-linux-gnu
 rustup show
 """
+# A host `make develop` leaves a built extension in the source tree, which the
+# container would otherwise package instead of its own. The template also runs
+# `cargo clean` here, which would wipe the cached target dir below.
+before-build = "rm -f {package}/nepenthe/*.so"
 # cibuildwheel mounts the host filesystem at /host, so pointing cargo there keeps
 # the wheel build's state outside the container, where the workflow can cache it.
 # The in-container build is the one compile in CI that Swatinem/rust-cache cannot
@@ -123,6 +131,7 @@ archs = "arm64"
 environment = {PATH="$UserProfile\\.cargo\bin;$PATH", CARGO_TERM_COLOR="always"}
 skip = "*win32 *arm_64"
 
+
 [tool.coverage.run]
 branch = true
 omit = [

rust/Makefile

@@ -42,7 +42,7 @@ checks: check
 
 .PHONY: test tests test-ci tests-ci
 test:  ## run the tests
-	cargo llvm-cov nextest --cobertura --output-path  junit.xml
+	cargo nextest run
 
 # alias
 tests: test