Add plane rendering

.github/workflows/ci.yml

@@ -43,6 +43,19 @@ jobs:
       # Empty elsewhere, which wgpu treats as unset.
       WGPU_BACKEND_TYPE: ${{ matrix.platform == 'windows-latest' && 'Vulkan' || '' }}
 
+      # Put pytest's tmp_path in the workspace (D:) on Windows. The default,
+      # the temp directory on C:, takes about 7 ms to sync a file, and the
+      # render tests write a few hundred zarr chunks each: 1.4 s a pyramid
+      # against 0.5 s here (scripts/ci_profiling/ci_profiling_investigation.md).
+      # A relative path, because pytest splits this like a shell would and a
+      # Windows path's backslashes would not survive.
+      PYTEST_ADDOPTS: ${{ matrix.platform == 'windows-latest' && '--basetemp=.pytest_basetemp' || '' }}
+
+      # Where spatial-graph (through witty) keeps the C++ modules it compiles on
+      # first use; saved between runs by the "Restore compiled graph modules"
+      # step below.
+      WITTY_CACHE_DIR: ${{ github.workspace }}/.witty_cache
+
       # spatial-graph >=0.1.0 ships prebuilt rtree wheels,
       # but I was getting incompatible CPython errors.
       SPATIAL_GRAPH_NO_PREBUILT: "1"
@@ -84,6 +97,34 @@ jobs:
           python -m pip install -U pip
           python -m pip install .[test,pyside,remote,anywidget,examples,graph]
 
+      # spatial-graph compiles a Cython/C++ module the first time each graph or
+      # rtree configuration is used. With nothing cached that is 30 to 50 s a
+      # job, all of it charged to whichever tests come first
+      # (test_roi_edge_pick_resolves_the_row_on_demand and
+      # test_mask_and_roi_agree[1-1-extent0]). witty finds a module by the hash
+      # of its source, so a stale cache only means compiling again. The key
+      # holds the versions the modules were built against; a new version starts
+      # from the last cache of the same Python and adds to it.
+      - name: Compiled graph modules - versions
+        id: witty
+        shell: bash
+        run: |
+          python -c "
+          import hashlib, sys
+          from importlib.metadata import version
+          v = ' '.join(f'{p}={version(p)}' for p in ('spatial-graph', 'witty', 'cython', 'numpy'))
+          print(v, file=sys.stderr)
+          print('versions=' + hashlib.sha256((v + sys.version).encode()).hexdigest()[:16])
+          " >> "$GITHUB_OUTPUT"
+
+      - name: Restore compiled graph modules
+        uses: actions/cache@v5
+        with:
+          path: .witty_cache
+          key: witty-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ steps.witty.outputs.versions }}
+          restore-keys: |
+            witty-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-
+
       # The render tests skip themselves when no adapter resolves, so a backend
       # that fails to load would turn ~450 tests into a silent green. Fail here
       # instead, and log which adapter each platform actually picked.
@@ -92,6 +133,14 @@ jobs:
         run: |
           python -c "from pygfx.renderers.wgpu import get_shared; print('adapter:', get_shared().device.adapter.summary)"
 
+      # Two jobs on the same code have differed by 1.6 times. This logs what
+      # could explain it: the CPU the runner drew, a fixed piece of work timed
+      # on it, and the coverage core (pytest's header repeats the core the run
+      # itself uses).
+      - name: Machine and coverage core
+        shell: bash
+        run: python scripts/ci_profiling/machine_info.py
+
       - name: 🧪 Run Tests
         run: pytest --color=yes --durations=25 --cov --cov-report=xml --cov-report=term-missing
 

.github/workflows/profile.yml

@@ -0,0 +1,240 @@
+name: Profile
+
+# Measures why the tests take longer on one platform than another. Not part of
+# CI, and it never starts by itself: run it by hand (Actions -> Profile -> Run
+# workflow, or `gh workflow run profile.yml --ref <branch>`). GitHub only
+# offers that for a workflow file that is on the default branch.
+#
+# To have it run on a pull request again while investigating, add back:
+#
+#   pull_request:
+#     paths:
+#       - ".github/workflows/profile.yml"
+#       - "scripts/ci_profiling/**"
+#
+# Each platform uploads a `probes-<platform>` artifact (the micro-probes, a
+# few minutes) and a `profile-<platform>` one; compare them with
+# scripts/ci_profiling/compare.py (see its docstring).
+
+on:
+  workflow_dispatch:
+    inputs:
+      tests:
+        description: "Test files to profile (space separated)"
+        default: >-
+          tests/render/test_render_plane_gizmo.py
+          tests/render/test_multiscale_level_alignment.py
+          tests/render/test_plane_planning.py
+      full_suite:
+        description: "Also run the whole suite as CI does, with per-test times"
+        type: boolean
+        default: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Minutes, not tens of minutes: the micro-probes alone, so their numbers are
+  # in before the profile job below is done. Uploads `probes-<platform>`.
+  probes:
+    name: probes ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, macos-latest, windows-latest]
+
+    env:
+      # As in ci.yml.
+      WGPU_BACKEND_TYPE: ${{ matrix.platform == 'windows-latest' && 'Vulkan' || '' }}
+      SPATIAL_GRAPH_NO_PREBUILT: "1"
+      OUT: probes_out
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v7
+
+      # The environment below is ci.yml's, step for step, so the numbers are
+      # the ones CI sees.
+      - name: Set up headless GUI
+        uses: pyvista/setup-headless-display-action@v5
+        with:
+          qt: true
+
+      - name: Install llvmpipe and lavapipe for offscreen canvas
+        if: matrix.platform == 'ubuntu-latest'
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt install -y libegl1-mesa-dev libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
+
+      - name: Install lavapipe for offscreen canvas
+        if: matrix.platform == 'windows-latest'
+        uses: jakoch/install-vulkan-sdk-action@v1
+        with:
+          install_lavapipe: true
+          stripdown: true
+          cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+          cache-dependency-path: "pyproject.toml"
+          cache: "pip"
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install .[test,pyside,remote,anywidget,examples,graph]
+
+      - name: Machine
+        run: |
+          mkdir -p "$OUT"
+          python -c "from pygfx.renderers.wgpu import get_shared; print('adapter:', get_shared().device.adapter.summary)"
+          python scripts/ci_profiling/machine_info.py | tee "$OUT/machine_info.txt"
+
+      # No cellier code involved in the first three.
+      - name: Probe - file writes
+        if: always()
+        run: python scripts/ci_profiling/probe_file_io.py --out "$OUT/probe_file_io.json"
+
+      - name: Probe - frame time
+        if: always()
+        run: python scripts/ci_profiling/probe_render.py --thread-sweep default,1,2,4 --out "$OUT/probe_render.json"
+
+      - name: Probe - single wgpu calls
+        if: always()
+        run: python scripts/ci_profiling/probe_gpu_calls.py --out "$OUT/probe_gpu_calls.json"
+
+      # Reads the datasets of two test files through cellier's stores.
+      - name: Probe - brick reads
+        if: always()
+        run: python scripts/ci_profiling/probe_chunk_read.py --out "$OUT/probe_chunk_read.json"
+
+      - name: Upload
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: probes-${{ matrix.platform }}
+          path: ${{ env.OUT }}
+          if-no-files-found: error
+
+  profile:
+    name: profile ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, macos-latest, windows-latest]
+
+    env:
+      # As in ci.yml.
+      WGPU_BACKEND_TYPE: ${{ matrix.platform == 'windows-latest' && 'Vulkan' || '' }}
+      SPATIAL_GRAPH_NO_PREBUILT: "1"
+      DEFAULT_TESTS: >-
+        tests/render/test_render_plane_gizmo.py
+        tests/render/test_multiscale_level_alignment.py
+        tests/render/test_plane_planning.py
+      INPUT_TESTS: ${{ inputs.tests }}
+      OUT: profile_out
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v7
+
+      # The environment below is ci.yml's, step for step, so the numbers are
+      # the ones CI sees.
+      - name: Set up headless GUI
+        uses: pyvista/setup-headless-display-action@v5
+        with:
+          qt: true
+
+      - name: Install llvmpipe and lavapipe for offscreen canvas
+        if: matrix.platform == 'ubuntu-latest'
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt install -y libegl1-mesa-dev libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
+
+      - name: Install lavapipe for offscreen canvas
+        if: matrix.platform == 'windows-latest'
+        uses: jakoch/install-vulkan-sdk-action@v1
+        with:
+          install_lavapipe: true
+          stripdown: true
+          cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+          cache-dependency-path: "pyproject.toml"
+          cache: "pip"
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install .[test,pyside,remote,anywidget,examples,graph]
+
+      - name: Machine
+        run: |
+          mkdir -p "$OUT/junit" "$OUT/prof"
+          echo "TESTS=${INPUT_TESTS:-$DEFAULT_TESTS}" >> "$GITHUB_ENV"
+          python -c "import os, platform, tempfile; print(platform.platform()); print('cpus:', os.cpu_count()); print('temp:', tempfile.gettempdir()); print('cwd:', os.getcwd())"
+          python -c "from pygfx.renderers.wgpu import get_shared; print('adapter:', get_shared().device.adapter.summary)"
+          python scripts/ci_profiling/machine_info.py
+
+      # --- cProfile of each test file ---------------------------------------
+      # Also warms every cache, so the timed runs below start equal.
+
+      - name: cProfile each test file
+        if: always()
+        run: |
+          for test in $TESTS; do
+            name=$(basename "$test" .py)
+            echo "::group::$name"
+            python -m cProfile -o "$OUT/prof/$name.prof" -m pytest "$test" -q --durations=10 || true
+            echo "::endgroup::"
+          done
+
+      # --- The same test files, one thing changed at a time -----------------
+
+      - name: Timed - baseline (no coverage)
+        if: always()
+        run: pytest $TESTS -q --junitxml="$OUT/junit/baseline.xml" || true
+
+      - name: Timed - with coverage
+        if: always()
+        run: pytest $TESTS -q --cov --cov-report= --junitxml="$OUT/junit/coverage.xml" || true
+
+      # pytest's tmp_path is under the system temp directory, which on the
+      # Windows runner is on C:; the workspace is on another drive.
+      - name: Timed - tmp_path on the workspace drive
+        if: always()
+        run: pytest $TESTS -q --basetemp="${{ github.workspace }}/.pytest_basetemp" --junitxml="$OUT/junit/basetemp_workspace.xml" || true
+
+      - name: Timed - llvmpipe on one thread
+        if: always() && matrix.platform != 'macos-latest'
+        env:
+          LP_NUM_THREADS: "1"
+        run: pytest $TESTS -q --junitxml="$OUT/junit/lp_threads_1.xml" || true
+
+      # --- The whole suite, as CI runs it, with every test's time -----------
+
+      - name: Timed - full suite as CI
+        if: always() && (github.event_name != 'workflow_dispatch' || inputs.full_suite)
+        run: pytest --color=yes --durations=50 --cov --cov-report= --junitxml="$OUT/junit/full.xml" || true
+
+      - name: Upload
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: profile-${{ matrix.platform }}
+          path: ${{ env.OUT }}
+          if-no-files-found: error

.gitignore

@@ -51,6 +51,8 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+.pytest_basetemp/
+.witty_cache/
 
 # Translations
 *.mo