Merge pull request #17 from platforma-open/feat/seq-properties-vectorization

Vectorize compute-properties: replace per-row BioPython with array math (~10x at scale)

Author: PaulNewling

.changeset/vectorize-sequence-properties.md

@@ -0,0 +1,8 @@
+---
+'@platforma-open/milaboratories.sequence-properties.software': patch
+'@platforma-open/milaboratories.sequence-properties.workflow': patch
+---
+
+Vectorize the property computation — single-threaded numpy/polars array math replaces per-row BioPython. ~5x faster at 50k clones; output unchanged within the quantized-equal contract (signed-zero canonicalized).
+
+Reduce the vectorized engine's peak memory (int8/int32 indices, share the clean intermediate between counts and instability, drop dead retention) — ~4.85 GB/1M clones vs ~8.5 GB before. Raise the compute-properties step's `mem()` from 4GiB to 16GiB to cover ~2M clones at the new footprint with headroom.

.github/workflows/python-tests.yaml

@@ -23,6 +23,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v5
         with:
+          version: "0.11.16"
           enable-cache: true
           cache-dependency-glob: software/uv.lock
 
@@ -36,4 +37,43 @@ jobs:
         run: uv run ruff format --check
 
       - name: Run pytest
-        run: uv run pytest
+        # Exclude @slow (the perf benchmark) from the gate — it asserts a
+        # wall-clock budget that flakes on shared runners. Run it explicitly
+        # with `uv run pytest -m slow`.
+        run: uv run pytest -m "not slow"
+
+  requirements-sync:
+    # src/requirements.txt is generated from pyproject.toml (the single source of
+    # truth) and is what pl-pkg builds from. This job fails if the committed file
+    # has drifted from pyproject — e.g. someone changed a dependency but forgot to
+    # regenerate. Fix locally with `pnpm deps:export` (see software/package.json).
+    if: github.repository != 'milaboratory/platforma-sequence-properties'
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: software
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.11.16"
+          enable-cache: true
+          cache-dependency-glob: software/uv.lock
+
+      - name: Regenerate requirements.txt from pyproject.toml
+        # Same script `pnpm deps:export` runs locally — single source of truth.
+        run: bash scripts/deps-export.sh
+
+      - name: Verify requirements.txt is in sync with pyproject.toml
+        run: |
+          if ! git diff --exit-code -- src/requirements.txt; then
+            echo ""
+            echo "::error::src/requirements.txt is out of sync with pyproject.toml"
+            echo "Dependencies are defined in software/pyproject.toml; src/requirements.txt is generated."
+            echo "Resolve locally with:"
+            echo "    cd software && pnpm deps:export"
+            echo "then commit the updated src/requirements.txt."
+            exit 1
+          fi
+          echo "src/requirements.txt is in sync with pyproject.toml."

software/package.json

@@ -8,6 +8,7 @@
     "changeset": "changeset",
     "version-packages": "changeset version",
     "build": "pl-pkg build",
+    "deps:export": "bash scripts/deps-export.sh",
     "prepublishOnly": "pl-pkg prepublish"
   },
   "block-software": {

software/pyproject.toml

@@ -2,16 +2,24 @@
 name = "sequence-properties-calc-script"
 version = "0.0.0"
 requires-python = ">=3.12.0, <3.13"
+# Runtime dependencies — single source of truth. These exact pins are exported to
+# src/requirements.txt (top-level only) via `pnpm deps:export`, which is what pl-pkg
+# builds from and what the scientific-slim runenv installs offline. Keep versions
+# aligned with the wheels shipped by the runenv.
+dependencies = [
+    "polars-lts-cpu==1.33.1",
+    "numpy==2.2.6",
+    "pyarrow==21.0.0",
+    "biopython==1.87",
+]
 
 [dependency-groups]
+# Dev-only tooling (not shipped). Runtime libs live in [project.dependencies] above.
 dev = [
-    "polars>=1.39.0",
-    "numpy>=2.0.0",
-    "pyarrow>=18.0.0",
-    "biopython>=1.84",
     "pytest>=9.0.2",
     "pytest-cov>=6.0.0",
     "ruff>=0.7.0",
+    "hypothesis>=6.155.2",
 ]
 
 [tool.pytest.ini_options]
@@ -51,6 +59,13 @@ ignore = []
 fixable = ["ALL"]
 unfixable = []
 
+[tool.ruff.lint.per-file-ignores]
+# E402: these modules intentionally run setup before imports — main.py pins
+# POLARS_MAX_THREADS before polars is imported; _corpus_gen.py adds src to
+# sys.path before importing the package (it runs standalone via `python -m`).
+"src/main.py" = ["E402"]
+"tests/_corpus_gen.py" = ["E402"]
+
 [tool.ruff.format]
 quote-style = "double"
 indent-style = "space"

software/scripts/deps-export.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Regenerate src/requirements.txt from pyproject.toml (the single source of truth).
+# One definition, shared by `pnpm deps:export` and the requirements-sync CI job.
+#
+# --no-deps: top-level pins only. The runenv supplies transitive deps for the
+# offline (--no-index) install, so requirements.txt must not pin a full closure.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+uv pip compile pyproject.toml --no-deps --no-annotate \
+  --custom-compile-command "pnpm deps:export" -o src/requirements.txt

software/src/main.py

@@ -9,6 +9,15 @@
 
 from __future__ import annotations
 
+import os
+
+# Pin the polars thread pool to 1 BEFORE polars (or anything importing it,
+# including `pipeline`) is loaded — POLARS_MAX_THREADS is read once at import.
+# The block reserves cpu(1), and single-threaded execution keeps output
+# byte-stable (no thread-count-dependent reduction order). setdefault so an
+# explicit override from the environment still wins.
+os.environ.setdefault("POLARS_MAX_THREADS", "1")
+
 import argparse
 import json
 import logging