pyproject.toml

# ============================== project ==============================

[project]
name = "seqforge"
description = "Compile arbitrary FASTQ + messy metadata into a validated, machine-independent sequencing library manifest and a runnable Snakemake config."
readme = "README.md"
license = "MIT"
license-files = ["LICENSE"]
requires-python = ">=3.13"
authors = [{ name = "Liu Lab" }]
keywords = ["bioinformatics", "single-cell", "fastq", "snakemake", "manifest"]
classifiers = [
    "Development Status :: 2 - Pre-Alpha",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Operating System :: POSIX :: Linux",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Bio-Informatics",
    "Typing :: Typed",
]
# CalVer YYYY.M.PATCH, read by hatch-vcs from the newest git tag. Never add a `version` key.
dynamic = ["version"]

dependencies = [
    "pydantic>=2.7",
    "typer>=0.12",
    "pyyaml>=6",
    "numpy>=2",
    "pooch>=1.8",
    "requests>=2.31",
    "tqdm>=4",
    "huggingface_hub>=0.25",  # publishing a fingerprint package; reading one needs no token
    # Document readers are hard deps, not extras: a paper is the normal shape of the prose we are handed.
    "pypdf>=4",
    "PyMuPDF>=1.24",     # default PDF backend; AGPL-3.0, the one non-permissive dep
    "pdfplumber>=0.11",  # in-text tables -> markdown, so a table cell greps back
    "openpyxl>=3.1",     # supplementary .xlsx sample tables
    "anndata>=0.10",     # the pilot's deliverable format
    "pysam>=0.22",       # BAM read/write for the plate-assay UMI path
    "gffutils>=0.13",    # reads the annotation database liulab-genome built
    # The LLM clients are lazy-imported; `run --no-llm` drives the compiler without them.
    "anthropic>=0.40",
    "openai>=1.40",      # any OpenAI-compatible endpoint (DeepSeek, vLLM, Ollama, ...)
    # liulab-genome and liulab-data are absent on purpose: PyPI rejects metadata carrying a
    # `git+https` reference. They are declared in the pixi tables, which never fold into the wheel.
    # A PyPI user who wants those features installs them from git:
    #     pip install "liulab-genome @ git+https://github.com/liuhlab/liulab-genome.git" \
    #                  "liulab-data   @ git+https://github.com/liuhlab/liulab-data.git"
]

[project.urls]
Homepage = "https://github.com/liuhlab/seqforge"
Issues = "https://github.com/liuhlab/seqforge/issues"
Changelog = "https://github.com/liuhlab/seqforge/blob/main/CHANGELOG.md"

[project.scripts]
seqforge = "seqforge.cli:app"

# ============================== build ==============================

[build-system]
requires = ["hatchling", "hatch-vcs"]
build-backend = "hatchling.build"

# No `fallback-version`: that is a hand-written version string. A clone with no tag in reach
# builds as `0.1.devN`, so every workflow checkout that builds a wheel takes the full history.
[tool.hatch.version]
source = "vcs"

[tool.hatch.build.targets.wheel]
# Carries every non-Python file under the tree already (onlists, .smk modules, KB specs, report
# assets); a `force-include` on top of it is a hard build error.
#
# That now includes the per-context `GLOSSARY.md` files and the records under
# `src/seqforge/*/docs/adr/` — about thirty markdown files. **Deliberately not excluded.** They are
# beside the code precisely so the decision and the code travel together, and an installed copy that
# drops them is a copy where a reader hits `ADR-0034` with nothing to open. The cost is a few hundred
# lines in the wheel, alongside the KB READMEs that already ship.
packages = ["src/seqforge"]

# No `[tool.hatch.metadata] allow-direct-references`, on purpose: its absence is a tripwire, so
# re-adding a git dependency above fails the build rather than a later PyPI upload.

# ============================== pixi ==============================

[tool.pixi.workspace]
channels = ["conda-forge", "bioconda"]
platforms = ["osx-arm64", "linux-64", "osx-64"]

[tool.pixi.pypi-dependencies]
seqforge = { path = ".", editable = true }
# Consumed, never redefined: every mapping module resolves its index through it, and `io/archive.py` calls
# labdata's `experiments_for`. Both ship in liulab-runtime's `lab` feature.
liulab-genome = { git = "https://github.com/liuhlab/liulab-genome.git" }
liulab-data = { git = "https://github.com/liuhlab/liulab-data.git" }

[tool.pixi.dependencies]
pydantic = ">=2.7"
pyyaml = ">=6"
numpy = ">=2"
pooch = ">=1.8"
typer = ">=0.12"
requests = ">=2.31"
tqdm = ">=4"
huggingface_hub = ">=0.25"
pypdf = ">=4"
openpyxl = ">=3.1"
anndata = ">=0.10"
anthropic = ">=0.40"
openai = ">=1.40"
pymupdf = ">=1.28.0,<2"
pdfplumber = ">=0.11.10,<0.12"
gffutils = ">=0.13"
# `pysam` is deliberately NOT mirrored here. bioconda's package carries no PyPI purl, so pixi cannot
# tell it is the same distribution the editable install asks for: it resolves both and the wheel
# overwrites the conda package's files. The wheel vendors htslib and exists for every platform
# declared above, so PyPI alone is sufficient. Do not add it back.
#
# liulab-genome's transitive needs, mirrored from liulab-runtime's `lab` feature. A missing one is an
# ImportError at `import genome`, not a wrong number.
pandas = ">=2.1,<3"
ucsc-fatotwobit = "*"
ucsc-twobitinfo = "*"

# The DAG builder, so compose's wiring gate runs rather than skipping green. `minimal` because
# submitting jobs is the user's. STAR and samtools belong to liulab-runtime, which the rules name;
# no table a RULE resolves against declares one -- see the `star` feature below for the one table
# that does, and why it cannot reach a rule.
[tool.pixi.feature.wf.dependencies]
snakemake-minimal = ">=8"

# THE ALIGNER LIVES HERE, IN A FEATURE NO ENVIRONMENT SHIPS AND NO RULE CAN REACH.
#
# `external` marks a test needing a binary seqforge does not own. Those tests are real coverage --
# the STAR one is the end-to-end proof that a UMI tag put on a read survives into the aligner's own
# output, which is the claim the uBAM route rests on -- and a test that cannot run is not a weaker
# test but a claim with nothing behind it, because a skip is green.
#
# The first attempt at owning them put `star`/`samtools`/`htslib` in a feature that was solved
# ALONGSIDE the rest, and it was reverted (#338) for a reason that was real and is still real: on
# `osx-arm64` bioconda's only STAR build caps `htslib <1.23`, whose `libdeflate 1.22` collides with
# the `libdeflate >=1.25` that `mupdf` requires. One package, two disjoint ranges, no solution. The
# revert then borrowed liulab-runtime's already-locked `align-rna` environment instead, which cost no
# table but bought an unpinned clone of another repository into this one's critical path.
#
# What that attempt missed: STAR is only ever on PATH and is NEVER IMPORTED, so it has no reason to
# share a solve with the PDF stack. `test-star` below omits `solve-group`, solving alone, and the
# collision cannot arise. Verified across all three platforms -- `star` 2.7.11b on each, with
# htslib/samtools 1.22.1 on `osx-arm64` and 1.23.1 on `linux-64` and `osx-64`.
#
# What is preserved is the half that always mattered: `test-star` carries no `seqforge`, joins no
# solve group, and reaches no environment the wheel ships or a Snakemake rule resolves against. An
# alignment environment a RULE names is still liulab-runtime's alone.
# `test_the_aligner_is_confined_to_the_test_only_environment` is what keeps that true, because the
# comments saying it do not fail a build.
#
# STAR IS PINNED AND THE TWO BESIDE IT ARE NOT. The knowledge base now encodes which of this binary's
# flag combinations are legal -- which `clipAdapterType` values exist, which end a clip may be
# declared at, and what arity a clip takes under `--soloType`. Every one of those was established by
# RUNNING this version, because STAR's shipped help is stale against its shipped code: it still
# advertises a `clipAdapterType` mode the binary refuses outright. A lock file alone pinned nothing --
# both repositories declared `*`, so a `pixi update` in either moved the aligner silently and nothing
# went red when the suite's STAR and production's STAR drifted apart. The version is part of the
# contract, so it is declared. liulab-runtime's `align-rna` carries the same pin, and it is the one
# that matters more: that is where a composed rule's STAR comes from.
#
# `samtools` and `htslib` stay floating deliberately. They encode nothing, and constraining them is
# exactly what reopens the `libdeflate` collision above.
[tool.pixi.feature.star.dependencies]
star = "2.7.11b.*"
samtools = "*"
htslib = "*"

[tool.pixi.feature.py313.dependencies]
python = "3.13.*"

[tool.pixi.feature.test.dependencies]
pytest = "*"
pytest-cov = "*"
pytest-xdist = "*"

# One thread per worker: every xdist worker is its own process and each would otherwise size a BLAS/
# OpenMP pool to the whole box, then fight the others for it. Three names because three libraries are
# underneath. On the FEATURE, not on a task, so all five ways of invoking the suite inherit it;
# `test_the_test_environment_pins_its_thread_pools` asserts both the declaration and its arrival.
#
# PATH is here for the same reason and it is the whole mechanism: one declaration puts `test-star`'s
# binaries in front of every way of running the suite -- `test`, `test-fast`, `test-external`,
# `check`, `pixi shell -e test`, and each xdist worker -- so no verb spells an aligner path and no
# developer types one. It replaces the incantation the borrowed environment needed:
#
#     PATH="<liulab-runtime>/.pixi/envs/align-rna/bin:$PATH" pixi run -e test test-external
#
# If `test-star` is not installed this names a directory that does not exist, which is harmless: PATH
# tolerates it and the binary-gated tests skip exactly as they did before. That is the degradation
# `test-fast` is the verb for. It is ALSO how a skip goes quiet again, so CI runs `STAR --version`
# before the selection rather than trusting the skip count.
[tool.pixi.feature.test.activation.env]
OMP_NUM_THREADS = "1"
OPENBLAS_NUM_THREADS = "1"
MKL_NUM_THREADS = "1"
PATH = "$PIXI_PROJECT_ROOT/.pixi/envs/test-star/bin:$PATH"

[tool.pixi.feature.dev.dependencies]
ruff = "*"
mypy = "*"
pre-commit = "*"
types-pyyaml = "*"
types-requests = "*"
types-openpyxl = "*"
pyinstrument = ">=5.1.2,<6"

[tool.pixi.feature.docs.dependencies]
mkdocs-material = "*"
# Exact, so a new rule cannot turn CI red without a deliberate bump. The one copy: CI's `markdown`
# job and the pre-commit hook both run the `markdownlint` task.
markdownlint-cli2 = "==0.23.1"
# Exact: the lab's jargon map and the grade-11 number were measured against this version.
vale = "3.19.0.*"

# What the `dist` environment holds: a build frontend and twine, and not seqforge. It solves alone
# because twine 7 needs packaging 26 and snakemake caps packaging below 26, while twine 6 under
# packaging 25 cannot read the metadata version hatchling now writes.
[tool.pixi.feature.dist.dependencies]
python-build = "*"
twine = "*"

# On the feature, so `pixi run build` finds the one environment that has them with no `-e` flag.
# CI's `build` job and the release job both run these, so the wheel that ships is built the way
# every pull request builds it.
[tool.pixi.feature.dist.tasks]
build = "python -m build"
# Inspects the wheel `build` just made: as a test it built a second one.
check-wheel = "python scripts/check_wheel_contents.py"
# Renders the metadata the way PyPI will, so a bad long description or a `git+https` dependency
# fails before the upload.
check-dist = "twine check dist/*"

[tool.pixi.environments]
default = { features = ["py313", "dev", "test", "docs", "wf"], solve-group = "default" }
test    = { features = ["py313", "test", "wf"], solve-group = "default" }
docs    = { features = ["py313", "docs"], solve-group = "default" }
# TWO absences, and together they are the entire design. No `solve-group`: the three above share one
# so the editor, the suite and the site see identical versions, and `test-star` must NOT join them,
# because STAR's `libdeflate` and the PDF stack's have no common solution on `osx-arm64`. And
# `no-default-feature`, because every environment otherwise inherits `[tool.pixi.dependencies]` --
# which is where `pymupdf` is, so without this the collision arrives by the back door and the lock
# fails exactly as the reverted attempt did.
#
# What is left solves alone, carries no Python and no `seqforge`, and exists only to put two binaries
# on PATH -- all a test ever needed, since nothing imports an aligner.
test-star = { features = ["star"], solve-group = "star", no-default-feature = true }
# No default feature either: a build needs a frontend and the source tree, and installing seqforge
# with all its dependencies would only slow the release job down.
dist = { features = ["py313", "dist"], solve-group = "dist", no-default-feature = true }

[tool.pixi.tasks]
lint      = "ruff check ."
fmt       = "ruff format ."
fmt-check = "ruff format --check ."
# Bare: `[tool.mypy] files` owns the scope, so the editor's checker cannot disagree with this one.
typecheck = "mypy"
# THREE LANES, AND TOGETHER THEY ARE THE SUITE. `test` is the unit lane, `test-corpus` grades the
# hermetic corpus, `test-external` needs binaries we do not own. Every test lands in exactly one:
# `external` against its negation is total by boolean negation, and the corpus is one file, so
# nothing outside the suite decides where a test runs. CI gives each lane a job and `check` runs all
# three at once. `test_every_gate_on_an_external_binary_carries_the_marker` is what keeps the first
# split honest — an unmarked gate would sit in the unit lane and skip itself green.
#
# `auto` capped: bare `auto` is slower on a big box, a bare number is wrong on a 2-core runner. The
# caps are DIVIDED between the lanes because `check` runs them concurrently and each would otherwise
# size itself to the whole box. 6/4/4 is proportional to the CPU each lane actually costs, which is
# what balances them — measured, and the three walls land within four seconds of each other. It sums
# to more than the cores on the maintainer's box ON PURPOSE: taking it down to exactly twelve has to
# come out of the unit lane, which then cannot finish inside the wall this split exists to beat, and
# the lanes do not all run to the end anyway. On a CI runner none of it binds: `auto` finds fewer
# cores than the smallest cap, so each job still takes the whole runner and the division costs CI
# nothing. `loadgroup` groups only tests carrying
# `xdist_group`; why not `loadfile`, and when to mark a module, is a decision record. Measurements:
# docs/research/xdist-worker-sweep-2026-08-01.md and docs/research/test-suite-cost-shape.md.
#
# The corpus path is spelled twice, here and in `test-corpus`. They are one decision and move
# together — a stale `--ignore` would run the corpus in both lanes rather than drop it from neither.
test      = "pytest -m 'not external' --ignore=tests/test_evals.py -n auto --maxprocesses 6 --dist=loadgroup"
# The corpus grades every case in `evals/cases/` through the real compiler. That is a different claim
# from a unit test's, and it was the module that dominated the suite's wall, so it gets its own lane
# rather than its own deletion. It leaves the unit lane, not the gate.
#
# `-m 'not external'` here even though the corpus holds no external test today: without it the two
# lanes are disjoint only by that happening to stay true, and an external test added to this module
# would run in BOTH. With it, the three selections are disjoint by construction rather than by
# inspection — which is the only kind of partition worth claiming.
test-corpus = "pytest tests/test_evals.py -m 'not external' -n auto --maxprocesses 4 --dist=loadgroup"
# Drops what needs a binary we do not own and what checks the repo — for a machine with no snakemake,
# or one that would rather not install `test-star`. Alone, so it keeps the whole box.
test-fast   = "pytest -m 'not external and not repo' -n auto --maxprocesses 12 --dist=loadgroup -q"
# `depends-on` here precisely BECAUSE pixi runs it sequentially: the binaries must be proved to answer
# before the selection happens. Without that, a missing `test-star` is a PATH entry that does not
# exist, every gated test skips itself, and this verb reports green having run nothing — which is the
# failure the lane exists to end. `test`'s activation already put `test-star/bin` in front, so on any
# machine that ran `pixi install` this is the whole command.
test-external = { cmd = "pytest -m external -n auto --maxprocesses 4 --dist=loadgroup", depends-on = ["require-binaries"] }
require-binaries = "bash scripts/require_binaries.sh"
# Serial on purpose: `-x` is unreliable under xdist, and 12 workers to re-run three tests loses.
test-failed = "pytest --lf --new-first -x -q"
test-cov  = "pytest --cov=seqforge --cov-report=term-missing"
# The networked validation benchmark, not part of `check`. Exits 3 on any false-accept.
eval      = "seqforge eval run"
# Not `depends-on`: pixi runs those sequentially. `check.sh` runs its steps concurrently and prints
# each one's output whole, in a fixed order. The three test lanes are separate steps rather than one,
# so the corpus and the binaries land in what was idle time — the suite does not saturate its workers,
# and a lane that finishes early gives its share back to the one that has not. The gate runs the same
# three selections CI does, which is the point: what goes red here is what goes red there.
check     = "bash scripts/check.sh lint fmt-check typecheck conformance test test-corpus test-external"
install-skills = "python skills/install.py --user"
# The template's rules, checked against this repo. `python`, so the skill-link rule runs
# `install.py --check` on the environment's interpreter. The script is a verbatim copy: change a rule
# upstream, not here.
conformance = "python scripts/conformance.py"
docs      = "mkdocs serve"
docs-build = "mkdocs build --strict"
# Publishes to `gh-pages`, so only docs.yml runs it. The committer comes from that step's `env:`,
# because a task that set it in git's config would leave it behind on a laptop.
docs-deploy = "mkdocs gh-deploy --force --strict"
# No file arguments: cli2 then reads its globs and ignores from `.markdownlint-cli2.yaml`, so CI and
# the pre-commit hook lint the same set. Outside `check`, for the reason AGENTS.md gives.
markdownlint = "markdownlint-cli2"
# Renders a benchmark run directory as one HTML page. benchmark.yml appends a `--title` naming the
# commit; `--source` is the command that wrote the directory, and moves with that step.
benchmark-report = "seqforge eval report report -o report/benchmark.html --source 'seqforge eval run --no-llm --cases evals/benchmark -C report'"
# Prose lint, beside `markdownlint` and outside `check` for the same reason: it answers a question a
# code change does not ask. `.`, never an absolute path: an absolute path matches no `.vale.ini`
# section and exits 0. The globs stop vale walking trees it never lints, worktrees among them.
vale = "vale --glob='!{.pixi,.claude,site,dist,build,seqforge}/**' ."

# ============================== liulab ==============================
# Read by `scripts/conformance.py`; its docstring has the rules. Keys are path prefixes matched
# against the tracked files, not globs; a trailing `/` matches a directory. Values name the Vale
# Length rule that applies, or `false` where none does.
#
# No key for a context package's GLOSSARY.md or docs/adr/. `.vale.ini` reaches those with a
# `src/seqforge/*/` glob inside the same section headers as `GLOSSARY-MAP.md` and `docs/adr/`, so
# pairing these two keys already checks the settings those files get. A key per context would
# match no header, because a key is a plain prefix and the header spells a glob.
[tool.liulab.agent-docs]
"AGENTS.md" = "LengthDoc"
"GLOSSARY-MAP.md" = false
"skills/" = "LengthDoc"
"docs/agents/" = "LengthDoc"
"docs/adr/" = "LengthAdr"
"docs/research/" = false

# Keyed by rule name, the reason as the value. Per-rule, never per-file, and printed on every run.
[tool.liulab.waived]
agent-docs-unpublished = "mkdocs honours `exclude_docs:`, which keeps docs/adr/ and docs/research/ out of the site, its nav and its search index; the template's front matter exists only because zensical ignores that key"

# ============================== ruff ==============================

[tool.ruff]
line-length = 100
src = ["src", "tests", "scripts"]
extend-exclude = [".pixi", "site", "build", "dist"]

[tool.ruff.lint]
select = [
    "E",   # pycodestyle errors
    "W",   # pycodestyle warnings
    "F",   # pyflakes
    "I",   # isort
    "UP",  # pyupgrade
    "B",   # flake8-bugbear
    "C4",  # flake8-comprehensions
    "SIM", # flake8-simplify
    "PT",  # flake8-pytest-style
    "PTH", # flake8-use-pathlib
    "N",   # pep8-naming
    "D",   # pydocstyle (numpy convention, configured below)
    "D415", # summary ends in terminal punctuation: stands in for D400, below
    "RUF", # ruff-specific
]
# E501: the formatter owns line length. UP046/UP047: keep classic Generic[T]/TypeVar, which has the
# better-tested pydantic-v2 + mypy support. N818: an exception here names the condition it signals
# (`CaseSkipped`, `CeilingExceeded`, `TermUnknown`), and several are control flow, not errors.
# D401: a docstring here states what is true ("The records, as a GENERATOR …"), in the declarative
# voice of the commit subjects, not the imperative. D400: a predicate's docstring asks its question
# ("Did a budget stop the read?"), so the summary may end in `?`, which D415 accepts and D400 does
# not.
ignore = ["E501", "UP046", "UP047", "N818", "D400", "D401"]
# Typography the prose means: a multiplication sign for a fold or a count, an en dash for a range, a
# prime for a nucleic-acid end (5′, 3′), a minus sign, curly quotes, and angle quotes for a pager's
# arrows.
allowed-confusables = ["×", "–", "′", "−", "‘", "’", "‹", "›"]

[tool.ruff.lint.pydocstyle]
convention = "numpy"

[tool.ruff.lint.per-file-ignores]
# SIM300: an UPPER_CASE name is a module table under test, so it is the actual on the left of `==`.
# D: a test's name and its assertion messages say what it checks.
"tests/**" = ["SIM300", "D"]

[tool.ruff.lint.flake8-bugbear]
# Typer uses call expressions as parameter defaults by design.
extend-immutable-calls = ["typer.Argument", "typer.Option"]

# ============================== mypy ==============================

[tool.mypy]
strict = true
warn_unreachable = true
plugins = ["pydantic.mypy"]
# The scope lives here, not in the `typecheck` task: an editor's language server never reads a task
# string. `test_nothing_tracked_escapes_the_type_checker` holds it open at both ends.
files = ["src", "tests", "scripts", "skills"]
# The gitignored scratch harnesses, and nothing else — that guard fails if an exclusion ever hides a
# file git is tracking.
exclude = ['^tests/_[^/]*_test\.py$', '^tests/_test_[^/]*\.py$']

# Untyped third parties. In each case what protects the deliverable is a test, not the stub: the
# transpose assertion for scipy, the registry checksum for pooch, the synthetic BAM + annotation for
# pysam/gffutils, and one thin guarded function per PDF engine in `harvest/normalize.py`.
[[tool.mypy.overrides]]
module = ["scipy.*", "pooch", "pooch.*", "pysam", "pysam.*", "gffutils", "gffutils.*"]
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = ["pymupdf", "pymupdf.*", "pdfplumber", "pdfplumber.*"]
ignore_missing_imports = true
follow_imports = "skip"

# The template's checker, copied verbatim, is typed against pyright, which does not flag the one
# untyped PyYAML call it makes. Relaxed here so the copy stays a copy.
[[tool.mypy.overrides]]
module = ["conformance"]
disallow_untyped_calls = false

[tool.pydantic-mypy]
init_forbid_extra = true
init_typed = true
warn_required_dynamic_aliases = true

# Pylance does not type-check; mypy does, and the editor runs mypy. Here rather than in `.vscode/` so
# any pyright-based server honours it. Hover, completion and go-to-definition are unaffected.
[tool.pyright]
typeCheckingMode = "off"

# ============================== pytest ==============================

[tool.pytest.ini_options]
testpaths = ["tests"]
# Pinned: the default also collects `*_test.py`, so an untracked scratch file was collected locally
# and absent in CI.
python_files = ["test_*.py"]
# `zarr` and `anyio` arrive transitively (anndata, httpx) and each registers a `pytest11` entry point
# every worker imports at startup, though no test here uses either.
addopts = [
  "-ra",
  "--strict-markers",
  "--strict-config",
  "-p", "no:zarr",
  "-p", "no:anyio",
]
xfail_strict = true
filterwarnings = ["error"]
# Both markers are semantic, not speed: what a test needs, and what it is about. A `slow` marker was
# rejected — a hand-maintained list keyed on a number that changes whenever you optimise.
markers = [
  "external: needs a binary seqforge does not own (snakemake/STAR/htslib/build)",
  "repo: checks repo consistency, not src/ behaviour",
]

[tool.coverage.run]
branch = true
source = ["src/seqforge"]