diff --git a/benchmarks/cases.py b/benchmarks/cases.py index 0733971..974ef0b 100644 --- a/benchmarks/cases.py +++ b/benchmarks/cases.py @@ -83,6 +83,12 @@ class Case: output_suffix: str = ".h5ad" #: Use the var-heavy shape rather than the tier's main shape. var_heavy: bool = False + #: Use a wide variant of the main input -- float64 values, int64 indices + #: -- named for how it stores X: "uncompressed" or "lzf". + wide: Optional[str] = None + #: Report the output's size against the input's, as before -> after. For + #: cases whose point is what they do to the file's size. + compare_size: bool = False #: An auxiliary input the case needs, e.g. the CSV an import reads. #: Built once from the first input and offered to CLI contenders as #: `{sidecar_csv}` and to scripts as $BENCH_SIDECAR_CSV. @@ -483,23 +489,41 @@ def _py(body: str) -> str: ], ), # -- recast (named convert until 0.7.0; the case names keep the history) -- + # + # `convert-dtype` ran until 0.7.0 on the main fixture, which is already + # float32 with int32 indices: it measured a rewrite that changed nothing, + # and its output was the size of its input. `recast-narrow` replaces it, + # on an input worth narrowing, and reports the size it saves. Case( - name="convert-dtype", - question="Rewrite X as float32.", + name="recast-narrow", + question=( + "Narrow X from float64 values with int64 indices to float32 and " + "int32, as issue #13 asked. X is uncompressed, as `write_h5ad` " + "leaves it by default, so the output keeps that and the baseline " + "writes it the same way." + ), + wide="uncompressed", + compare_size=True, contenders=[ Contender( "adata-cli", argv=[ "adata", "recast", "{input0}", "X", "-o", "{output}", - "--dtype", "float32", "--force", + "--dtype", "float32", "--indices-dtype", "int32", ], ), Contender( "anndata (in memory)", + # compression=None, not COMPRESSION: the input's X is + # uncompressed and adata-cli keeps it that way, so lzf here + # would credit anndata with a saving the cast did not make. script=_py( "obj = ad.read_h5ad(IN[0])\n" - "obj.X = obj.X.astype('float32')\n" - "obj.write_h5ad(OUT, compression=COMPRESSION)\n" + "X = obj.X.astype('float32')\n" + "X.indices = X.indices.astype('int32')\n" + "X.indptr = X.indptr.astype('int32')\n" + "obj.X = X\n" + "obj.write_h5ad(OUT, compression=None)\n" ), ), Contender( @@ -507,10 +531,47 @@ def _py(body: str) -> str: unsupported="no dtype rewrite; scanpy defers to anndata", ), ], + tags=["headline"], + ), + Case( + name="recast-narrow-lzf", + question=( + "The same narrowing on an lzf-compressed X. Integer counts in " + "float64 are mostly zero bytes, which lzf already removes, so " + "expect a small saving on disk; the halving is in memory, once " + "the matrix is loaded." + ), + wide="lzf", + compare_size=True, + contenders=[ + Contender( + "adata-cli", + argv=[ + "adata", "recast", "{input0}", "X", "-o", "{output}", + "--dtype", "float32", "--indices-dtype", "int32", + ], + ), + Contender( + "anndata (in memory)", + script=_py( + "obj = ad.read_h5ad(IN[0])\n" + "X = obj.X.astype('float32')\n" + "X.indices = X.indices.astype('int32')\n" + "X.indptr = X.indptr.astype('int32')\n" + "obj.X = X\n" + "obj.write_h5ad(OUT, compression=COMPRESSION)\n" + ), + ), + ], ), Case( name="convert-layout", - question="Transpose X from CSR to CSC.", + question=( + "Transpose X from CSR to CSC. The output should be the size of " + "the input: 0.7.0 fixed a `--layout` that wrote it three times " + "larger." + ), + compare_size=True, contenders=[ Contender( "adata-cli (streaming)", diff --git a/benchmarks/datasets.py b/benchmarks/datasets.py index f3d9281..24568ec 100644 --- a/benchmarks/datasets.py +++ b/benchmarks/datasets.py @@ -13,13 +13,23 @@ source's compression to its output while `write_h5ad` defaults to none, which would otherwise make adata-cli's output look smaller for reasons that have nothing to do with the tool. + +**Wide variants for `recast`.** The main fixture is already float32 with +int32 indices, so narrowing it is a rewrite that changes nothing. The wide +variants hold the same matrix as float64 values and int64 indices and indptr, +which is what issue #13 was about and what a real narrowing starts from. One +stores X uncompressed, as `write_h5ad` does by default and as most float64 +stores therefore are; the other keeps lzf. The difference matters: integer +counts in float64 are mostly zero bytes, which a compressor removes anyway, +so narrowing an lzf X saves a few percent on disk where an uncompressed one +halves. Both are measured, so neither number stands alone. """ from __future__ import annotations from dataclasses import dataclass from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import numpy as np @@ -135,3 +145,52 @@ def build(directory: Path, tier: Tier, *, count: int = 2) -> List[Path]: _write(path, tier, seed) paths.append(path) return paths + + +#: The wide variants, by name: how each stores X. +WIDE_COMPRESSION: Dict[str, Optional[str]] = {"uncompressed": None, "lzf": "lzf"} + + +def _widen(src: Path, dst: Path, compression: Optional[str]) -> Path: + """Copy `src` with X stored as float64 values and int64 indices/indptr. + + Streamed a block at a time with h5py, so the large tier does not need + the whole matrix in memory twice over. Everything but X is copied as is. + """ + import h5py + + block = 1 << 24 + with h5py.File(src, "r") as source, h5py.File(dst, "w") as target: + for key, value in source.attrs.items(): + target.attrs[key] = value + for key in source: + if key != "X": + source.copy(source[key], target, name=key) + x_in = source["X"] + x_out = target.create_group("X") + for key, value in x_in.attrs.items(): + x_out.attrs[key] = value + for name, dtype in ( + ("data", "float64"), ("indices", "int64"), ("indptr", "int64") + ): + column = x_in[name] + out = x_out.create_dataset( + name, + shape=column.shape, + dtype=dtype, + chunks=column.chunks or True, + compression=compression, + ) + for start in range(0, column.shape[0], block): + stop = min(start + block, column.shape[0]) + out[start:stop] = column[start:stop].astype(dtype) + return dst + + +def build_wide(directory: Path, tier: Tier, variant: str) -> List[Path]: + """The first input of `tier`, widened. Built once and reused, like `build`.""" + (source,) = build(directory, tier, count=1) + path = directory / f"{tier.name}-0-wide-{variant}.h5ad" + if not path.exists(): + _widen(source, path, WIDE_COMPRESSION[variant]) + return [path] diff --git a/benchmarks/page_template.md b/benchmarks/page_template.md index 937b776..cd94400 100644 --- a/benchmarks/page_template.md +++ b/benchmarks/page_template.md @@ -12,11 +12,14 @@ for time, and a table that hid the cost would not be worth publishing. ## What is measured -Seventeen cases on a GitHub-hosted runner: concat (inner, outer, `--merge same`), +Eighteen cases on a GitHub-hosted runner: concat (inner, outer, `--merge same`), subset by query, h5ad→zarr (`convert`), `view`, `ls`, `create`, `export` dataframe / -array / sparse, `import dataframe`, `split`, `recast` dtype and layout, and a +array / sparse, `import dataframe`, `split`, `recast` narrowing and layout, and a peak-RSS-against-input-size sweep. Inputs are 50,000 × 20,000 at 5% density, plus the 2,000 × 36,601 shape that hung in 0.5.1. +`recast` narrowing starts from a copy of the main input stored as float64 values +with int64 indices, once with X uncompressed (as `write_h5ad` leaves it) and once +with lzf, and its tables report the file's size before and after. Every run also reports a startup floor (`--version` for each contender), because the CLI costs 0.3–1 s to import and `import scanpy` costs 3–8 s; on small inputs that is diff --git a/benchmarks/report.py b/benchmarks/report.py index c49652f..4334ab7 100644 --- a/benchmarks/report.py +++ b/benchmarks/report.py @@ -34,7 +34,15 @@ def _bytes(n: int) -> str: return f"{n} B" -def _cell(record: Dict) -> tuple: +def _size_change(record: Dict) -> str: + """`583.2 MB -> 291.1 MB (-50%)`: what the case did to the file's size.""" + before, after = record.get("input_bytes"), record.get("output_bytes") + if not before or not after: + return _bytes(after) + return f"{_bytes(before)} → {_bytes(after)} ({(after - before) / before:+.0%})" + + +def _cell(record: Dict, *, compare_size: bool = False) -> tuple: """(wall, rss, output) as they should read in the table.""" status = record.get("status") if status == "n/a": @@ -53,7 +61,9 @@ def _cell(record: Dict) -> tuple: if status == "failed": return "**failed**", _bytes(record["maxrss_bytes"]), "-" - output = _bytes(record["output_bytes"]) + output = ( + _size_change(record) if compare_size else _bytes(record["output_bytes"]) + ) if record.get("output_files", 0) > 1: # Zarr: a directory of small chunks looks far larger under `du` than # the bytes it holds, so the file count goes alongside. @@ -111,10 +121,14 @@ def render(payload: Dict, previous: Optional[Dict] = None) -> str: if case: lines.append(f"{case.question}") lines.append("") - lines.append("| Contender | Wall time | Peak RSS | Output | vs previous |") + compare_size = bool(case and case.compare_size) + output_header = "Input → output" if compare_size else "Output" + lines.append( + f"| Contender | Wall time | Peak RSS | {output_header} | vs previous |" + ) lines.append("|---|---|---|---|---|") for record in records: - wall, rss, output = _cell(record) + wall, rss, output = _cell(record, compare_size=compare_size) lines.append( f"| {record['contender']} | {wall} | {rss} | {output} | " f"{_delta(record, before.get((name, record['contender'])))} |" @@ -141,7 +155,9 @@ def render(payload: Dict, previous: Optional[Dict] = None) -> str: "several times the stored bytes and the column says more about the " "writer's allocation strategy than about the data; at `ci` and above " "it is noise. Zarr stores report a file count alongside, because a " - "directory of small chunks measures much larger than it holds." + "directory of small chunks measures much larger than it holds. " + "Where a case exists to change a file's size, the column reads " + "input → output, with the change." ) return "\n".join(lines) + "\n" diff --git a/benchmarks/run.py b/benchmarks/run.py index 290667b..93d99b4 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -32,6 +32,7 @@ from benchmarks._measure import ( DEFAULT_MEMORY_LIMIT, Measurement, + _tree_size, drop_page_cache, measure, ) @@ -219,13 +220,23 @@ def run( flush=True) main_inputs = datasets.build(fixtures, tier, count=2) var_heavy_inputs = datasets.build(fixtures, datasets.VAR_HEAVY, count=2) + wide_inputs = { + variant: datasets.build_wide(fixtures, tier, variant) + for variant in sorted({case.wide for case in selected if case.wide}) + } cache_dropped = drop_page_cache() if drop_cache else False results: List[Dict] = [] for case in selected: - pool = var_heavy_inputs if case.var_heavy else main_inputs + if case.wide: + pool = wide_inputs[case.wide] + elif case.var_heavy: + pool = var_heavy_inputs + else: + pool = main_inputs inputs = pool[: case.inputs] + input_bytes = sum(_tree_size(path)[0] for path in inputs) print(f"[case] {case.name}", flush=True) for contender in case.contenders: if contender.unsupported: @@ -270,6 +281,7 @@ def run( # The fastest run, because a slower one differs only by what # else the machine was doing. **best.as_dict(), + "input_bytes": input_bytes, "runs": len(runs), "wall_s_all": [m.wall_s for m in runs], } diff --git a/tests/test_benchmark_harness.py b/tests/test_benchmark_harness.py index 7bc384b..4f4d647 100644 --- a/tests/test_benchmark_harness.py +++ b/tests/test_benchmark_harness.py @@ -180,3 +180,51 @@ def test_publishing_keeps_the_page_prose(): assert page.count("\n# ") + page.startswith("# ") == 1, ( "the published page must have exactly one H1" ) + + +def test_the_wide_fixture_holds_the_same_matrix_in_wider_dtypes(tmp_path): + """`recast-narrow` measures nothing unless its input is actually wide. + + Until 0.7.0 the dtype case narrowed the main fixture, which is already + float32 with int32 indices: a rewrite that changed nothing. + """ + ad = pytest.importorskip("anndata") + h5py = pytest.importorskip("h5py") + from benchmarks import datasets + + tiny = datasets.Tier("tiny", 50, 40, 0.1, "test") + narrow = ad.read_h5ad(datasets.build(tmp_path, tiny, count=1)[0]) + + for variant, compression in datasets.WIDE_COMPRESSION.items(): + (path,) = datasets.build_wide(tmp_path, tiny, variant) + with h5py.File(path, "r") as handle: + assert handle["X/data"].dtype == "float64" + assert handle["X/indices"].dtype == "int64" + assert handle["X/indptr"].dtype == "int64" + assert handle["X/data"].compression == compression + + wide = ad.read_h5ad(path) + assert (wide.X != narrow.X).nnz == 0 + assert list(wide.obs_names) == list(narrow.obs_names) + assert list(wide.var.columns) == list(narrow.var.columns) + + +def test_a_size_case_reports_input_to_output(): + from benchmarks.report import render + + record = { + "case": "recast-narrow", "contender": "adata-cli", "status": "ok", + "wall_s": 1.0, "maxrss_bytes": 100_000_000, + "input_bytes": 600_000_000, "output_bytes": 300_000_000, + "output_files": 1, + } + payload = { + "ref": "test", "tier": "smoke", "tier_shape": [1, 1, 0.05], + "var_heavy_shape": [1, 1], "page_cache_dropped": True, + "memory_limit_bytes": 1, "repeats": 1, "platform": "x", + "python": "3", "versions": {}, "generated": "now", + "results": [record], + } + text = render(payload) + assert "Input → output" in text + assert "600.0 MB → 300.0 MB (-50%)" in text