benchmarks: narrow a store worth narrowing, and report sizes

benchmarks/cases.py

@@ -83,6 +83,12 @@ class Case:
     output_suffix: str = ".h5ad"
     #: Use the var-heavy shape rather than the tier's main shape.
     var_heavy: bool = False
+    #: Use a wide variant of the main input -- float64 values, int64 indices
+    #: -- named for how it stores X: "uncompressed" or "lzf".
+    wide: Optional[str] = None
+    #: Report the output's size against the input's, as before -> after. For
+    #: cases whose point is what they do to the file's size.
+    compare_size: bool = False
     #: An auxiliary input the case needs, e.g. the CSV an import reads.
     #: Built once from the first input and offered to CLI contenders as
     #: `{sidecar_csv}` and to scripts as $BENCH_SIDECAR_CSV.
@@ -483,34 +489,89 @@ def _py(body: str) -> str:
         ],
     ),
     # -- recast (named convert until 0.7.0; the case names keep the history) --
+    #
+    # `convert-dtype` ran until 0.7.0 on the main fixture, which is already
+    # float32 with int32 indices: it measured a rewrite that changed nothing,
+    # and its output was the size of its input. `recast-narrow` replaces it,
+    # on an input worth narrowing, and reports the size it saves.
     Case(
-        name="convert-dtype",
-        question="Rewrite X as float32.",
+        name="recast-narrow",
+        question=(
+            "Narrow X from float64 values with int64 indices to float32 and "
+            "int32, as issue #13 asked. X is uncompressed, as `write_h5ad` "
+            "leaves it by default, so the output keeps that and the baseline "
+            "writes it the same way."
+        ),
+        wide="uncompressed",
+        compare_size=True,
         contenders=[
             Contender(
                 "adata-cli",
                 argv=[
                     "adata", "recast", "{input0}", "X", "-o", "{output}",
-                    "--dtype", "float32", "--force",
+                    "--dtype", "float32", "--indices-dtype", "int32",
                 ],
             ),
             Contender(
                 "anndata (in memory)",
+                # compression=None, not COMPRESSION: the input's X is
+                # uncompressed and adata-cli keeps it that way, so lzf here
+                # would credit anndata with a saving the cast did not make.
                 script=_py(
                     "obj = ad.read_h5ad(IN[0])\n"
-                    "obj.X = obj.X.astype('float32')\n"
-                    "obj.write_h5ad(OUT, compression=COMPRESSION)\n"
+                    "X = obj.X.astype('float32')\n"
+                    "X.indices = X.indices.astype('int32')\n"
+                    "X.indptr = X.indptr.astype('int32')\n"
+                    "obj.X = X\n"
+                    "obj.write_h5ad(OUT, compression=None)\n"
                 ),
             ),
             Contender(
                 "scanpy",
                 unsupported="no dtype rewrite; scanpy defers to anndata",
             ),
         ],
+        tags=["headline"],
+    ),
+    Case(
+        name="recast-narrow-lzf",
+        question=(
+            "The same narrowing on an lzf-compressed X. Integer counts in "
+            "float64 are mostly zero bytes, which lzf already removes, so "
+            "expect a small saving on disk; the halving is in memory, once "
+            "the matrix is loaded."
+        ),
+        wide="lzf",
+        compare_size=True,
+        contenders=[
+            Contender(
+                "adata-cli",
+                argv=[
+                    "adata", "recast", "{input0}", "X", "-o", "{output}",
+                    "--dtype", "float32", "--indices-dtype", "int32",
+                ],
+            ),
+            Contender(
+                "anndata (in memory)",
+                script=_py(
+                    "obj = ad.read_h5ad(IN[0])\n"
+                    "X = obj.X.astype('float32')\n"
+                    "X.indices = X.indices.astype('int32')\n"
+                    "X.indptr = X.indptr.astype('int32')\n"
+                    "obj.X = X\n"
+                    "obj.write_h5ad(OUT, compression=COMPRESSION)\n"
+                ),
+            ),
+        ],
     ),
     Case(
         name="convert-layout",
-        question="Transpose X from CSR to CSC.",
+        question=(
+            "Transpose X from CSR to CSC. The output should be the size of "
+            "the input: 0.7.0 fixed a `--layout` that wrote it three times "
+            "larger."
+        ),
+        compare_size=True,
         contenders=[
             Contender(
                 "adata-cli (streaming)",

benchmarks/datasets.py

@@ -13,13 +13,23 @@
 source's compression to its output while `write_h5ad` defaults to none, which
 would otherwise make adata-cli's output look smaller for reasons that have
 nothing to do with the tool.
+
+**Wide variants for `recast`.** The main fixture is already float32 with
+int32 indices, so narrowing it is a rewrite that changes nothing. The wide
+variants hold the same matrix as float64 values and int64 indices and indptr,
+which is what issue #13 was about and what a real narrowing starts from. One
+stores X uncompressed, as `write_h5ad` does by default and as most float64
+stores therefore are; the other keeps lzf. The difference matters: integer
+counts in float64 are mostly zero bytes, which a compressor removes anyway,
+so narrowing an lzf X saves a few percent on disk where an uncompressed one
+halves. Both are measured, so neither number stands alone.
 """
 
 from __future__ import annotations
 
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import numpy as np
 
@@ -135,3 +145,52 @@ def build(directory: Path, tier: Tier, *, count: int = 2) -> List[Path]:
             _write(path, tier, seed)
         paths.append(path)
     return paths
+
+
+#: The wide variants, by name: how each stores X.
+WIDE_COMPRESSION: Dict[str, Optional[str]] = {"uncompressed": None, "lzf": "lzf"}
+
+
+def _widen(src: Path, dst: Path, compression: Optional[str]) -> Path:
+    """Copy `src` with X stored as float64 values and int64 indices/indptr.
+
+    Streamed a block at a time with h5py, so the large tier does not need
+    the whole matrix in memory twice over. Everything but X is copied as is.
+    """
+    import h5py
+
+    block = 1 << 24
+    with h5py.File(src, "r") as source, h5py.File(dst, "w") as target:
+        for key, value in source.attrs.items():
+            target.attrs[key] = value
+        for key in source:
+            if key != "X":
+                source.copy(source[key], target, name=key)
+        x_in = source["X"]
+        x_out = target.create_group("X")
+        for key, value in x_in.attrs.items():
+            x_out.attrs[key] = value
+        for name, dtype in (
+            ("data", "float64"), ("indices", "int64"), ("indptr", "int64")
+        ):
+            column = x_in[name]
+            out = x_out.create_dataset(
+                name,
+                shape=column.shape,
+                dtype=dtype,
+                chunks=column.chunks or True,
+                compression=compression,
+            )
+            for start in range(0, column.shape[0], block):
+                stop = min(start + block, column.shape[0])
+                out[start:stop] = column[start:stop].astype(dtype)
+    return dst
+
+
+def build_wide(directory: Path, tier: Tier, variant: str) -> List[Path]:
+    """The first input of `tier`, widened. Built once and reused, like `build`."""
+    (source,) = build(directory, tier, count=1)
+    path = directory / f"{tier.name}-0-wide-{variant}.h5ad"
+    if not path.exists():
+        _widen(source, path, WIDE_COMPRESSION[variant])
+    return [path]

benchmarks/page_template.md

@@ -12,11 +12,14 @@ for time, and a table that hid the cost would not be worth publishing.
 
 ## What is measured
 
-Seventeen cases on a GitHub-hosted runner: concat (inner, outer, `--merge same`),
+Eighteen cases on a GitHub-hosted runner: concat (inner, outer, `--merge same`),
 subset by query, h5ad→zarr (`convert`), `view`, `ls`, `create`, `export` dataframe /
-array / sparse, `import dataframe`, `split`, `recast` dtype and layout, and a
+array / sparse, `import dataframe`, `split`, `recast` narrowing and layout, and a
 peak-RSS-against-input-size sweep. Inputs
 are 50,000 × 20,000 at 5% density, plus the 2,000 × 36,601 shape that hung in 0.5.1.
+`recast` narrowing starts from a copy of the main input stored as float64 values
+with int64 indices, once with X uncompressed (as `write_h5ad` leaves it) and once
+with lzf, and its tables report the file's size before and after.
 
 Every run also reports a startup floor (`--version` for each contender), because the
 CLI costs 0.3–1 s to import and `import scanpy` costs 3–8 s; on small inputs that is

benchmarks/report.py

@@ -34,7 +34,15 @@ def _bytes(n: int) -> str:
     return f"{n} B"
 
 
-def _cell(record: Dict) -> tuple:
+def _size_change(record: Dict) -> str:
+    """`583.2 MB -> 291.1 MB (-50%)`: what the case did to the file's size."""
+    before, after = record.get("input_bytes"), record.get("output_bytes")
+    if not before or not after:
+        return _bytes(after)
+    return f"{_bytes(before)} → {_bytes(after)} ({(after - before) / before:+.0%})"
+
+
+def _cell(record: Dict, *, compare_size: bool = False) -> tuple:
     """(wall, rss, output) as they should read in the table."""
     status = record.get("status")
     if status == "n/a":
@@ -53,7 +61,9 @@ def _cell(record: Dict) -> tuple:
     if status == "failed":
         return "**failed**", _bytes(record["maxrss_bytes"]), "-"
 
-    output = _bytes(record["output_bytes"])
+    output = (
+        _size_change(record) if compare_size else _bytes(record["output_bytes"])
+    )
     if record.get("output_files", 0) > 1:
         # Zarr: a directory of small chunks looks far larger under `du` than
         # the bytes it holds, so the file count goes alongside.
@@ -111,10 +121,14 @@ def render(payload: Dict, previous: Optional[Dict] = None) -> str:
         if case:
             lines.append(f"{case.question}")
             lines.append("")
-        lines.append("| Contender | Wall time | Peak RSS | Output | vs previous |")
+        compare_size = bool(case and case.compare_size)
+        output_header = "Input → output" if compare_size else "Output"
+        lines.append(
+            f"| Contender | Wall time | Peak RSS | {output_header} | vs previous |"
+        )
         lines.append("|---|---|---|---|---|")
         for record in records:
-            wall, rss, output = _cell(record)
+            wall, rss, output = _cell(record, compare_size=compare_size)
             lines.append(
                 f"| {record['contender']} | {wall} | {rss} | {output} | "
                 f"{_delta(record, before.get((name, record['contender'])))} |"
@@ -141,7 +155,9 @@ def render(payload: Dict, previous: Optional[Dict] = None) -> str:
         "several times the stored bytes and the column says more about the "
         "writer's allocation strategy than about the data; at `ci` and above "
         "it is noise. Zarr stores report a file count alongside, because a "
-        "directory of small chunks measures much larger than it holds."
+        "directory of small chunks measures much larger than it holds. "
+        "Where a case exists to change a file's size, the column reads "
+        "input → output, with the change."
     )
     return "\n".join(lines) + "\n"
 

benchmarks/run.py

@@ -32,6 +32,7 @@
 from benchmarks._measure import (
     DEFAULT_MEMORY_LIMIT,
     Measurement,
+    _tree_size,
     drop_page_cache,
     measure,
 )
@@ -219,13 +220,23 @@ def run(
           flush=True)
     main_inputs = datasets.build(fixtures, tier, count=2)
     var_heavy_inputs = datasets.build(fixtures, datasets.VAR_HEAVY, count=2)
+    wide_inputs = {
+        variant: datasets.build_wide(fixtures, tier, variant)
+        for variant in sorted({case.wide for case in selected if case.wide})
+    }
 
     cache_dropped = drop_page_cache() if drop_cache else False
 
     results: List[Dict] = []
     for case in selected:
-        pool = var_heavy_inputs if case.var_heavy else main_inputs
+        if case.wide:
+            pool = wide_inputs[case.wide]
+        elif case.var_heavy:
+            pool = var_heavy_inputs
+        else:
+            pool = main_inputs
         inputs = pool[: case.inputs]
+        input_bytes = sum(_tree_size(path)[0] for path in inputs)
         print(f"[case] {case.name}", flush=True)
         for contender in case.contenders:
             if contender.unsupported:
@@ -270,6 +281,7 @@ def run(
                 # The fastest run, because a slower one differs only by what
                 # else the machine was doing.
                 **best.as_dict(),
+                "input_bytes": input_bytes,
                 "runs": len(runs),
                 "wall_s_all": [m.wall_s for m in runs],
             }

tests/test_benchmark_harness.py

@@ -180,3 +180,51 @@ def test_publishing_keeps_the_page_prose():
     assert page.count("\n# ") + page.startswith("# ") == 1, (
         "the published page must have exactly one H1"
     )
+
+
+def test_the_wide_fixture_holds_the_same_matrix_in_wider_dtypes(tmp_path):
+    """`recast-narrow` measures nothing unless its input is actually wide.
+
+    Until 0.7.0 the dtype case narrowed the main fixture, which is already
+    float32 with int32 indices: a rewrite that changed nothing.
+    """
+    ad = pytest.importorskip("anndata")
+    h5py = pytest.importorskip("h5py")
+    from benchmarks import datasets
+
+    tiny = datasets.Tier("tiny", 50, 40, 0.1, "test")
+    narrow = ad.read_h5ad(datasets.build(tmp_path, tiny, count=1)[0])
+
+    for variant, compression in datasets.WIDE_COMPRESSION.items():
+        (path,) = datasets.build_wide(tmp_path, tiny, variant)
+        with h5py.File(path, "r") as handle:
+            assert handle["X/data"].dtype == "float64"
+            assert handle["X/indices"].dtype == "int64"
+            assert handle["X/indptr"].dtype == "int64"
+            assert handle["X/data"].compression == compression
+
+        wide = ad.read_h5ad(path)
+        assert (wide.X != narrow.X).nnz == 0
+        assert list(wide.obs_names) == list(narrow.obs_names)
+        assert list(wide.var.columns) == list(narrow.var.columns)
+
+
+def test_a_size_case_reports_input_to_output():
+    from benchmarks.report import render
+
+    record = {
+        "case": "recast-narrow", "contender": "adata-cli", "status": "ok",
+        "wall_s": 1.0, "maxrss_bytes": 100_000_000,
+        "input_bytes": 600_000_000, "output_bytes": 300_000_000,
+        "output_files": 1,
+    }
+    payload = {
+        "ref": "test", "tier": "smoke", "tier_shape": [1, 1, 0.05],
+        "var_heavy_shape": [1, 1], "page_cache_dropped": True,
+        "memory_limit_bytes": 1, "repeats": 1, "platform": "x",
+        "python": "3", "versions": {}, "generated": "now",
+        "results": [record],
+    }
+    text = render(payload)
+    assert "Input → output" in text
+    assert "600.0 MB → 300.0 MB (-50%)" in text