feat: Generalize metrics and add nullability delta metric (#43)

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: EgeKaraismailogluQC <128645043+EgeKaraismailogluQC@users.noreply.github.com>
Co-authored-by: quant-ranger[bot] <132915763+quant-ranger[bot]@users.noreply.github.com>
Co-authored-by: Marius Merkle <marius.merkle@quantco.com>
Co-authored-by: Marius Merkle <122545105+MariusMerkleQC@users.noreply.github.com>

Author: 5 people

diffly/cli.py

@@ -12,10 +12,18 @@
 
 from ._compat import typer
 from ._utils import ABS_TOL_DEFAULT, ABS_TOL_TEMPORAL_DEFAULT, REL_TOL_DEFAULT
-from .metrics import DEFAULT_METRICS
+from .metrics import Metric, MetricFn
+from .metrics.change import DEFAULT_CHANGE_METRICS
+from .metrics.data import DEFAULT_DATA_METRICS
 
 app = typer.Typer()
 
+#: All metric presets selectable via ``--metric``, combining the change and data sets.
+AVAILABLE_METRICS: dict[str, MetricFn | Metric] = {
+    **DEFAULT_CHANGE_METRICS,
+    **DEFAULT_DATA_METRICS,
+}
+
 
 @app.command()
 def main(
@@ -139,8 +147,8 @@ def main(
         list[str],
         typer.Option(
             help=(
-                "Metric presets to display per numerical column. Repeatable. "
-                f"Available: {', '.join(DEFAULT_METRICS)}."
+                "Metric presets to display per column. Repeatable. "
+                f"Available: {', '.join(AVAILABLE_METRICS)}."
             )
         ),
     ] = [],
@@ -154,11 +162,11 @@ def main(
         hidden_column = [*hidden_column, *hidden_columns]
 
     for name in metric:
-        if name not in DEFAULT_METRICS:
+        if name not in AVAILABLE_METRICS:
             raise typer.BadParameter(
-                f"Unknown metric: {name!r}. Available: {', '.join(DEFAULT_METRICS)}."
+                f"Unknown metric: {name!r}. Available: {', '.join(AVAILABLE_METRICS)}."
             )
-    metrics = {name: DEFAULT_METRICS[name] for name in metric}
+    metrics = {name: AVAILABLE_METRICS[name] for name in metric}
 
     comparison = compare_frames(
         pl.scan_parquet(left),

diffly/comparison.py

@@ -25,7 +25,7 @@
     lazy_len,
     make_and_validate_mapping,
 )
-from .metrics import MetricFn, _make_numeric_metric
+from .metrics import Metric, MetricFn, _make_numeric_metric
 
 if TYPE_CHECKING:  # pragma: no cover
     # NOTE: We cannot import at runtime as we're otherwise running into circular
@@ -920,7 +920,7 @@ def summary(
         right_name: str = Side.RIGHT,
         slim: bool = False,
         hidden_columns: list[str] | None = None,
-        metrics: Mapping[str, MetricFn] | None = None,
+        metrics: Mapping[str, MetricFn | Metric] | None = None,
     ) -> Summary:
         """Generate a summary of all aspects of the comparison.
 
@@ -950,16 +950,18 @@ def summary(
                 advanced users who are familiar with the summary format.
             hidden_columns: Columns for which no values are printed, e.g. because they
                 contain sensitive information.
-            metrics: Optional mapping from display label to a metric callable
-                ``(left_expr, right_expr) -> pl.Expr``. Each callable receives two
+            metrics: Optional mapping from display label to a metric. A value may be a
+                callable ``(left_expr, right_expr) -> pl.Expr`` or a
+                :class:`~diffly.metrics.Metric`. Each callable receives two
                 :class:`polars.Expr` referring to the left and right values of a single
-                numerical column across all joined rows, and must return a scalar
-                aggregation expression. See :doc:`/api/metrics` for the full list of
-                presets and the :data:`~diffly.metrics.MetricFn` type. When ``None``
-                (default), no metrics are computed; presets are not applied
-                automatically. Metrics are only computed for numerical columns. Prefer
-                short labels — the summary has a fixed width and many or long labels
-                degrade rendering.
+                column across all joined rows, and must return a scalar aggregation
+                expression. Bare callables are only computed for numerical columns; wrap
+                one in a :class:`~diffly.metrics.Metric` with a column selector to target
+                other column types (e.g. ``Metric(fn, selector=cs.all())``).
+                See :doc:`/api/metrics` for the full list of presets and the
+                :data:`~diffly.metrics.MetricFn` type. When ``None`` (default), no metrics
+                are computed; presets are not applied automatically. Prefer short labels —
+                the summary has a fixed width and many or long labels degrade rendering.
 
         Returns:
             A summary which can be printed or written to a file.
@@ -976,7 +978,10 @@ def summary(
         from .summary import Summary
 
         resolved_metrics = (
-            {label: _make_numeric_metric(fn) for label, fn in metrics.items()}
+            {
+                label: v if isinstance(v, Metric) else _make_numeric_metric(v)
+                for label, v in metrics.items()
+            }
             if metrics is not None
             else None
         )

diffly/metrics/__init__.py

@@ -0,0 +1,50 @@
+# Copyright (c) QuantCo 2025-2026
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Metrics computed per column when generating a summary.
+
+Two families are provided:
+
+- Metrics in :mod:`~diffly.metrics.change` describe the change between numeric
+  columns itself by aggregating over ``right - left``.
+- Metrics in :mod:`~diffly.metrics.data` describe the left and right datasets
+  individually, explaining how a change affects the data.
+"""
+
+from __future__ import annotations
+
+from . import change, data
+from ._common import Metric, MetricFn
+from .change import (
+    _make_numeric_metric,
+    max,
+    mean,
+    mean_absolute_deviation,
+    mean_relative_deviation,
+    median,
+    min,
+    quantile,
+    std,
+)
+
+DEFAULT_METRICS: dict[str, MetricFn | Metric] = {
+    **change.DEFAULT_CHANGE_METRICS,
+}
+"""The default preset metrics, consisting of the change default set."""
+
+__all__ = [
+    "DEFAULT_METRICS",
+    "Metric",
+    "MetricFn",
+    "change",
+    "data",
+    "max",
+    "mean",
+    "mean_absolute_deviation",
+    "mean_relative_deviation",
+    "median",
+    "min",
+    "quantile",
+    "std",
+    "_make_numeric_metric",
+]

diffly/metrics/_common.py

@@ -0,0 +1,27 @@
+# Copyright (c) QuantCo 2025-2026
+# SPDX-License-Identifier: BSD-3-Clause
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import polars as pl
+import polars.selectors as cs
+
+
+@dataclass(frozen=True)
+class Metric:
+    """A metric function paired with a column-applicability selector."""
+
+    fn: MetricFn
+    selector: cs.Selector
+
+
+MetricFn = Callable[[pl.Expr, pl.Expr], pl.Expr]
+"""A metric function maps ``(left_expr, right_expr)`` to a scalar aggregation
+expression.
+
+The expressions refer to the left-side and right-side values of a single column across
+all joined rows.
+"""

diffly/metrics.py diffly/metrics/change.pydiffly/metrics.py renamed to diffly/metrics/change.py

@@ -1,33 +1,17 @@
 # Copyright (c) QuantCo 2025-2026
 # SPDX-License-Identifier: BSD-3-Clause
 
-from __future__ import annotations
+"""Metrics describing the change between numeric columns.
+
+These aggregate over ``right - left`` to characterize the change itself.
+"""
 
-from collections.abc import Callable
-from dataclasses import dataclass
+from __future__ import annotations
 
 import polars as pl
 import polars.selectors as cs
 
-
-@dataclass(frozen=True)
-class Metric:
-    """A metric function paired with a column-applicability selector.
-
-    Internal only.
-    """
-
-    fn: MetricFn
-    selector: cs.Selector
-
-
-MetricFn = Callable[[pl.Expr, pl.Expr], pl.Expr]
-"""A metric function maps ``(left_expr, right_expr)`` to a scalar aggregation
-expression.
-
-The expressions refer to the left-side and right-side values of a single column across
-all joined rows.
-"""
+from ._common import Metric, MetricFn
 
 
 def _make_numeric_metric(fn: MetricFn) -> Metric:
@@ -82,7 +66,7 @@ def _quantile(left: pl.Expr, right: pl.Expr) -> pl.Expr:
     return _quantile
 
 
-DEFAULT_METRICS: dict[str, MetricFn] = {
+DEFAULT_CHANGE_METRICS: dict[str, MetricFn] = {
     "Mean": mean,
     "Median": median,
     "Min": min,
@@ -91,3 +75,4 @@ def _quantile(left: pl.Expr, right: pl.Expr) -> pl.Expr:
     "Mean absolute deviation": mean_absolute_deviation,
     "Mean relative deviation": mean_relative_deviation,
 }
+"""Preset metrics describing the change between numeric columns."""

diffly/metrics/data.py

@@ -0,0 +1,75 @@
+# Copyright (c) QuantCo 2025-2026
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Metrics describing the left and right datasets individually.
+
+These characterize each side of a change so you can understand how the change affects
+the data, rather than describing the change itself.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import polars as pl
+import polars.selectors as cs
+
+from ._common import Metric, MetricFn
+
+
+def null_fraction_change(left: pl.Expr, right: pl.Expr) -> pl.Expr:
+    """Change in the fraction of null entries, rendered as ``<old> -> <new> (<delta>)``.
+
+    ``old`` and ``new`` are the null percentages of the left and right side, and
+    ``delta`` is their signed difference (``+`` when the right side has proportionally
+    more nulls, ``-`` when it has fewer). This metric function can be applied to columns
+    of any type.
+    """
+    return _render_change(
+        left.is_null().mean(),
+        right.is_null().mean(),
+        lambda value, signed: _percentage_string(
+            value, signed=signed, percent_sign=not signed
+        ),
+    )
+
+
+DEFAULT_DATA_METRICS: dict[str, MetricFn | Metric] = {
+    "Null%": Metric(fn=null_fraction_change, selector=cs.all()),
+}
+"""Preset metrics describing the left and right datasets individually."""
+
+
+# ------------------------------------------------------------------------------------ #
+#                                    UTILITY METHODS                                   #
+# ------------------------------------------------------------------------------------ #
+
+
+def _percentage_string(
+    fraction: pl.Expr, *, signed: bool = False, percent_sign: bool = True
+) -> pl.Expr:
+    """Format a fraction as a percentage string, optionally with an explicit sign."""
+    pct = (fraction * 100).round(2)
+    body = pl.format("{}%", pct) if percent_sign else pl.format("{}", pct)
+    if signed:
+        return pl.when(pct >= 0).then(pl.format("+{}", body)).otherwise(body)
+    return body
+
+
+def _render_change(
+    old: pl.Expr,
+    new: pl.Expr,
+    format_value: Callable[[pl.Expr, bool], pl.Expr],
+) -> pl.Expr:
+    """Render a change as ``<old> -> <new> (<delta>)``.
+
+    ``format_value(value, signed)`` formats a value for display; ``old`` and ``new`` are
+    rendered unsigned and the delta ``new - old`` is rendered signed (with an explicit
+    ``+`` prefix for positive values).
+    """
+    return pl.format(
+        "{} -> {} ({})",
+        format_value(old, False),
+        format_value(new, False),
+        format_value(new - old, True),
+    )

diffly/summary.py

@@ -1131,10 +1131,13 @@ def _format_value(value: Any, *, float_format: str | None = None) -> str:
 def _format_metric_value(value: Any) -> str:
     """Format a metric value for the column summary.
 
-    Blanks out ``None`` and renders floats with ``.4g`` precision.
+    Blanks out ``None``, renders string values verbatim, and renders floats with ``.4g``
+    precision.
     """
     if value is None:
         return ""
+    if isinstance(value, str):
+        return _yellow(value)
     return _format_value(value, float_format=".4g")
 
 

diffly/testing.py

@@ -19,7 +19,7 @@
 
 from ._compat import dy
 from .comparison import DataFrameComparison, compare_frames
-from .metrics import MetricFn
+from .metrics import Metric, MetricFn
 
 
 def assert_collection_equal(
@@ -40,7 +40,7 @@ def assert_collection_equal(
     right_name: str = Side.RIGHT,
     slim: bool = False,
     hidden_columns: list[str] | None = None,
-    metrics: Mapping[str, MetricFn] | None = None,
+    metrics: Mapping[str, MetricFn | Metric] | None = None,
 ) -> None:
     """Assert that two :mod:`dataframely` collections are equal.
 
@@ -85,9 +85,11 @@ def assert_collection_equal(
         hidden_columns: Columns for which no values are printed, e.g. because they
             contain sensitive information.
         metrics: Optional mapping from display label to a metric callable
-            ``(left_expr, right_expr) -> pl.Expr``. See :mod:`diffly.metrics` for
-            presets. When ``None`` (default), no metrics are computed; presets are
-            not applied automatically.
+            ``(left_expr, right_expr) -> pl.Expr`` or a :class:`~diffly.metrics.Metric`.
+            Bare callables are only computed for numerical columns; wrap one in a
+            :class:`~diffly.metrics.Metric` with a column selector to target other column
+            types. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
+            metrics are computed; presets are not applied automatically.
 
     Raises:
         AssertionError: If the collections are not equal.
@@ -174,7 +176,7 @@ def assert_frame_equal(
     right_name: str = Side.RIGHT,
     slim: bool = False,
     hidden_columns: list[str] | None = None,
-    metrics: Mapping[str, MetricFn] | None = None,
+    metrics: Mapping[str, MetricFn | Metric] | None = None,
 ) -> None:
     """Assert that two :mod:`polars` data frames are equal.
 
@@ -226,9 +228,11 @@ def assert_frame_equal(
         hidden_columns: Columns for which no values are printed, e.g. because they
             contain sensitive information.
         metrics: Optional mapping from display label to a metric callable
-            ``(left_expr, right_expr) -> pl.Expr``. See :mod:`diffly.metrics` for
-            presets. When ``None`` (default), no metrics are computed; presets are
-            not applied automatically.
+            ``(left_expr, right_expr) -> pl.Expr`` or a :class:`~diffly.metrics.Metric`.
+            Bare callables are only computed for numerical columns; wrap one in a
+            :class:`~diffly.metrics.Metric` with a column selector to target other column
+            types. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
+            metrics are computed; presets are not applied automatically.
 
     Raises:
         AssertionError: If the data frames are not equal.