Skip to content

Commit dcf3478

Browse files
MoritzPotthoffQCCopilot
MoritzPotthoffQC
and
Copilot
authored
feat: Allow specifying the floating point precision for percentage outputs (#71)
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent 48588c1 commit dcf3478

39 files changed

Lines changed: 992 additions & 20 deletions

File tree

‎diffly/cli.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,16 @@ def main(
159159
)
160160
),
161161
] = [],
162+
percentage_decimals: Annotated[
163+
int,
164+
typer.Option(
165+
min=0,
166+
help=(
167+
"The number of decimal places to show for percentages in the summary, "
168+
"e.g. column match rates."
169+
),
170+
),
171+
] = 2,
162172
) -> None:
163173
"""Compare two `parquet` files and print the comparison result."""
164174
if hidden_columns:
@@ -202,6 +212,7 @@ def main(
202212
hidden_columns=hidden_column,
203213
data_metrics=data_metrics,
204214
change_metrics=change_metrics,
215+
percentage_decimals=percentage_decimals,
205216
)
206217
if output_json:
207218
typer.echo(summary.to_json())

‎diffly/comparison.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -942,6 +942,7 @@ def summary(
942942
hidden_columns: list[str] | None = None,
943943
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
944944
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
945+
percentage_decimals: int = 2,
945946
) -> Summary:
946947
"""Generate a summary of all aspects of the comparison.
947948
@@ -995,6 +996,8 @@ def summary(
995996
(default), no metrics are computed; presets are not applied
996997
automatically. Prefer short labels — the summary has a fixed width and
997998
many or long labels degrade rendering.
999+
percentage_decimals: The number of decimal places to show for percentages
1000+
in the summary, e.g. column match rates.
9981001
9991002
Returns:
10001003
A summary which can be printed or written to a file.
@@ -1033,6 +1036,7 @@ def summary(
10331036
hidden_columns=hidden_columns,
10341037
data_metrics=resolved_data_metrics,
10351038
change_metrics=resolved_change_metrics,
1039+
percentage_decimals=percentage_decimals,
10361040
)
10371041

10381042
# ----------------------------------- UTILITIES ----------------------------------- #

‎diffly/summary.py‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,10 @@ def __init__(
6464
hidden_columns: list[str] | None,
6565
data_metrics: Mapping[str, DataMetric] | None,
6666
change_metrics: Mapping[str, ChangeMetric] | None,
67+
percentage_decimals: int,
6768
):
6869
self.slim = slim
70+
self.percentage_decimals = percentage_decimals
6971
self._data = _compute_summary_data(
7072
comparison,
7173
show_perfect_column_matches=show_perfect_column_matches,
@@ -392,9 +394,13 @@ def _section_row_counts(self) -> RenderableType:
392394
if rows.n_left > 0:
393395
fraction_rows_right = rows.n_right / rows.n_left
394396
if fraction_rows_right > 1:
395-
gain_loss = f"(+{(fraction_rows_right - 1):.2%})"
397+
gain_loss = (
398+
f"(+{(fraction_rows_right - 1):.{self.percentage_decimals}%})"
399+
)
396400
elif fraction_rows_right < 1:
397-
gain_loss = f"(-{(1 - fraction_rows_right):.2%})"
401+
gain_loss = (
402+
f"(-{(1 - fraction_rows_right):.{self.percentage_decimals}%})"
403+
)
398404
else:
399405
gain_loss = "(no change)"
400406

@@ -424,6 +430,7 @@ def _section_row_matches(self) -> RenderableType:
424430
assert rows.n_joined_unequal is not None
425431
assert rows.n_right_only is not None
426432
n_joined = rows.n_joined_equal + rows.n_joined_unequal
433+
decimals = self.percentage_decimals
427434

428435
columns: list[RenderableType] = []
429436
num_dummy_cols = 5
@@ -512,31 +519,31 @@ def _section_row_matches(self) -> RenderableType:
512519
grid.add_row(
513520
f"{rows.n_left_only:,}",
514521
f"{self._data._truncated_left_name} only",
515-
f"({_format_fraction_as_percentage(fraction_left_only)})",
522+
f"({_format_fraction_as_percentage(fraction_left_only, decimals)})",
516523
)
517524
grid.add_section()
518525
if rows.n_joined_equal > 0:
519526
fraction_equal = rows.n_joined_equal / n_joined
520527
grid.add_row(
521528
f"{rows.n_joined_equal:,}",
522529
"equal",
523-
f"({_format_fraction_as_percentage(fraction_equal)})",
530+
f"({_format_fraction_as_percentage(fraction_equal, decimals)})",
524531
)
525532
grid.add_section()
526533
if rows.n_joined_unequal > 0:
527534
fraction_unequal = rows.n_joined_unequal / n_joined
528535
grid.add_row(
529536
f"{rows.n_joined_unequal:,}",
530537
"unequal",
531-
f"({_format_fraction_as_percentage(fraction_unequal)})",
538+
f"({_format_fraction_as_percentage(fraction_unequal, decimals)})",
532539
)
533540
grid.add_section()
534541
if rows.n_right_only > 0:
535542
fraction_right_only = rows.n_right_only / rows.n_right
536543
grid.add_row(
537544
f"{rows.n_right_only:,}",
538545
f"{self._data._truncated_right_name} only",
539-
f"({_format_fraction_as_percentage(fraction_right_only)})",
546+
f"({_format_fraction_as_percentage(fraction_right_only, decimals)})",
540547
)
541548
columns.append(grid)
542549

@@ -594,10 +601,11 @@ def _section_columns(self) -> RenderableType:
594601
if has_top_changes_column:
595602
matches.add_column("Top Changes", justify="right")
596603
max_col_len = max(len(c.name) for c in columns)
604+
decimals = self.percentage_decimals
597605
for col in columns:
598606
row_items: list[RenderableType] = [
599607
Text(col.name, style="cyan"),
600-
f"{_format_fraction_as_percentage(col.match_rate)}",
608+
f"{_format_fraction_as_percentage(col.match_rate, decimals)}",
601609
]
602610
for label in self._data._change_metric_labels:
603611
value = (
@@ -1228,22 +1236,24 @@ def _format_colname(name: str) -> str:
12281236
return f"[cyan]{name}[/cyan]"
12291237

12301238

1231-
def _format_fraction_as_percentage(fraction: float) -> str:
1232-
"""Format a fraction as a percentage with two decimal places.
1239+
def _format_fraction_as_percentage(fraction: float, decimals: int) -> str:
1240+
"""Format a fraction as a percentage with the given number of decimal places.
12331241
12341242
Only shows 0%/100% if the fraction is exactly 0 or 1, respectively.
12351243
12361244
Args:
12371245
fraction: The fraction to format as a percentage. Must be in the range [0, 1].
1246+
decimals: The number of decimal places to show.
12381247
12391248
Returns:
12401249
The formatted percentage, including a percent sign.
12411250
"""
12421251
assert 0 <= fraction <= 1, "Fraction must be between 0 and 1"
12431252
percentage = fraction * 100
12441253
if percentage not in [0, 100]:
1245-
percentage = min(max(percentage, 0.01), 99.99)
1246-
return f"{percentage:.2f}%"
1254+
smallest = 10**-decimals
1255+
percentage = min(max(percentage, smallest), 100 - smallest)
1256+
return f"{percentage:.{decimals}f}%"
12471257

12481258

12491259
def _yellow(raw: Any) -> str:

‎diffly/testing.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ def assert_collection_equal(
104104
hidden_columns: list[str] | None = None,
105105
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
106106
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
107+
percentage_decimals: int = 2,
107108
) -> None:
108109
"""Assert that two :mod:`dataframely` collections are equal.
109110
@@ -159,6 +160,8 @@ def assert_collection_equal(
159160
target other column types, construct the metric explicitly with a column
160161
selector. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
161162
change metrics are computed.
163+
percentage_decimals: The number of decimal places to show for percentages in
164+
the summary, e.g. column match rates.
162165
163166
Raises:
164167
CollectionComparisonAssertionError: If the collections are not equal.
@@ -217,6 +220,7 @@ def assert_collection_equal(
217220
hidden_columns=hidden_columns,
218221
data_metrics=data_metrics,
219222
change_metrics=change_metrics,
223+
percentage_decimals=percentage_decimals,
220224
)
221225
)
222226
}"""
@@ -251,6 +255,7 @@ def assert_frame_equal(
251255
hidden_columns: list[str] | None = None,
252256
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
253257
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
258+
percentage_decimals: int = 2,
254259
) -> None:
255260
"""Assert that two :mod:`polars` data frames are equal.
256261
@@ -313,6 +318,8 @@ def assert_frame_equal(
313318
target other column types, construct the metric explicitly with a column
314319
selector. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
315320
change metrics are computed.
321+
percentage_decimals: The number of decimal places to show for percentages in
322+
the summary, e.g. column match rates.
316323
317324
Raises:
318325
FrameComparisonAssertionError: If the data frames are not equal.
@@ -360,6 +367,7 @@ def assert_frame_equal(
360367
hidden_columns=hidden_columns,
361368
data_metrics=data_metrics,
362369
change_metrics=change_metrics,
370+
percentage_decimals=percentage_decimals,
363371
)
364372
text = textwrap.indent(str(summary), " " * 2)
365373
raise FrameComparisonAssertionError(

‎tests/cli/test_cli.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,27 @@ def test_cli_hidden_columns_alias_warns(tmp_path: Path) -> None:
103103
assert result.exit_code == 0
104104

105105

106+
def test_cli_percentage_decimals(tmp_path: Path) -> None:
107+
left = pl.DataFrame({"id": [1, 2, 3], "x": [1, 2, 3]})
108+
right = pl.DataFrame({"id": [1, 2, 3], "x": [1, 2, 4]})
109+
left.write_parquet(tmp_path / "left.parquet")
110+
right.write_parquet(tmp_path / "right.parquet")
111+
112+
result = runner.invoke(
113+
app,
114+
[
115+
str(tmp_path / "left.parquet"),
116+
str(tmp_path / "right.parquet"),
117+
"--primary-key",
118+
"id",
119+
"--percentage-decimals",
120+
"3",
121+
],
122+
)
123+
assert result.exit_code == 0
124+
assert "66.667%" in result.output
125+
126+
106127
@pytest.mark.parametrize(
107128
"flag, name, expected",
108129
[

‎tests/summary/fixtures/percentage_decimals/gen/pretty_False_perfect_False_top_False_slim_False_sample_rows_False_sample_pk_False.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
2+
┃ Diffly Summary ┃
3+
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
4+
Primary key: key
5+
6+
Schemas
7+
▔▔▔▔▔▔▔
8+
Schemas match exactly (column count: 2).
9+
10+
Rows
11+
▔▔▔▔
12+
Left count Right count
13+
100,000 (+0.0030%) 100,003
14+
15+
┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╮
16+
┃ │ │ │ │ ┃ = ┃ │ │ │ │ ┃ 99,999 equal (99.9990%) │
17+
┠─┼─┼─┼─┼─┨╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌├╴ 100,000 joined
18+
┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃ 1 unequal (0.0010%) │
19+
┗━┷━┷━┷━┷━┛╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╯
20+
┃+│+│+│+│+┃ 3 right only (0.0030%)
21+
┗━┷━┷━┷━┷━┛
22+
23+
Columns
24+
▔▔▔▔▔▔▔
25+
┌───────┬──────────┐
26+
│ value │ 99.9990% │
27+
└───────┴──────────┘

‎tests/summary/fixtures/percentage_decimals/gen/pretty_False_perfect_False_top_False_slim_False_sample_rows_True_sample_pk_False.txt‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
2+
┃ Diffly Summary ┃
3+
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
4+
Primary key: key
5+
6+
Schemas
7+
▔▔▔▔▔▔▔
8+
Schemas match exactly (column count: 2).
9+
10+
Rows
11+
▔▔▔▔
12+
Left count Right count
13+
100,000 (+0.0030%) 100,003
14+
15+
┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╮
16+
┃ │ │ │ │ ┃ = ┃ │ │ │ │ ┃ 99,999 equal (99.9990%) │
17+
┠─┼─┼─┼─┼─┨╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌├╴ 100,000 joined
18+
┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃ 1 unequal (0.0010%) │
19+
┗━┷━┷━┷━┷━┛╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╯
20+
┃+│+│+│+│+┃ 3 right only (0.0030%)
21+
┗━┷━┷━┷━┷━┛
22+
23+
Columns
24+
▔▔▔▔▔▔▔
25+
┌───────┬──────────┐
26+
│ value │ 99.9990% │
27+
└───────┴──────────┘
28+
29+
Rows right only
30+
▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔
31+
┏━━━━━━━━┓
32+
┃ key ┃
33+
┡━━━━━━━━┩
34+
│ 100000 │
35+
│ 100001 │
36+
│ 100002 │
37+
└────────┘

‎tests/summary/fixtures/percentage_decimals/gen/pretty_False_perfect_False_top_False_slim_True_sample_rows_False_sample_pk_False.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
Rows
2+
▔▔▔▔
3+
Left count Right count
4+
100,000 (+0.0030%) 100,003
5+
6+
┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╮
7+
┃ │ │ │ │ ┃ = ┃ │ │ │ │ ┃ 99,999 equal (99.9990%) │
8+
┠─┼─┼─┼─┼─┨╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌├╴ 100,000 joined
9+
┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃ 1 unequal (0.0010%) │
10+
┗━┷━┷━┷━┷━┛╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╯
11+
┃+│+│+│+│+┃ 3 right only (0.0030%)
12+
┗━┷━┷━┷━┷━┛
13+
14+
Columns
15+
▔▔▔▔▔▔▔
16+
┌───────┬──────────┐
17+
│ value │ 99.9990% │
18+
└───────┴──────────┘

‎tests/summary/fixtures/percentage_decimals/gen/pretty_False_perfect_False_top_False_slim_True_sample_rows_True_sample_pk_False.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
Rows
2+
▔▔▔▔
3+
Left count Right count
4+
100,000 (+0.0030%) 100,003
5+
6+
┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╮
7+
┃ │ │ │ │ ┃ = ┃ │ │ │ │ ┃ 99,999 equal (99.9990%) │
8+
┠─┼─┼─┼─┼─┨╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌├╴ 100,000 joined
9+
┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃ 1 unequal (0.0010%) │
10+
┗━┷━┷━┷━┷━┛╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╯
11+
┃+│+│+│+│+┃ 3 right only (0.0030%)
12+
┗━┷━┷━┷━┷━┛
13+
14+
Columns
15+
▔▔▔▔▔▔▔
16+
┌───────┬──────────┐
17+
│ value │ 99.9990% │
18+
└───────┴──────────┘
19+
20+
Rows right only
21+
▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔
22+
┏━━━━━━━━┓
23+
┃ key ┃
24+
┡━━━━━━━━┩
25+
│ 100000 │
26+
│ 100001 │
27+
│ 100002 │
28+
└────────┘

‎tests/summary/fixtures/percentage_decimals/gen/pretty_False_perfect_False_top_True_slim_False_sample_rows_False_sample_pk_False.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
2+
┃ Diffly Summary ┃
3+
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
4+
Primary key: key
5+
6+
Schemas
7+
▔▔▔▔▔▔▔
8+
Schemas match exactly (column count: 2).
9+
10+
Rows
11+
▔▔▔▔
12+
Left count Right count
13+
100,000 (+0.0030%) 100,003
14+
15+
┏━┯━┯━┯━┯━┓╌╌╌┏━┯━┯━┯━┯━┓╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╮
16+
┃ │ │ │ │ ┃ = ┃ │ │ │ │ ┃ 99,999 equal (99.9990%) │
17+
┠─┼─┼─┼─┼─┨╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌├╴ 100,000 joined
18+
┃ │ │ │ │ ┃ ≠ ┃ │ │ │ │ ┃ 1 unequal (0.0010%) │
19+
┗━┷━┷━┷━┷━┛╌╌╌┠─┼─┼─┼─┼─┨╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╯
20+
┃+│+│+│+│+┃ 3 right only (0.0030%)
21+
┗━┷━┷━┷━┷━┛
22+
23+
Columns
24+
▔▔▔▔▔▔▔
25+
┌───────┬──────────┬─────────────┐
26+
│ value │ 99.9990% │ 0 -> 1 (1x) │
27+
└───────┴──────────┴─────────────┘

0 commit comments

Comments
 (0)