Skip to content

Commit 5c5dfd0

Browse files
MoritzPotthoffQCMoritzPotthoffQC
authored
feat: Expose comparison objects upon failure in testing assertions to enable interactive debugging (#48)
1 parent 3769d7d commit 5c5dfd0

3 files changed

Lines changed: 154 additions & 49 deletions

File tree

‎diffly/testing.py‎

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,67 @@
2222
from .metrics.change import ChangeMetric, ChangeMetricFn
2323
from .metrics.data import DataMetric, DataMetricFn
2424

25+
# ------------------------------------ EXCEPTIONS ------------------------------------- #
26+
27+
28+
class ComparisonAssertionError(AssertionError):
29+
"""Base class for diffly assertion failures."""
30+
31+
32+
class FrameComparisonAssertionError(ComparisonAssertionError):
33+
"""Raised when :func:`assert_frame_equal` fails.
34+
35+
Access the underlying comparison from the exception using ``e.comparison``
36+
or in a post-mortem debugger via the ``$_exception`` convenience variable
37+
(requires Python >= 3.12)::
38+
39+
(Pdb) cmp = $_exception.comparison
40+
(Pdb) cmp.joined_unequal()
41+
42+
Attributes:
43+
comparison: The comparison between the two data frames.
44+
"""
45+
46+
def __init__(self, message: str, *, comparison: DataFrameComparison) -> None:
47+
super().__init__(message)
48+
self.comparison = comparison
49+
50+
51+
class CollectionComparisonAssertionError(ComparisonAssertionError):
52+
"""Raised when :func:`assert_collection_equal` fails.
53+
54+
Access the failing member comparisons from the exception using ``e.comparisons``
55+
or from a post-mortem debugger via the ``$_exception`` convenience variable
56+
(requires Python >= 3.12)::
57+
58+
(Pdb) cmps = $_exception.comparisons
59+
(Pdb) cmps["some_member"].joined_unequal()
60+
61+
Attributes:
62+
comparisons: A mapping from member name to comparison for the failing members.
63+
"""
64+
65+
def __init__(
66+
self, message: str, *, comparisons: dict[str, DataFrameComparison]
67+
) -> None:
68+
super().__init__(message)
69+
self.comparisons = comparisons
70+
71+
72+
# ------------------------------------ ASSERTIONS ------------------------------------- #
73+
74+
_DEBUG_HINT_FRAME = (
75+
"To debug interactively (e.g. with `pytest --pdb`), access the comparison via "
76+
"`$_exception.comparison` at the debugger prompt (requires Python >= 3.12) or via "
77+
"the `.comparison` attribute of this error."
78+
)
79+
_DEBUG_HINT_COLLECTION = (
80+
"To debug interactively (e.g. with `pytest --pdb`), access the failing member "
81+
"comparisons via `$_exception.comparisons` (a mapping from member name to "
82+
"comparison) at the debugger prompt (requires Python >= 3.12) or via the "
83+
"`.comparisons` attribute of this error."
84+
)
85+
2586

2687
def assert_collection_equal(
2788
left: dy.Collection,
@@ -100,7 +161,7 @@ def assert_collection_equal(
100161
change metrics are computed.
101162
102163
Raises:
103-
AssertionError: If the collections are not equal.
164+
CollectionComparisonAssertionError: If the collections are not equal.
104165
105166
Examples:
106167
>>> import dataframely as dy
@@ -163,7 +224,10 @@ def assert_collection_equal(
163224
),
164225
" " * 2,
165226
)
166-
raise AssertionError(f"The following members are not equal:\n\n{text}")
227+
raise CollectionComparisonAssertionError(
228+
f"The following members are not equal:\n\n{text}\n\n{_DEBUG_HINT_COLLECTION}",
229+
comparisons=failed_member_comparisons,
230+
)
167231

168232

169233
def assert_frame_equal(
@@ -251,7 +315,7 @@ def assert_frame_equal(
251315
change metrics are computed.
252316
253317
Raises:
254-
AssertionError: If the data frames are not equal.
318+
FrameComparisonAssertionError: If the data frames are not equal.
255319
256320
Note:
257321
Contrary to :meth:`polars.testing.assert_frame_equal`, the data frames ``left``
@@ -298,7 +362,10 @@ def assert_frame_equal(
298362
change_metrics=change_metrics,
299363
)
300364
text = textwrap.indent(str(summary), " " * 2)
301-
raise AssertionError(f"Data frames are not equal:\n\n{text}")
365+
raise FrameComparisonAssertionError(
366+
f"Data frames are not equal:\n\n{text}\n\n{_DEBUG_HINT_FRAME}",
367+
comparison=comparison,
368+
)
302369

303370

304371
def _get_heading(title: str) -> str:

‎tests/test_assert_collection_equal.py‎

Lines changed: 62 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@
44
import polars as pl
55
import pytest
66

7-
from diffly.testing import assert_collection_equal
7+
from diffly.comparison import DataFrameComparison
8+
from diffly.testing import (
9+
CollectionComparisonAssertionError,
10+
assert_collection_equal,
11+
)
812

913
pytest.importorskip("dataframely", reason="requires dataframely")
1014
import dataframely as dy
@@ -37,89 +41,103 @@ class Quux(dy.Collection):
3741
bar: dy.LazyFrame[Bar]
3842

3943

40-
def test_identical() -> None:
44+
@pytest.fixture
45+
def matching() -> pl.DataFrame:
46+
return pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]})
47+
48+
49+
@pytest.fixture
50+
def mismatching() -> pl.DataFrame:
51+
return pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]})
52+
53+
54+
def test_identical(matching: pl.DataFrame) -> None:
4155
qux = Qux.validate(
4256
{
43-
"foo": Foo.validate(
44-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
45-
),
46-
"bar": Bar.validate(
47-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
48-
),
57+
"foo": Foo.validate(matching, cast=True),
58+
"bar": Bar.validate(matching, cast=True),
4959
}
5060
)
5161
assert_collection_equal(qux, qux)
5262

5363

54-
def test_different_types() -> None:
64+
def test_different_types(matching: pl.DataFrame) -> None:
5565
qux = Qux.validate(
5666
{
57-
"foo": Foo.validate(
58-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
59-
),
60-
"bar": Bar.validate(
61-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
62-
),
67+
"foo": Foo.validate(matching, cast=True),
68+
"bar": Bar.validate(matching, cast=True),
6369
}
6470
)
6571
quux = Quux.validate(qux.to_dict())
6672
with pytest.raises(AssertionError, match="The collection definitions do not match"):
6773
assert_collection_equal(qux, quux)
6874

6975

70-
def test_missing_member() -> None:
76+
def test_missing_member(matching: pl.DataFrame) -> None:
7177
qux1 = Qux.validate(
7278
{
73-
"foo": Foo.validate(
74-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
75-
),
76-
"bar": Bar.validate(
77-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
78-
),
79-
"baz": Baz.validate(
80-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
81-
),
79+
"foo": Foo.validate(matching, cast=True),
80+
"bar": Bar.validate(matching, cast=True),
81+
"baz": Baz.validate(matching, cast=True),
8282
}
8383
)
8484
qux2 = Qux.validate(
8585
{
86-
"foo": Foo.validate(
87-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
88-
),
89-
"bar": Bar.validate(
90-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
91-
),
86+
"foo": Foo.validate(matching, cast=True),
87+
"bar": Bar.validate(matching, cast=True),
9288
}
9389
)
9490
with pytest.raises(AssertionError, match="The collections have different members"):
9591
assert_collection_equal(qux1, qux2)
9692

9793

98-
def test_unequal_members() -> None:
94+
def test_unequal_members(matching: pl.DataFrame, mismatching: pl.DataFrame) -> None:
9995
qux1 = Qux.validate(
10096
{
101-
"foo": Foo.validate(
102-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
103-
),
104-
"bar": Bar.validate(
105-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 3.0]}), cast=True
106-
),
97+
"foo": Foo.validate(matching, cast=True),
98+
"bar": Bar.validate(matching, cast=True),
10799
}
108100
)
109101
qux2 = Qux.validate(
110102
{
111-
"foo": Foo.validate(
112-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}), cast=True
113-
),
114-
"bar": Bar.validate(
115-
pl.DataFrame({"index": [1, 2, 3], "value": [1.0, 2.0, 4.0]}), cast=True
116-
),
103+
"foo": Foo.validate(mismatching, cast=True),
104+
"bar": Bar.validate(mismatching, cast=True),
117105
}
118106
)
119107
with pytest.raises(AssertionError, match="The following members are not equal"):
120108
assert_collection_equal(qux1, qux2)
121109

122110

111+
def test_error_exposes_comparisons(
112+
matching: pl.DataFrame, mismatching: pl.DataFrame
113+
) -> None:
114+
# Arrange
115+
qux1 = Qux.validate(
116+
{
117+
"foo": Foo.validate(matching, cast=True),
118+
"bar": Bar.validate(matching, cast=True),
119+
}
120+
)
121+
qux2 = Qux.validate(
122+
{
123+
"foo": Foo.validate(mismatching, cast=True),
124+
"bar": Bar.validate(matching, cast=True),
125+
}
126+
)
127+
128+
# Act
129+
with pytest.raises(CollectionComparisonAssertionError) as exc_info:
130+
assert_collection_equal(qux1, qux2)
131+
132+
# Assert
133+
assert isinstance(exc_info.value, CollectionComparisonAssertionError)
134+
# Only the failing member is exposed, and its comparison is usable.
135+
comparisons = exc_info.value.comparisons
136+
assert set(comparisons) == {"foo"}
137+
assert isinstance(comparisons["foo"], DataFrameComparison)
138+
assert comparisons["foo"].fraction_same("value") == pytest.approx(2 / 3)
139+
140+
123141
def test_no_primary_key() -> None:
124142
no_pk_schema = create_schema("NoPKSchema", {"a": dy.Integer(nullable=True)})
125143
collection = create_collection("Test", {"first": Foo, "second": no_pk_schema})

‎tests/test_assert_frame_equal.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77
import pytest
88

99
from diffly import compare_frames
10-
from diffly.testing import assert_frame_equal
10+
from diffly.comparison import DataFrameComparison
11+
from diffly.testing import (
12+
FrameComparisonAssertionError,
13+
assert_frame_equal,
14+
)
1115

1216

1317
def test_success_equal() -> None:
@@ -60,3 +64,19 @@ def test_success_with_nan() -> None:
6064
df = pl.DataFrame({"id": [1, 2], "value": [1.0, float("nan")]})
6165
assert_frame_equal(df, df)
6266
assert_frame_equal(df, df, primary_key="id")
67+
68+
69+
def test_error_exposes_comparison() -> None:
70+
# Arrange
71+
left = pl.DataFrame({"id": [1, 2], "value": [10.0, 20.0]})
72+
right = pl.DataFrame({"id": [1, 2], "value": [10.0, 25.0]})
73+
74+
# Act
75+
with pytest.raises(FrameComparisonAssertionError) as exc_info:
76+
assert_frame_equal(left, right, primary_key="id")
77+
78+
# Assert
79+
assert isinstance(exc_info.value, FrameComparisonAssertionError)
80+
comparison = exc_info.value.comparison
81+
assert isinstance(comparison, DataFrameComparison)
82+
assert comparison.fraction_same("value") == 0.5

0 commit comments

Comments
 (0)