Skip to content

Commit 2712044

Browse files
vahid-ahmadiclaude
andcommitted
Preserve weights through pickling and deepcopy
weights was a plain instance attribute, so it was absent from the pickle state: pickle.loads(pickle.dumps(MicroSeries([1, 2], weights=[3, 4]))).sum() # AttributeError: 'MicroSeries' object has no attribute 'weights' Declaring _metadata = ["weights"] fixes it: pandas includes _metadata attributes in the pickle state. MicroDataFrame needed one more step. Its weighted aggregations are installed as per-instance closures by override_df_functions, which only runs in __init__ — a path unpickling skips. So an unpickled frame kept its weights but mdf.sum() fell through to the unweighted pandas implementation and returned 6 instead of 14, with no error. __setstate__ now reinstalls them. This covers the pickle half of #300; the _constructor rework that would stop sort_values/fillna/sample from dropping weights is still open there. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 46cffbd commit 2712044

4 files changed

Lines changed: 66 additions & 0 deletions

File tree

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
- bump: patch
2+
changes:
3+
fixed:
4+
- Weights now survive pickling, to_pickle/read_pickle and copy.deepcopy;
5+
previously they vanished, and an unpickled MicroDataFrame silently
6+
returned unweighted aggregations.

microdf/microdataframe.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,12 @@ def __getattr__(self, name):
9595

9696

9797
class MicroDataFrame(pd.DataFrame):
98+
# Declare ``weights`` as pandas metadata. pandas includes
99+
# _metadata attributes in the pickle state, so weights now survive
100+
# pickling, to_pickle/read_pickle and copy.deepcopy instead of
101+
# vanishing and leaving an AttributeError on the next aggregation.
102+
_metadata = ["weights"]
103+
98104
def __init__(self, *args, weights=None, **kwargs):
99105
"""A DataFrame-inheriting class for weighted microdata.
100106
@@ -110,6 +116,20 @@ def __init__(self, *args, weights=None, **kwargs):
110116
self._link_all_weights()
111117
self.override_df_functions()
112118

119+
def __setstate__(self, state) -> None:
120+
"""Restore a pickled MicroDataFrame.
121+
122+
The weighted aggregations are installed as per-instance closures by
123+
``override_df_functions``, which only runs in ``__init__`` — a path
124+
unpickling skips. Without reinstalling them, ``mdf.sum()`` on an
125+
unpickled frame silently fell through to the unweighted pandas
126+
implementation.
127+
"""
128+
super().__setstate__(state)
129+
if getattr(self, "weights", None) is None:
130+
self._link_all_weights()
131+
self.override_df_functions()
132+
113133
@property
114134
def loc(self) -> _MicroLocIndexer:
115135
"""Label-based indexer that preserves MicroDataFrame type and weights.

microdf/microseries.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,12 @@ def _weighted_top_share(
5050

5151

5252
class MicroSeries(pd.Series):
53+
# Declare ``weights`` as pandas metadata. pandas includes
54+
# _metadata attributes in the pickle state, so weights now survive
55+
# pickling, to_pickle/read_pickle and copy.deepcopy instead of
56+
# vanishing and leaving an AttributeError on the next aggregation.
57+
_metadata = ["weights"]
58+
5359
def __init__(self, *args, weights: np.array = None, **kwargs):
5460
"""A Series-inheriting class for weighted microdata.
5561

microdf/tests/test_microseries_dataframe.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import copy
12
import warnings
23

34
import numpy as np
@@ -815,3 +816,36 @@ def test_rank_ties_share_bucket() -> None:
815816
# existing ``test_rank`` expectations hold.
816817
s = mdf.MicroSeries([1, 2, 3], weights=[4, 5, 6])
817818
np.testing.assert_array_equal(s.rank().values, [4, 9, 15])
819+
820+
821+
def test_microseries_survives_pickling():
822+
"""Weights must survive a pickle round-trip."""
823+
import pickle
824+
825+
s = mdf.MicroSeries([1, 2, 3], index=[7, 8, 9], weights=[1, 2, 3])
826+
restored = pickle.loads(pickle.dumps(s))
827+
assert isinstance(restored, mdf.MicroSeries)
828+
assert restored.sum() == 14
829+
assert list(restored.weights) == [1.0, 2.0, 3.0]
830+
831+
832+
def test_microdataframe_survives_pickling():
833+
"""Weights and the weighted aggregations must survive a round-trip."""
834+
import pickle
835+
836+
df = mdf.MicroDataFrame(
837+
pd.DataFrame({"x": [1, 2, 3]}, index=[7, 8, 9]), weights=[1, 2, 3]
838+
)
839+
restored = pickle.loads(pickle.dumps(df))
840+
assert isinstance(restored, mdf.MicroDataFrame)
841+
assert isinstance(restored.weights, pd.Series)
842+
# Would be 6 (unweighted) if the aggregation overrides were not
843+
# reinstalled after unpickling.
844+
assert restored.sum()["x"] == 14
845+
846+
847+
def test_deepcopy_preserves_weights():
848+
df = mdf.MicroDataFrame(pd.DataFrame({"x": [1, 2, 3]}), weights=[1, 2, 3])
849+
assert copy.deepcopy(df).sum()["x"] == 14
850+
s = mdf.MicroSeries([1, 2, 3], weights=[1, 2, 3])
851+
assert copy.deepcopy(s).sum() == 14

0 commit comments

Comments
 (0)