From dd9de440545cf7d58d1ece60543a0420e59ce584 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 12:38:18 -0400 Subject: [PATCH 1/6] Remove unused functionality not used by PolicyEngine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a major breaking change that removes all functionality not actively used by PolicyEngine repositories, significantly simplifying the package. Kept: - MicroDataFrame class - MicroSeries class (including .gini() method) - MicroSeriesGroupBy and MicroDataFrameGroupBy classes Removed: - All standalone modules: agg, concat, constants, custom_taxes, income_measures, inequality, io, poverty, tax, ubi, utils, weighted - _optional module (no longer needed) - Associated test files for deleted modules - Test for concat functionality in test_microseries_dataframe.py The package now focuses solely on providing weighted pandas-like data structures that PolicyEngine depends on for microsimulation and inequality calculations. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- changelog_entry.yaml | 9 + microdf/__init__.py | 135 +---------- microdf/_optional.py | 83 ------- microdf/agg.py | 90 -------- microdf/concat.py | 29 --- microdf/constants.py | 40 ---- microdf/custom_taxes.py | 174 --------------- microdf/income_measures.py | 63 ------ microdf/inequality.py | 219 ------------------ microdf/io.py | 30 --- microdf/poverty.py | 136 ----------- microdf/tax.py | 78 ------- microdf/tests/test_compare.py | 48 ---- microdf/tests/test_decile_rank.py | 91 -------- microdf/tests/test_inequality.py | 20 -- microdf/tests/test_io.py | 10 - microdf/tests/test_microseries_dataframe.py | 16 -- microdf/tests/test_optional_dependency.py | 54 ----- microdf/tests/test_percentile_actual.csv | 101 --------- microdf/tests/test_percentile_expected.csv | 101 --------- microdf/tests/test_poverty.py | 73 ------ microdf/tests/test_quantile_chg.py | 14 -- microdf/tests/test_tax.py | 54 ----- microdf/tests/test_utils.py | 39 ---- microdf/tests/test_weighted.py | 74 ------ microdf/ubi.py | 46 ---- microdf/utils.py | 73 ------ microdf/weighted.py | 235 -------------------- 28 files changed, 10 insertions(+), 2125 deletions(-) delete mode 100644 microdf/_optional.py delete mode 100644 microdf/agg.py delete mode 100644 microdf/concat.py delete mode 100644 microdf/constants.py delete mode 100644 microdf/custom_taxes.py delete mode 100644 microdf/income_measures.py delete mode 100644 microdf/inequality.py delete mode 100644 microdf/io.py delete mode 100644 microdf/poverty.py delete mode 100644 microdf/tax.py delete mode 100644 microdf/tests/test_compare.py delete mode 100644 microdf/tests/test_decile_rank.py delete mode 100644 microdf/tests/test_inequality.py delete mode 100644 microdf/tests/test_io.py delete mode 100644 microdf/tests/test_optional_dependency.py delete mode 100644 microdf/tests/test_percentile_actual.csv delete mode 100644 microdf/tests/test_percentile_expected.csv delete mode 100644 microdf/tests/test_poverty.py delete mode 100644 microdf/tests/test_quantile_chg.py delete mode 100644 microdf/tests/test_tax.py delete mode 100644 microdf/tests/test_utils.py delete mode 100644 microdf/tests/test_weighted.py delete mode 100644 microdf/ubi.py delete mode 100644 microdf/utils.py delete mode 100644 microdf/weighted.py diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29b..f324d094 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,9 @@ +- bump: major + changes: + removed: + - Remove all modules not used by PolicyEngine repositories. + - Keep only MicroDataFrame, MicroSeries, and their GroupBy classes. + - Remove agg, concat, constants, custom_taxes, income_measures, inequality (standalone functions), io, poverty (standalone functions), tax, ubi, utils, and weighted modules. + - Remove _optional module as it's no longer needed. + - Remove associated test files for deleted modules. + - Simplify package to focus on core weighted data structures used by PolicyEngine. \ No newline at end of file diff --git a/microdf/__init__.py b/microdf/__init__.py index 905bca37..75612a42 100644 --- a/microdf/__init__.py +++ b/microdf/__init__.py @@ -1,147 +1,14 @@ -from .agg import agg, combine_base_reform, pctchg_base_reform -from .concat import concat -from .constants import ( - BENS, - ECI_REMOVE_COLS, - HOUSING_CASH_SHARE, - MCAID_CASH_SHARE, - MCARE_CASH_SHARE, - MED_BENS, - OTHER_CASH_SHARE, - SNAP_CASH_SHARE, - SSI_CASH_SHARE, - TANF_CASH_SHARE, - VET_CASH_SHARE, - WIC_CASH_SHARE, -) -from .custom_taxes import ( - CARBON_TAX_INCIDENCE, - FTT_INCIDENCE, - VAT_INCIDENCE, - add_carbon_tax, - add_custom_tax, - add_ftt, - add_vat, -) -from .income_measures import cash_income, market_income, tpc_eci -from .inequality import ( - bottom_50_pct_share, - bottom_x_pct_share, - gini, - t10_b50, - top_0_1_pct_share, - top_1_pct_share, - top_10_pct_share, - top_50_pct_share, - top_x_pct_share, -) -from .io import read_stata_zip from .microdataframe import MicroDataFrame, MicroDataFrameGroupBy from .microseries import MicroSeries, MicroSeriesGroupBy -from .poverty import ( - deep_poverty_gap, - deep_poverty_rate, - fpl, - poverty_gap, - poverty_rate, - squared_poverty_gap, -) -from .tax import mtr, tax_from_mtrs -from .ubi import ubi_or_bens -from .utils import ( - cartesian_product, - dedup_list, - flatten, - listify, - ordinal_label, -) -from .weighted import ( - add_weighted_quantiles, - quantile_chg, - weight, - weighted_mean, - weighted_median, - weighted_quantile, - weighted_sum, -) name = "microdf" __version__ = "0.1.0" __all__ = [ - # agg.py - "combine_base_reform", - "pctchg_base_reform", - "agg", - # concat.py - "concat", - # constants.py - "BENS", - "ECI_REMOVE_COLS", - "HOUSING_CASH_SHARE", - "MCAID_CASH_SHARE", - "MCARE_CASH_SHARE", - "MED_BENS", - "OTHER_CASH_SHARE", - "SNAP_CASH_SHARE", - "SSI_CASH_SHARE", - "TANF_CASH_SHARE", - "VET_CASH_SHARE", - "WIC_CASH_SHARE", - # custom_taxes.py - "CARBON_TAX_INCIDENCE", - "FTT_INCIDENCE", - "VAT_INCIDENCE", - "add_custom_tax", - "add_vat", - "add_carbon_tax", - "add_ftt", - # income_measures.py - "cash_income", - "tpc_eci", - "market_income", - # inequality.py - "gini", - "top_x_pct_share", - "bottom_x_pct_share", - "bottom_50_pct_share", - "top_10_pct_share", - "top_1_pct_share", - "top_0_1_pct_share", - "top_50_pct_share", - "t10_b50", - # io.py - "read_stata_zip", - # poverty.py - "fpl", - "poverty_rate", - "deep_poverty_rate", - "poverty_gap", - "squared_poverty_gap", - "deep_poverty_gap", - # tax.py - "mtr", - "tax_from_mtrs", - # ubi.py - "ubi_or_bens", - # utils.py - "ordinal_label", - "dedup_list", - "listify", - "flatten", - "cartesian_product", - # weighted.py - "weight", - "weighted_sum", - "weighted_mean", - "weighted_quantile", - "weighted_median", - "add_weighted_quantiles", - "quantile_chg", # microseries.py "MicroSeries", "MicroSeriesGroupBy", # microdataframe.py "MicroDataFrame", "MicroDataFrameGroupBy", -] +] \ No newline at end of file diff --git a/microdf/_optional.py b/microdf/_optional.py deleted file mode 100644 index 3d795745..00000000 --- a/microdf/_optional.py +++ /dev/null @@ -1,83 +0,0 @@ -import distutils.version -import importlib -import types -import warnings -from typing import Optional, Union - -# Adapted from: -# https://github.com/pandas-dev/pandas/blob/master/pandas/compat/_optional.py - -VERSIONS = {} - - -def _get_version(module: types.ModuleType) -> str: - """ - - :param module: types.ModuleType: - :param module: types.ModuleType: - - """ - version = getattr(module, "__version__", None) - if version is None: - # xlrd uses a capitalized attribute name - version = getattr(module, "__VERSION__", None) - - if version is None: - raise ImportError(f"Can't determine version for {module.__name__}") - return version - - -def import_optional_dependency( - name: str, - extra: Optional[str] = "", - raise_on_missing: Optional[bool] = True, - on_version: Optional[str] = "raise", -) -> Union[types.ModuleType, None]: - """Import an optional dependency. By default, if a dependency is missing an - ImportError with a nice message will be raised. If a dependency is present, - but too old, we raise. - - :param name: The module name. This should be top-level only, so that the - version may be checked. - :type name: str - :param extra: Additional text to include in the ImportError message. - :type extra: str - :param raise_on_missing: Whether to raise if the optional dependency is - not found. When False and the module is not present, None is returned. - :type raise_on_missing: bool, default True - :param on_version: What to do when a dependency's version is too old. - * raise : Raise an ImportError - * warn : Warn that the version is too old. Returns None - * ignore: Return the module, even if the version is too old. - It's expected that users validate the version locally when - :type on_version: str {'raise', 'warn'} - """ - msg = ( - f"Missing optional dependency '{name}'. {extra} " - f"Use pip or conda to install {name}." - ) - try: - module = importlib.import_module(name) - except ImportError: - if raise_on_missing: - raise ImportError(msg) from None - else: - return None - - minimum_version = VERSIONS.get(name) - if minimum_version: - version = _get_version(module) - if distutils.version.LooseVersion(version) < minimum_version: - assert on_version in {"warn", "raise", "ignore"} - msg = ( - f"microdf requires version '{minimum_version}' or newer of " - f"'{name}' " - f"(version '{version}' currently installed)." - ) - if on_version == "warn": - warnings.warn(msg, UserWarning) - return None - elif on_version == "raise": - raise ImportError(msg) - - return module diff --git a/microdf/agg.py b/microdf/agg.py deleted file mode 100644 index baa161cf..00000000 --- a/microdf/agg.py +++ /dev/null @@ -1,90 +0,0 @@ -from typing import Optional - -import pandas as pd - -import microdf as mdf - - -def combine_base_reform( - base: pd.DataFrame, - reform: pd.DataFrame, - base_cols: Optional[list] = None, - cols: Optional[list] = None, - reform_cols: Optional[list] = None, -) -> pd.DataFrame: - """Combine base and reform with certain columns. - - :param base: Base DataFrame. Index must match reform. - :type base: pd.DataFrame - :param reform: Reform DataFrame. Index must match base. - :type reform: pd.DataFrame - :param base_cols: Columns in base to keep. - :type base_cols: list, optional - :param cols: Columns to keep from both base and reform. - :type cols: list, optional - :param reform_cols: Columns in reform to keep. - :type reform_cols: list, optional - :returns: DataFrame with columns for base ("_base") and reform ("_reform"). - :rtype: pd.DataFrame - """ - all_base_cols = mdf.listify([base_cols] + [cols]) - all_reform_cols = mdf.listify([reform_cols] + [cols]) - return base[all_base_cols].join( - reform[all_reform_cols], lsuffix="_base", rsuffix="_reform" - ) - - -def pctchg_base_reform(combined: pd.DataFrame, metric: str) -> pd.Series: - """Calculates the percentage change in a metric for a combined dataset. - - :param combined: Combined DataFrame with _base and _reform columns. - :type combined: pd.DataFrame - :param metric: String of the column to calculate the difference. Must exist - as metric_m_base and metric_m_reform in combined. - :type metric: str - :returns: Series with percentage change. - :rtype: pd.Series - """ - return combined[metric + "_m_reform"] / combined[metric + "_m_base"] - 1 - - -def agg( - base: pd.DataFrame, - reform: pd.DataFrame, - groupby: str, - metrics: list, - base_metrics: Optional[list] = None, - reform_metrics: Optional[list] = None, -) -> pd.DataFrame: - """Aggregates differences between base and reform. - - :param base: Base DataFrame. Index must match reform. - :type base: pd.DataFrame - :param reform: Reform DataFrame. Index must match base. - :type reform: pd.DataFrame - :param groupby: Variable in base to group on. - :type groupby: str - :param metrics: List of variables to agg and calculate the % change of. - These should have associated weighted columns ending in _m in base and - reform. - :type metrics: list - :param base_metrics: List of variables from base to sum. - :type base_metrics: Optional[list] - :param reform_metrics: List of variables from reform to sum. - :type reform_metrics: Optional[list] - :returns: DataFrame with groupby and metrics, and _pctchg metrics. - :rtype: pd.DataFrame - """ - metrics = mdf.listify(metrics) - metrics_m = [i + "_m" for i in metrics] - combined = combine_base_reform( - base, - reform, - base_cols=mdf.listify([groupby, base_metrics]), - cols=mdf.listify(metrics_m), - reform_cols=mdf.listify(reform_metrics), - ) - grouped = combined.groupby(groupby).sum() - for metric in metrics: - grouped[metric + "_pctchg"] = pctchg_base_reform(grouped, metric) - return grouped diff --git a/microdf/concat.py b/microdf/concat.py deleted file mode 100644 index cc469205..00000000 --- a/microdf/concat.py +++ /dev/null @@ -1,29 +0,0 @@ -import inspect - -import pandas as pd - -import microdf as mdf -from microdf.microdataframe import MicroDataFrame - - -def concat(*args, **kwargs) -> "MicroDataFrame": - """Concatenates MicroDataFrame objects, preserving weights. If - concatenating horizontally, the first set of weights are used. All args and - kwargs are passed to pd.concat. - - :return: MicroDataFrame with concatenated weights. - :rtype: mdf.MicroDataFrame - """ - # Extract args with respect to pd.concat. - pd_args = inspect.getcallargs(pd.concat, *args, **kwargs) - objs = pd_args["objs"] - axis = pd_args["axis"] - # Create result, starting with pd.concat. - res = mdf.MicroDataFrame(pd.concat(*args, **kwargs)) - # Assign weights depending on axis. - if axis == 0: - res.weights = pd.concat([obj.weights for obj in objs]) - else: - # If concatenating horizontally, use the first set of weights. - res.weights = objs[0].weights - return res diff --git a/microdf/constants.py b/microdf/constants.py deleted file mode 100644 index e8e3c3fe..00000000 --- a/microdf/constants.py +++ /dev/null @@ -1,40 +0,0 @@ -# Constants for share of each benefit that is cash. -HOUSING_CASH_SHARE = 0.0 -MCAID_CASH_SHARE = 0.0 -MCARE_CASH_SHARE = 0.0 -# https://github.com/open-source-economics/taxdata/issues/148 -# https://docs.google.com/spreadsheets/d/1g_YdFd5idgLL764G0pZBiBnIlnCBGyxBmapXCOZ1OV4 -OTHER_CASH_SHARE = 0.35 -SNAP_CASH_SHARE = 0.0 -SSI_CASH_SHARE = 1.0 -TANF_CASH_SHARE = 0.25 -# https://github.com/open-source-economics/C-TAM/issues/62. -VET_CASH_SHARE = 0.48 -WIC_CASH_SHARE = 0.0 - -# Columns to remove from expanded_income to approximate TPC's Expanded Cash -# Income. -ECI_REMOVE_COLS = [ - "wic_ben", - "housing_ben", - "vet_ben", - "mcare_ben", - "mcaid_ben", -] - -# Benefits. -BENS = [ - "housing_ben", - "mcaid_ben", - "mcare_ben", - "vet_ben", - "other_ben", - "snap_ben", - "ssi_ben", - "tanf_ben", - "wic_ben", - "e02400", # Social Security (OASDI). - "e02300", # Unemployment insurance. -] - -MED_BENS = ["mcaid_ben", "mcare_ben", "vet_ben"] diff --git a/microdf/custom_taxes.py b/microdf/custom_taxes.py deleted file mode 100644 index 4a61be90..00000000 --- a/microdf/custom_taxes.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Functions and data for estimating taxes outside the income tax system. - -Examples include value added tax, financial transaction tax, and carbon tax. -""" - -from typing import Optional - -import numpy as np -import pandas as pd - -import microdf as mdf - -# Source: -# https://www.taxpolicycenter.org/briefing-book/who-would-bear-burden-vat -VAT_INCIDENCE = pd.Series( - index=[-1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99, 99.9], - data=[3.9, 3.9, 3.6, 3.6, 3.6, 3.6, 3.6, 3.4, 3.4, 3.2, 2.8, 2.5, 2.5], -) -VAT_INCIDENCE /= 100 - -# Source: Table 5 in -# https://www.treasury.gov/resource-center/tax-policy/tax-analysis/Documents/WP-115.pdf -CARBON_TAX_INCIDENCE = pd.Series( - index=[-1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99, 99.9], - data=[0.8, 1.2, 1.4, 1.5, 1.6, 1.7, 1.8, 1.8, 1.8, 1.8, 1.6, 1.4, 0.7], -) -CARBON_TAX_INCIDENCE /= 100 - -# Source: Figure 1 in -# https://www.taxpolicycenter.org/sites/default/files/alfresco/publication-pdfs/2000587-financial-transaction-taxes.pdf -FTT_INCIDENCE = pd.Series( - index=[-1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99, 99.9], - data=[0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.3, 0.4, 0.8, 1.0], -) -FTT_INCIDENCE /= 100 - - -def add_custom_tax( - df: pd.DataFrame, - segment_income: str, - w: str, - base_income: str, - incidence: pd.Series, - name: str, - total: Optional[float] = None, - ratio: Optional[float] = None, - verbose: Optional[bool] = True, -) -> None: - """Add a custom tax based on incidence analysis driven by percentiles. - - :param df: DataFrame. - :param segment_income: Income measure used to segment tax units into - quantiles. - :param w: Weight used to segment into quantiles (either s006 or XTOT_m). - :param base_income: Income measure by which incidence is multiplied to - estimate liability. - :param incidence: pandas Series indexed on the floor of an income - percentile, with values for the tax rate. - :param name: Name of the column to add. - :param total: Total amount the tax should generate. If not provided, - liabilities are calculated only based on the incidence schedule. - (Default value = None) - :param ratio: Ratio to adjust the tax by, compared to the original tax. - This acts as a multiplier for the incidence argument. (Default value = - None) - :param verbose: Whether to print the tax adjustment factor if needed. - Defaults to True. - :returns: Nothing. Adds the column name to df representing the tax - liability. df is also sorted by segment_income. - """ - if ratio is not None: - incidence = incidence * ratio - assert total is None, "ratio and total cannot both be provided." - df.sort_values(segment_income, inplace=True) - income_percentile = 100 * df[w].cumsum() / df[w].sum() - tu_incidence = incidence.iloc[ - pd.cut( - income_percentile, - # Add a right endpoint. Should be 100 but sometimes a decimal - # gets added. - bins=incidence.index.tolist() + [101], - labels=False, - ) - ].values - df[name] = np.maximum(0, tu_incidence * df[base_income]) - if total is not None: - initial_total = mdf.weighted_sum(df, name, "s006") - if verbose: - print( - "Multiplying tax by " - + str(round(total / initial_total, 2)) - + "." - ) - df[name] *= total / initial_total - - -def add_vat( - df: pd.DataFrame, - segment_income: Optional[str] = "tpc_eci", - w: Optional[str] = "XTOT_m", - base_income: Optional[str] = "aftertax_income", - incidence: Optional[pd.Series] = VAT_INCIDENCE, - name: Optional[str] = "vat", - **kwargs, -) -> None: - """Add value added tax based on incidence estimate from Tax Policy Center. - - :param df: DataFrame with columns for tpc_eci, XTOT_m, and aftertax_income. - :param Other: arguments: Args to add_custom_tax with VAT defaults. - :param segment_income: Default value = "tpc_eci") - :param w: Default value = "XTOT_m") - :param base_income: Default value = "aftertax_income") - :param incidence: Default value = VAT_INCIDENCE) - :param name: Default value = "vat") :param **kwargs: Other arguments passed - to add_custom_tax(). - :returns: Nothing. Adds vat to df. df is also sorted by tpc_eci. - """ - add_custom_tax( - df, segment_income, w, base_income, incidence, name, **kwargs - ) - - -def add_carbon_tax( - df: pd.DataFrame, - segment_income: Optional[str] = "tpc_eci", - w: Optional[str] = "XTOT_m", - base_income: Optional[str] = "aftertax_income", - incidence: Optional[pd.Series] = CARBON_TAX_INCIDENCE, - name: Optional[str] = "carbon_tax", - **kwargs, -) -> None: - """Add carbon tax based on incidence estimate from the US Treasury - Department. - - :param df: DataFrame with columns for tpc_eci, XTOT_m, and aftertax_income. - :param Other: arguments: Args to add_custom_tax with carbon tax defaults. - :param segment_income: Default value = "tpc_eci") - :param w: Default value = "XTOT_m") - :param base_income: Default value = "aftertax_income") - :param incidence: Default value = CARBON_TAX_INCIDENCE) - :param name: Default value = "carbon_tax") :param **kwargs: Other arguments - passed to add_custom_tax(). - :returns: Nothing. Adds carbon_tax to df. df is also sorted by tpc_eci. - """ - add_custom_tax( - df, segment_income, w, base_income, incidence, name, **kwargs - ) - - -def add_ftt( - df: pd.DataFrame, - segment_income: Optional[str] = "tpc_eci", - w: Optional[str] = "XTOT_m", - base_income: Optional[str] = "aftertax_income", - incidence: Optional[pd.Series] = FTT_INCIDENCE, - name: Optional[str] = "ftt", - **kwargs, -) -> None: - """Add financial transaction tax based on incidence estimate from Tax - Policy Center. - - :param df: DataFrame with columns for tpc_eci, XTOT_m, and aftertax_income. - :param Other: arguments: Args to add_custom_tax with FTT defaults. - :param segment_income: Default value = "tpc_eci") - :param w: Default value = "XTOT_m") - :param base_income: Default value = "aftertax_income") - :param incidence: Default value = FTT_INCIDENCE) - :param name: Default value = "ftt") :param **kwargs: Other arguments passed - to add_custom_tax(). - :returns: Nothing. Adds ftt to df. df is also sorted by tpc_eci. - """ - add_custom_tax( - df, segment_income, w, base_income, incidence, name, **kwargs - ) diff --git a/microdf/income_measures.py b/microdf/income_measures.py deleted file mode 100644 index ba6090b9..00000000 --- a/microdf/income_measures.py +++ /dev/null @@ -1,63 +0,0 @@ -import pandas as pd - -import microdf as mdf - -# See -# https://docs.google.com/spreadsheets/d/1I-Qe8uD58bLnPkimc9eaPgs4AE7x5FZYmTZwVX_WyT8 -# for a comparison of income measures used here. - - -def cash_income(df: pd.DataFrame) -> pd.Series: - """Calculates income after taxes and cash transfers. - - Defined as aftertax_income minus non-cash benefits. - - :param df: A Tax-Calculator pandas DataFrame with columns for - * aftertax_income - * housing_ben - * mcaid_ben - * mcare_ben - * other_ben - * snap_ben - * ssi_bn - * tanf_ben - * vet_ben - * wic_ben - :returns: A pandas Series with the cash income for each row in df. - """ - return ( - df.aftertax_income - - (1 - mdf.HOUSING_CASH_SHARE) * df.housing_ben - - (1 - mdf.MCAID_CASH_SHARE) * df.mcaid_ben - - (1 - mdf.MCARE_CASH_SHARE) * df.mcare_ben - - (1 - mdf.OTHER_CASH_SHARE) * df.other_ben - - (1 - mdf.SNAP_CASH_SHARE) * df.snap_ben - - (1 - mdf.SSI_CASH_SHARE) * df.ssi_ben - - (1 - mdf.TANF_CASH_SHARE) * df.tanf_ben - - (1 - mdf.VET_CASH_SHARE) * df.vet_ben - - (1 - mdf.WIC_CASH_SHARE) * df.wic_ben - ) - - -def tpc_eci(df: pd.DataFrame) -> pd.Series: - """Approximates Tax Policy Center's Expanded Cash Income measure. - - Subtracts WIC, housing assistance, veteran's benefits, Medicare, and - Medicaid from expanded_income. ECI adds income measures not modeled in Tax- - Calculator, so these are ignored and will create a discrepancy compared to - TPC's ECI. - - :param df: DataFrame with columns from Tax-Calculator. - :returns: pandas Series with TPC's ECI. - """ - return df.expanded_income - df[mdf.ECI_REMOVE_COLS].sum(axis=1) - - -def market_income(df: pd.DataFrame) -> pd.Series: - """Approximates CBO's market income concept, which is income before social - insurance, means-tested transfers, and taxes. - - :param df: DataFrame with expanded_income and benefits. - :returns: pandas Series of the same length as df. - """ - return df.expanded_income - df[mdf.BENS].sum(axis=1) diff --git a/microdf/inequality.py b/microdf/inequality.py deleted file mode 100644 index 6fa6255c..00000000 --- a/microdf/inequality.py +++ /dev/null @@ -1,219 +0,0 @@ -from typing import List, Optional, Union - -import numpy as np -import pandas as pd - -import microdf as mdf - - -def gini( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - negatives: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates Gini index. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param negatives: An optional string indicating how to treat negative - values of x: - 'zero' replaces negative values with zeroes. - 'shift' subtracts the minimum value from all values of x, - when this minimum is negative. That is, it adds the absolute - minimum value. - Defaults to None, which leaves negative values as they are. - :param groupby: Column, or list of columns, to group by. - :returns: A float, the Gini index. - """ - - def _gini( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - negatives: Optional[str] = None, - ) -> float: - # Requires float numpy arrays (not pandas Series or lists) to work. - x = np.array(df[col]).astype("float") - if negatives == "zero": - x[x < 0] = 0 - if negatives == "shift" and np.amin(x) < 0: - x -= np.amin(x) - if w is not None: - w = np.array(df[w]).astype("float") - sorted_indices = np.argsort(x) - sorted_x = x[sorted_indices] - sorted_w = w[sorted_indices] - cumw = np.cumsum(sorted_w) - cumxw = np.cumsum(sorted_x * sorted_w) - return np.sum(cumxw[1:] * cumw[:-1] - cumxw[:-1] * cumw[1:]) / ( - cumxw[-1] * cumw[-1] - ) - else: - sorted_x = np.sort(x) - n = len(x) - cumxw = np.cumsum(sorted_x) - # The above formula, with all weights equal to 1 simplifies to: - return (n + 1 - 2 * np.sum(cumxw) / cumxw[-1]) / n - - if groupby is None: - return _gini(df, col, w, negatives) - return df.groupby(groupby).apply(lambda x: _gini(x, col, w, negatives)) - - -def top_x_pct_share( - df: pd.DataFrame, - col: str, - top_x_pct: float, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates top x% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param top_x_pct: Decimal between 0 and 1 of the top %, e.g. 0.1, 0.001. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the top x%. - """ - - def _top_x_pct_share( - df: pd.DataFrame, col: str, top_x_pct: float, w: Optional[str] = None - ) -> float: - threshold = mdf.weighted_quantile(df, col, w, 1 - top_x_pct) - top_x_pct_sum = mdf.weighted_sum(df[df[col] >= threshold], col, w) - total_sum = mdf.weighted_sum(df, col, w) - return top_x_pct_sum / total_sum - - if groupby is None: - return _top_x_pct_share(df, col, top_x_pct, w) - return df.groupby(groupby).apply( - lambda x: _top_x_pct_share(x, col, top_x_pct, w) - ) - - -def bottom_x_pct_share( - df: pd.DataFrame, - col: str, - bottom_x_pct: float, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates bottom x% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param bottom_x_pct: Decimal between 0 and 1 of the top %, e.g. 0.1, 0.001. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the bottom x%. - """ - return 1 - top_x_pct_share(df, col, 1 - bottom_x_pct, w, groupby) - - -def bottom_50_pct_share( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates bottom 50% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the bottom 50%. - """ - return bottom_x_pct_share(df, col, 0.5, w, groupby) - - -def top_50_pct_share( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates top 50% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the top 50%. - """ - return top_x_pct_share(df, col, 0.5, w, groupby) - - -def top_10_pct_share( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates top 10% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the top 10%. - """ - return top_x_pct_share(df, col, 0.1, w, groupby) - - -def top_1_pct_share( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates top 1% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the top 1%. - """ - return top_x_pct_share(df, col, 0.01, w, groupby) - - -def top_0_1_pct_share( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates top 0.1% share. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the top 0.1%. - """ - return top_x_pct_share(df, col, 0.001, w, groupby) - - -def t10_b50( - df: pd.DataFrame, - col: str, - w: Optional[str] = None, - groupby: Optional[Union[str, List[str]]] = None, -) -> float: - """Calculates ratio between the top 10% and bottom 50% shares. - - :param df: DataFrame. - :param col: Name of column in df representing value. - :param w: Column representing weight in df. - :param groupby: Column, or list of columns, to group by. - :returns: The share of w-weighted val held by the top 10% divided by the - share of w-weighted val held by the bottom 50%. - """ - t10 = top_10_pct_share(df, col, w, groupby) - b50 = bottom_50_pct_share(df, col, w, groupby) - return t10 / b50 diff --git a/microdf/io.py b/microdf/io.py deleted file mode 100644 index 8c449660..00000000 --- a/microdf/io.py +++ /dev/null @@ -1,30 +0,0 @@ -import io -import zipfile - -import pandas as pd -import requests - -HEADER = { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) " - + "AppleWebKit/537.36 (KHTML, like Gecko) " - + "Chrome/50.0.2661.102 Safari/537.36" -} - - -def read_stata_zip(url: str, **kwargs) -> pd.DataFrame: - """Reads zipped Stata file by URL. - - From https://stackoverflow.com/a/59122689/1840471 - - Pending native support in - https://github.com/pandas-dev/pandas/issues/26599. - - :param url: URL string of .zip file containing a single .dta file. :param - **kwargs: Arguments passed to pandas.read_stata(). - :returns: DataFrame. - """ - r = requests.get(url, headers=HEADER) - data = io.BytesIO(r.content) - with zipfile.ZipFile(data) as archive: - with archive.open(archive.namelist()[0]) as stata: - return pd.read_stata(stata, **kwargs) diff --git a/microdf/poverty.py b/microdf/poverty.py deleted file mode 100644 index cd841150..00000000 --- a/microdf/poverty.py +++ /dev/null @@ -1,136 +0,0 @@ -import numpy as np -import pandas as pd - - -def fpl(people: int) -> float: - """Calculates the federal poverty guideline for a household of a certain - size. - - :param XTOT: The number of people in the household. - :param people: returns: The federal poverty guideline for the contiguous 48 - states. - :returns: The federal poverty guideline for the contiguous 48 states. - """ - return 7820 + 4320 * people - - -def poverty_rate( - df: pd.DataFrame, income: str, threshold: str, w: str = None -) -> float: - """Calculate poverty rate, i.e., the population share with income below - their poverty threshold. - - :param df: DataFrame with income, threshold, and possibly weight columns - for each person/household. - :type df: pd.DataFrame - :param income: Column indicating income. - :type income: str - :param threshold: Column indicating threshold. - :type threshold: str - :param w: Column indicating weight, defaults to None (unweighted). - :type w: str, optional - :return: Poverty rate between zero and one. - :rtype: float - """ - pov = df[income] < df[threshold] - if w is None: - return pov.mean() - return (pov * df[w]).sum() / df[w].sum() - - -def deep_poverty_rate( - df: pd.DataFrame, income: str, threshold: str, w: str = None -) -> float: - """Calculate deep poverty rate, i.e., the population share with income - below half their poverty threshold. - - :param df: DataFrame with income, threshold, and possibly weight columns - for each person/household. - :type df: pd.DataFrame - :param income: Column indicating income. - :type income: str - :param threshold: Column indicating threshold. - :type threshold: str - :param w: Column indicating weight, defaults to None (unweighted). - :type w: str, optional - :return: Deep poverty rate between zero and one. - :rtype: float - """ - pov = df[income] < df[threshold] / 2 - if w is None: - return pov.mean() - return (pov * df[w]).sum() / df[w].sum() - - -def poverty_gap( - df: pd.DataFrame, income: str, threshold: str, w: str = None -) -> float: - """Calculate poverty gap, i.e., the total gap between income and poverty - thresholds for all people in poverty. - - :param df: DataFrame with income, threshold, and possibly weight columns - for each household (data should represent households, not persons). - :type df: pd.DataFrame - :param income: Column indicating income. - :type income: str - :param threshold: Column indicating threshold. - :type threshold: str - :param w: Column indicating weight, defaults to None (unweighted). - :type w: str, optional - :return: Poverty gap. - :rtype: float - """ - gap = np.maximum(df[threshold] - df[income], 0) - if w is None: - return gap.sum() - return (gap * df[w]).sum() - - -def squared_poverty_gap( - df: pd.DataFrame, income: str, threshold: str, w: str = None -) -> float: - """Calculate squared poverty gap, i.e., the total squared gap between - income and poverty thresholds for all people in poverty. Also known as - poverty severity index. - - :param df: DataFrame with income, threshold, and possibly weight columns - for each household (data should represent households, not persons). - :type df: pd.DataFrame - :param income: Column indicating income. - :type income: str - :param threshold: Column indicating threshold. - :type threshold: str - :param w: Column indicating weight, defaults to None (unweighted). - :type w: str, optional - :return: Squared poverty gap. - :rtype: float - """ - gap = np.maximum(df[threshold] - df[income], 0) - sq_gap = np.power(gap, 2) - if w is None: - return sq_gap.sum() - return (sq_gap * df[w]).sum() - - -def deep_poverty_gap( - df: pd.DataFrame, income: str, threshold: str, w: str = None -) -> float: - """Calculate deep poverty gap, i.e., the total gap between income and - halved poverty thresholds for all people in deep poverty. - - :param df: DataFrame with income, threshold, and possibly weight columns - for each household (data should represent households, not persons). - :type df: pd.DataFrame - :param income: Column indicating income. - :type income: str - :param threshold: Column indicating threshold. - :type threshold: str - :param w: Column indicating weight, defaults to None (unweighted). - :type w: str, optional - :return: Deep poverty gap. - :rtype: float - """ - gap = np.maximum((df[threshold] / 2) - df[income], 0) - if w is None: - return gap.sum() - return (gap * df[w]).sum() diff --git a/microdf/tax.py b/microdf/tax.py deleted file mode 100644 index f809c224..00000000 --- a/microdf/tax.py +++ /dev/null @@ -1,78 +0,0 @@ -from typing import Optional - -import numpy as np -import pandas as pd - - -def mtr(val: pd.Series, brackets: pd.Series, rates: pd.Series) -> pd.Series: - """Calculates the marginal tax rate applied to a value depending on a tax - schedule. - - :param val: Value to assess tax on, e.g. wealth or income (list or Series). - :param brackets: Left side of each bracket (list or Series). - :param rates: Rate corresponding to each bracket. - :returns: Series of the size of val representing the marginal tax rate. - """ - df_tax = pd.DataFrame({"brackets": brackets, "rates": rates}) - df_tax["base_tax"] = ( - df_tax.brackets.sub(df_tax.brackets.shift(fill_value=0)) - .mul(df_tax.rates.shift(fill_value=0)) - .cumsum() - ) - rows = df_tax.brackets.searchsorted(val, side="right") - 1 - income_bracket_df = df_tax.loc[rows].reset_index(drop=True) - return income_bracket_df.rates - - -def tax_from_mtrs( - val: pd.Series, - brackets: pd.Series, - rates: pd.Series, - avoidance_rate: Optional[float] = 0, - avoidance_elasticity: Optional[float] = 0, - avoidance_elasticity_flat: Optional[float] = 0, -) -> pd.Series: - """Calculates tax liability based on a marginal tax rate schedule. - - :param val: Value to assess tax on, e.g. wealth or income (list or Series). - :param brackets: Left side of each bracket (list or Series). - :param rates: Rate corresponding to each bracket. - :param avoidance_rate: Constant avoidance/evasion rate in percentage terms. - Defaults to zero. - :param avoidance_elasticity: Avoidance/evasion elasticity. Response of log - taxable value with respect to tax rate. Defaults to zero. Should be - positive. - :param avoidance_elasticity_flat: Response of taxable value with respect to - tax rate. Use avoidance_elasticity in most cases. Defaults to zero. - Should be positive. - :returns: Series of tax liabilities with the same size as val. - """ - assert ( - avoidance_rate == 0 - or avoidance_elasticity == 0 - or avoidance_elasticity_flat == 0 - ), "Cannot supply multiple avoidance parameters." - assert ( - avoidance_elasticity >= 0 - ), "Provide nonnegative avoidance_elasticity." - df_tax = pd.DataFrame({"brackets": brackets, "rates": rates}) - df_tax["base_tax"] = ( - df_tax.brackets.sub(df_tax.brackets.shift(fill_value=0)) - .mul(df_tax.rates.shift(fill_value=0)) - .cumsum() - ) - if avoidance_rate == 0: # Only need MTRs if elasticity is supplied. - mtrs = mtr(val, brackets, rates) - if avoidance_elasticity > 0: - avoidance_rate = 1 - np.exp(-avoidance_elasticity * mtrs) - if avoidance_elasticity_flat > 0: - avoidance_rate = avoidance_elasticity_flat * mtrs - taxable = pd.Series(val) * (1 - avoidance_rate) - rows = df_tax.brackets.searchsorted(taxable, side="right") - 1 - income_bracket_df = df_tax.loc[rows].reset_index(drop=True) - return ( - pd.Series(taxable) - .sub(income_bracket_df.brackets) - .mul(income_bracket_df.rates) - .add(income_bracket_df.base_tax) - ) diff --git a/microdf/tests/test_compare.py b/microdf/tests/test_compare.py deleted file mode 100644 index d19361b2..00000000 --- a/microdf/tests/test_compare.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -import numpy as np -import pandas as pd - -import microdf as mdf - - -def differences( - actual: pd.DataFrame, - expected: pd.DataFrame, - f_actual: str, - f_expected: str, -) -> None: - """Check for differences between results in afilename and efilename files. - - :param actual: Actual DataFrame. - :param expected: Expected DataFrame. - :param f_actual: Filename of the actual CSV. - :param f_expected: Filename of the expected CSV. - """ - if not np.allclose(actual, expected): - msg = "COMPARE RESULTS DIFFER\n" - msg += "-------------------------------------------------\n" - msg += "--- NEW RESULTS IN {} FILE ---\n" - msg += "--- if new OK, copy {} to ---\n" - msg += "--- {} ---\n" - msg += "--- and rerun test. ---\n" - msg += "-------------------------------------------------\n" - raise ValueError(msg.format(f_actual, f_actual, f_expected)) - - -def test_percentile_agg_compare(tests_path: str) -> None: - """ - :param tests_path: Folder path to write test results. - """ - N = 1000 - np.random.seed(0) - df = pd.DataFrame({"val": np.random.rand(N), "w": np.random.rand(N)}) - mdf.add_weighted_quantiles(df, "val", "w") - percentile_sum = df.groupby("val_percentile")[["val", "w"]].sum() - F_ACTUAL = "test_percentile_actual.csv" - F_EXPECTED = "test_percentile_expected.csv" - percentile_sum.to_csv(os.path.join(tests_path, F_ACTUAL)) - # Re-read as CSV to remove index and ensure CSVs are equal. - actual = pd.read_csv(os.path.join(tests_path, F_ACTUAL)) - expected = pd.read_csv(os.path.join(tests_path, F_EXPECTED)) - differences(actual, expected, F_ACTUAL, F_EXPECTED) diff --git a/microdf/tests/test_decile_rank.py b/microdf/tests/test_decile_rank.py deleted file mode 100644 index 23a2d96c..00000000 --- a/microdf/tests/test_decile_rank.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Test script for enhanced decile_rank functionality.""" - -import numpy as np - -from microdf import MicroSeries - - -def test_decile_rank() -> None: - """Test the enhanced decile_rank method with assert statements.""" - print("Running enhanced decile_rank tests...") - - # Create test data with some negative values - test_data = [-5, -2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20] - test_weights = [1] * len(test_data) - test_ms = MicroSeries(test_data, weights=test_weights) - - default_deciles = test_ms.decile_rank() - assert all( - 1 <= d <= 10 for d in default_deciles.values - ), "Default deciles should be between 1 and 10" - - negative_indices = [i for i, val in enumerate(test_data) if val < 0] - negative_deciles = [default_deciles.values[i] for i in negative_indices] - assert all( - d >= 1 for d in negative_deciles - ), "Negative values should be ranked 1-10 in default mode" - - assert isinstance( - default_deciles, MicroSeries - ), "decile_rank should return MicroSeries" - - zero_deciles = test_ms.decile_rank(negatives_in_zero=True) - negative_zero_deciles = [zero_deciles.values[i] for i in negative_indices] - assert all( - d == 0 for d in negative_zero_deciles - ), "Negative values should be in decile 0 when negatives_in_zero=True" - - non_negative_indices = [i for i, val in enumerate(test_data) if val >= 0] - non_negative_deciles = [ - zero_deciles.values[i] for i in non_negative_indices - ] - assert all( - 1 <= d <= 10 for d in non_negative_deciles - ), "Non-negative values should be ranked 1-10 when negatives_in_zero=True" - - unique_deciles = sorted(zero_deciles.unique()) - assert 0 in unique_deciles, "Decile 0 should exist with negative values" - assert all( - d in unique_deciles for d in range(1, 11) - ), "Deciles 1-10 should exist with non-negative values" - - positive_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - positive_ms = MicroSeries(positive_data, weights=[1] * 10) - - pos_default = positive_ms.decile_rank() - pos_zero = positive_ms.decile_rank(negatives_in_zero=True) - assert np.array_equal( - pos_default.values, pos_zero.values - ), "Both modes should produce identical results for all positive data" - assert ( - 0 not in pos_zero.unique() - ), "Decile 0 should not exist when all values are positive" - - negative_data = [-10, -8, -6, -4, -2] - negative_ms = MicroSeries(negative_data, weights=[1] * 5) - - neg_default = negative_ms.decile_rank() - neg_zero = negative_ms.decile_rank(negatives_in_zero=True) - assert all( - 1 <= d <= 10 for d in neg_default.values - ), "Default mode should rank all negative values 1-10" - assert all( - d == 0 for d in neg_zero.values - ), "All negative values should be in decile 0 when negatives_in_zero=True" - - weighted_data = [-1, 0, 1, 2, 3] - weighted_weights = [0.5, 1.0, 1.5, 2.0, 2.5] - weighted_ms = MicroSeries(weighted_data, weights=weighted_weights) - - w_default = weighted_ms.decile_rank() - w_zero = weighted_ms.decile_rank(negatives_in_zero=True) - assert np.array_equal( - w_default.weights.values, weighted_weights - ), "Weights should be preserved in default mode" - assert np.array_equal( - w_zero.weights.values, weighted_weights - ), "Weights should be preserved in negatives_in_zero mode" - - -if __name__ == "__main__": - test_decile_rank() diff --git a/microdf/tests/test_inequality.py b/microdf/tests/test_inequality.py deleted file mode 100644 index dfd19289..00000000 --- a/microdf/tests/test_inequality.py +++ /dev/null @@ -1,20 +0,0 @@ -import pandas as pd - -import microdf as mdf - - -def test_top_pct() -> None: - x = list(range(1, 11)) # 1 to 10. Sum = 10 * 11 / 2 = 55. - df = pd.DataFrame({"x": x}) - ms = mdf.MicroSeries(x) - RES = 10 / 55 - assert mdf.top_10_pct_share(df, "x") == RES - assert ms.top_10_pct_share() == RES - x = list(range(1, 4)) - df = pd.DataFrame({"x": x, "w": x}) - ms = mdf.MicroSeries(x, weights=x) - # This is equivalent to [1, 2, 2, 3, 3, 3] - # Sum = 14, top half is 9. - RES = 9 / 14 - assert mdf.top_50_pct_share(df, "x", "w") == RES - assert ms.top_50_pct_share() == RES diff --git a/microdf/tests/test_io.py b/microdf/tests/test_io.py deleted file mode 100644 index 1f44161c..00000000 --- a/microdf/tests/test_io.py +++ /dev/null @@ -1,10 +0,0 @@ -import microdf as mdf - - -def test_read_stata_zip() -> None: - """""" - SCF2016 = "https://www.federalreserve.gov/econres/files/scfp2016s.zip" - COLS = ["wgt", "networth"] - df = mdf.read_stata_zip(SCF2016, columns=COLS) - assert df.columns.tolist() == COLS - assert df.shape[0] > 0 diff --git a/microdf/tests/test_microseries_dataframe.py b/microdf/tests/test_microseries_dataframe.py index 0cfc4657..9d259664 100644 --- a/microdf/tests/test_microseries_dataframe.py +++ b/microdf/tests/test_microseries_dataframe.py @@ -105,22 +105,6 @@ def test_multiple_groupby() -> None: assert (df.groupby(["x", "y"]).z.sum() == np.array([5, 6])).all() -def test_concat() -> None: - df1 = mdf.MicroDataFrame({"x": [1, 2]}, weights=[3, 4]) - df2 = mdf.MicroDataFrame({"y": [5, 6]}, weights=[7, 8]) - # Verify that pd.concat returns DataFrame (probably no way to fix this). - pd_long = pd.concat([df1, df2]) - assert isinstance(pd_long, pd.DataFrame) - assert not isinstance(pd_long, mdf.MicroDataFrame) - # Verify that mdf.concat works. - mdf_long = mdf.concat([df1, df2]) - assert isinstance(mdf_long, mdf.MicroDataFrame) - # Weights should be preserved. - assert mdf_long.weights.equals(pd.concat([df1.weights, df2.weights])) - # Verify it works horizontally too (take the first set of weights). - mdf_wide = mdf.concat([df1, df2], axis=1) - assert isinstance(mdf_wide, mdf.MicroDataFrame) - assert mdf_wide.weights.equals(df1.weights) def test_set_index() -> None: diff --git a/microdf/tests/test_optional_dependency.py b/microdf/tests/test_optional_dependency.py deleted file mode 100644 index a4b9b718..00000000 --- a/microdf/tests/test_optional_dependency.py +++ /dev/null @@ -1,54 +0,0 @@ -import sys -import types - -import pytest - -from microdf._optional import VERSIONS, import_optional_dependency - - -def test_import_optional() -> None: - """""" - match = "Missing .*notapackage.* pip .* conda .* notapackage" - with pytest.raises(ImportError, match=match): - import_optional_dependency("notapackage") - - result = import_optional_dependency("notapackage", raise_on_missing=False) - assert result is None - - -def test_xlrd_version_fallback() -> None: - """""" - pytest.importorskip("xlrd") - import_optional_dependency("xlrd") - - -def test_bad_version() -> None: - """""" - name = "fakemodule" - module = types.ModuleType(name) - module.__version__ = "0.9.0" - sys.modules[name] = module - VERSIONS[name] = "1.0.0" - - match = "microdf requires .*1.0.0.* of .fakemodule.*'0.9.0'" - with pytest.raises(ImportError, match=match): - import_optional_dependency("fakemodule") - - with pytest.warns(UserWarning): - result = import_optional_dependency("fakemodule", on_version="warn") - assert result is None - - module.__version__ = "1.0.0" # exact match is OK - result = import_optional_dependency("fakemodule") - assert result is module - - -def test_no_version_raises() -> None: - """""" - name = "fakemodule" - module = types.ModuleType(name) - sys.modules[name] = module - VERSIONS[name] = "1.0.0" - - with pytest.raises(ImportError, match="Can't determine .* fakemodule"): - import_optional_dependency(name) diff --git a/microdf/tests/test_percentile_actual.csv b/microdf/tests/test_percentile_actual.csv deleted file mode 100644 index 93adab4a..00000000 --- a/microdf/tests/test_percentile_actual.csv +++ /dev/null @@ -1,101 +0,0 @@ -val_percentile,val,w -1,0.04936696707980226,5.003048046601982 -2,0.17830779834685495,5.114684559704431 -3,0.2857988855674821,5.003202737366776 -4,0.33805302460864795,4.733066370107501 -5,0.5703591960673162,5.239285359053462 -6,0.6096331244255117,5.429420284539545 -7,0.5985039643349068,4.348564066721563 -8,0.7510884464659845,5.440450569600465 -9,1.0876105767407074,5.499343558417598 -10,0.8899045403172029,5.153834875170943 -11,1.3946834434117863,5.235029272275189 -12,0.9305982112361821,4.417722946149524 -13,0.9650569353812546,5.348799899633306 -14,1.3214822270450117,5.332570420963053 -15,1.2715100776802908,5.061289605022859 -16,1.6720424048893723,5.305386843295495 -17,1.942473751244783,5.255981017771978 -18,1.0348899262653468,4.336843836714941 -19,1.7975116284606523,5.1773144696335915 -20,2.1006171498258643,5.589603555523628 -21,1.8709513510005413,5.335531722226071 -22,1.7207120623536305,4.695751363757509 -23,2.0018736826457206,5.054306807303892 -24,2.313857097420784,4.88414068067632 -25,2.8967006378513713,5.5236380227234845 -26,2.489820947315488,4.791953107597212 -27,2.8411561635459712,5.5661465432344635 -28,2.426135741855565,4.446073644558097 -29,3.3575910230398183,5.291769801304535 -30,2.314506073869091,5.149625279108836 -31,3.551962771845865,5.402587912747317 -32,4.002233897939343,5.093801654006394 -33,3.5270226772464675,5.10624451980372 -34,2.654138701549258,5.0433975684692856 -35,2.7068858476732167,4.636131024815089 -36,4.511798986257632,5.713008105710415 -37,3.583675870814159,4.894200392814213 -38,3.6698366244319387,5.033490962922036 -39,3.3630281088967364,5.715252956264232 -40,2.269128705743366,4.687099540814201 -41,3.0962247363581996,5.0205442523389685 -42,5.586938042279174,5.2375613191718084 -43,4.062902694908745,5.482945818496651 -44,3.3160969156123503,4.9389434053257 -45,4.241531465516575,5.242988142385608 -46,5.632990791952208,4.604524871392653 -47,4.0031095981553575,5.449983106760844 -48,3.6271942171985487,4.5252648036434655 -49,4.574489241426403,5.827965751590956 -50,3.26114249800974,5.07472388942889 -51,5.770693937000146,4.924561771632173 -52,6.928803378451052,5.390871201576556 -53,5.605614198095166,4.867506840251817 -54,5.712794397822729,4.802840821886715 -55,6.330048423625909,5.043215026134605 -56,6.499176323898079,5.549363062730123 -57,4.455249905003632,5.388617265562159 -58,3.9747504049202647,4.804030090560459 -59,4.591734071798827,4.7972302917252705 -60,9.365137265130915,5.674437752031215 -61,7.187149592469649,4.968027795024001 -62,5.514056987390571,5.21537151235826 -63,6.199762618472242,5.007944951253934 -64,4.396249378665469,5.013683226930631 -65,7.678394207429429,5.4324805777516385 -66,8.535490990338737,4.983177889646899 -67,7.380492997015748,5.160276660633112 -68,7.495870406959787,5.166303035743687 -69,6.234591484720509,4.800326067433167 -70,4.8795947527845165,5.144612976891027 -71,6.331517082169764,4.923377667981698 -72,10.028408222423991,4.7524585511425315 -73,7.267010600317143,5.928033173228412 -74,5.856718614341236,5.153535683492982 -75,6.651308658725665,4.4540875011243095 -76,6.009374270984309,5.563696325737204 -77,6.903278042059778,4.9610776312587195 -78,9.33170057141403,5.442361429739662 -79,4.741840898628048,4.385127095815581 -80,10.441998036333617,5.681838547736348 -81,8.15549502111278,5.3122709245136726 -82,8.264309038308454,4.777889650212625 -83,5.887391058178539,4.86549347601554 -84,7.674673599420435,5.193387666428651 -85,8.617091500906668,5.330823383917327 -86,5.2052420216702355,5.116263366906492 -87,6.110525427774986,5.087746608113361 -88,10.555886286183666,4.846433766268649 -89,9.795170764821448,5.551267106013299 -90,8.999880920529858,4.896491332049084 -91,7.269834366073169,5.094073872052731 -92,8.288552906946475,5.482297252107353 -93,9.264743724824315,4.4854074855836235 -94,8.420719600334431,5.369812734262109 -95,10.393061379249968,5.3974465660187665 -96,9.545057271504412,5.083199029497632 -97,9.61390230771744,5.147118076577214 -98,8.719700395822183,4.537389380232293 -99,14.67085066573317,5.177534031309767 -100,12.905505941170981,5.72212924471392 diff --git a/microdf/tests/test_percentile_expected.csv b/microdf/tests/test_percentile_expected.csv deleted file mode 100644 index ebc0a40b..00000000 --- a/microdf/tests/test_percentile_expected.csv +++ /dev/null @@ -1,101 +0,0 @@ -val_percentile,val,w -1,0.04936696707980226,5.0030480466019815 -2,0.17830779834685495,5.114684559704431 -3,0.2857988855674821,5.003202737366776 -4,0.33805302460864795,4.733066370107501 -5,0.5703591960673162,5.239285359053462 -6,0.6096331244255117,5.429420284539545 -7,0.5985039643349068,4.348564066721563 -8,0.7510884464659845,5.440450569600465 -9,1.0876105767407074,5.499343558417599 -10,0.8899045403172029,5.1538348751709435 -11,1.394683443411786,5.23502927227519 -12,0.9305982112361821,4.417722946149524 -13,0.9650569353812546,5.348799899633306 -14,1.3214822270450117,5.3325704209630524 -15,1.2715100776802908,5.061289605022857 -16,1.6720424048893725,5.305386843295495 -17,1.9424737512447825,5.255981017771979 -18,1.0348899262653468,4.336843836714941 -19,1.7975116284606523,5.1773144696335915 -20,2.1006171498258643,5.589603555523628 -21,1.8709513510005413,5.335531722226071 -22,1.7207120623536305,4.695751363757509 -23,2.00187368264572,5.054306807303892 -24,2.313857097420784,4.884140680676319 -25,2.896700637851371,5.5236380227234845 -26,2.489820947315488,4.791953107597212 -27,2.8411561635459712,5.5661465432344635 -28,2.4261357418555654,4.446073644558097 -29,3.357591023039818,5.291769801304535 -30,2.314506073869091,5.149625279108836 -31,3.551962771845865,5.402587912747316 -32,4.002233897939344,5.093801654006394 -33,3.527022677246467,5.10624451980372 -34,2.654138701549258,5.043397568469286 -35,2.7068858476732176,4.636131024815089 -36,4.511798986257631,5.713008105710415 -37,3.583675870814159,4.894200392814213 -38,3.669836624431939,5.033490962922036 -39,3.363028108896736,5.715252956264232 -40,2.269128705743366,4.687099540814201 -41,3.0962247363582,5.0205442523389685 -42,5.586938042279173,5.2375613191718084 -43,4.062902694908744,5.482945818496652 -44,3.3160969156123503,4.938943405325701 -45,4.241531465516574,5.242988142385608 -46,5.632990791952207,4.604524871392653 -47,4.003109598155357,5.449983106760843 -48,3.627194217198549,4.5252648036434655 -49,4.574489241426403,5.827965751590956 -50,3.26114249800974,5.07472388942889 -51,5.770693937000146,4.924561771632174 -52,6.928803378451052,5.390871201576556 -53,5.605614198095166,4.867506840251817 -54,5.712794397822728,4.802840821886715 -55,6.330048423625907,5.043215026134606 -56,6.49917632389808,5.549363062730124 -57,4.455249905003633,5.388617265562158 -58,3.9747504049202647,4.804030090560459 -59,4.591734071798827,4.79723029172527 -60,9.365137265130915,5.674437752031215 -61,7.18714959246965,4.968027795024001 -62,5.514056987390571,5.21537151235826 -63,6.199762618472243,5.007944951253934 -64,4.396249378665468,5.013683226930631 -65,7.678394207429429,5.4324805777516385 -66,8.535490990338737,4.983177889646899 -67,7.380492997015747,5.160276660633111 -68,7.495870406959786,5.166303035743688 -69,6.23459148472051,4.800326067433167 -70,4.8795947527845165,5.144612976891027 -71,6.331517082169763,4.923377667981699 -72,10.028408222423991,4.7524585511425315 -73,7.267010600317143,5.928033173228412 -74,5.856718614341236,5.153535683492981 -75,6.6513086587256645,4.4540875011243095 -76,6.009374270984309,5.563696325737205 -77,6.903278042059779,4.9610776312587195 -78,9.33170057141403,5.442361429739662 -79,4.741840898628048,4.385127095815581 -80,10.441998036333615,5.681838547736346 -81,8.15549502111278,5.3122709245136726 -82,8.264309038308452,4.777889650212625 -83,5.887391058178539,4.86549347601554 -84,7.6746735994204345,5.1933876664286505 -85,8.617091500906668,5.330823383917326 -86,5.205242021670235,5.116263366906492 -87,6.110525427774986,5.0877466081133615 -88,10.555886286183668,4.846433766268649 -89,9.79517076482145,5.551267106013298 -90,8.999880920529858,4.8964913320490835 -91,7.269834366073169,5.094073872052731 -92,8.288552906946475,5.482297252107353 -93,9.264743724824317,4.4854074855836235 -94,8.42071960033443,5.3698127342621085 -95,10.393061379249968,5.397446566018766 -96,9.545057271504412,5.083199029497632 -97,9.61390230771744,5.147118076577214 -98,8.719700395822183,4.537389380232294 -99,14.670850665733171,5.177534031309767 -100,12.905505941170981,5.722129244713919 diff --git a/microdf/tests/test_poverty.py b/microdf/tests/test_poverty.py deleted file mode 100644 index adada820..00000000 --- a/microdf/tests/test_poverty.py +++ /dev/null @@ -1,73 +0,0 @@ -import numpy as np -import pandas as pd - -import microdf as mdf - -df = pd.DataFrame( - { - "income": [-10, 0, 10, 20], - "threshold": [15, 10, 15, 10], - "weight": [1, 2, 3, 4], - } -) -md = mdf.MicroDataFrame(df[["income", "threshold"]], weights=df.weight) - - -def test_poverty_rate() -> None: - # Unweighted - assert np.allclose(mdf.poverty_rate(df, "income", "threshold"), 3 / 4) - # Weighted - assert np.allclose( - mdf.poverty_rate(df, "income", "threshold", "weight"), 6 / 10 - ) - assert np.allclose(md.poverty_rate("income", "threshold"), 6 / 10) - - -def test_deep_poverty_rate() -> None: - # Unweighted - assert np.allclose(mdf.deep_poverty_rate(df, "income", "threshold"), 2 / 4) - # Weighted - assert np.allclose( - mdf.deep_poverty_rate(df, "income", "threshold", "weight"), 3 / 10 - ) - assert np.allclose(md.deep_poverty_rate("income", "threshold"), 3 / 10) - - -def test_poverty_gap() -> None: - # Unweighted - assert np.allclose(mdf.poverty_gap(df, "income", "threshold"), 25 + 10 + 5) - # Weighted - RES = 25 * 1 + 10 * 2 + 5 * 3 - assert np.allclose( - mdf.poverty_gap(df, "income", "threshold", "weight"), RES - ) - assert np.allclose(md.poverty_gap("income", "threshold"), RES) - - -def test_squared_poverty_gap() -> None: - # Unweighted - assert np.allclose( - mdf.squared_poverty_gap(df, "income", "threshold"), - 25**2 + 10**2 + 5**2, - ) - # Weighted - RES = 1 * (25**2) + 2 * (10**2) + 3 * (5**2) - assert np.allclose( - mdf.squared_poverty_gap(df, "income", "threshold", "weight"), - RES, - ) - assert np.allclose(md.squared_poverty_gap("income", "threshold"), RES) - - -def test_deep_poverty_gap() -> None: - # Unweighted - assert np.allclose( - mdf.deep_poverty_gap(df, "income", "threshold"), 17.5 + 5 + 0 + 0 - ) - # Weighted - RES = 17.5 * 1 + 5 * 2 + 0 * 3 + 0 * 4 - assert np.allclose( - mdf.deep_poverty_gap(df, "income", "threshold", "weight"), RES - ) - # Same in MicroDataFrame. - assert np.allclose(md.deep_poverty_gap("income", "threshold"), RES) diff --git a/microdf/tests/test_quantile_chg.py b/microdf/tests/test_quantile_chg.py deleted file mode 100644 index bd77844f..00000000 --- a/microdf/tests/test_quantile_chg.py +++ /dev/null @@ -1,14 +0,0 @@ -import pandas as pd - -import microdf as mdf - -V1 = [1, 2, 3] -V2 = [4, 5, 6] -W1 = [7, 8, 9] -W2 = [10, 11, 12] -DF1 = pd.DataFrame({"v": V1, "w": W1}) -DF2 = pd.DataFrame({"v": V2, "w": W2}) - - -def test_quantile_chg() -> None: - mdf.quantile_chg(DF1, DF2, "v", "w", "v", "w") diff --git a/microdf/tests/test_tax.py b/microdf/tests/test_tax.py deleted file mode 100644 index abb3a211..00000000 --- a/microdf/tests/test_tax.py +++ /dev/null @@ -1,54 +0,0 @@ -import numpy as np -import pandas as pd -import pytest - -import microdf as mdf - - -def test_tax() -> None: - """""" - # Consider a MTR schedule of 0% up to 10,000, then 10% after that. - BRACKETS = [0, 10e3] - RATES = [0, 0.1] - INCOME = [0, 5e3, 10e3, 10e3 + 1, 20e3] - EXPECTED = [0, 0, 0, 0.1, 1e3] - res = mdf.tax_from_mtrs(INCOME, BRACKETS, RATES) - pd.testing.assert_series_equal(res, pd.Series(EXPECTED)) - # Try with 10% avoidance. - EXPECTED_10PCT_AVOIDANCE = [0, 0, 0, 0, 800.0] - res_10pct_avoidance = mdf.tax_from_mtrs(INCOME, BRACKETS, RATES, 0.1) - pd.testing.assert_series_equal( - res_10pct_avoidance, pd.Series(EXPECTED_10PCT_AVOIDANCE) - ) - # Try with avoidance elasticity of 2. - EXPECTED_E2_AVOIDANCE = [ - 0, - 0, - 0, - 0, # Taxable base becomes (10e3 + 1) * (1 - 2 * 0.1) - # Taxable base becomes 20e3 * (exp(-2 * 0.1)). - 0.1 * (20e3 * np.exp(-0.2) - 10e3), - ] - res_e2_avoidance = mdf.tax_from_mtrs( - INCOME, BRACKETS, RATES, avoidance_elasticity=2 - ) - pd.testing.assert_series_equal( - res_e2_avoidance, pd.Series(EXPECTED_E2_AVOIDANCE) - ) - # Try with flat avoidance elasticity of 2. - EXPECTED_E2_AVOIDANCE_FLAT = [ - 0, - 0, - 0, - 0, # Taxable base becomes (10e3 + 1) * (1 - 2 * 0.1) - 600.0, - ] # Taxable base becomes 20e3 * (1 - 2 * 0.1) = 16e3. - res_e2_avoidance_flat = mdf.tax_from_mtrs( - INCOME, BRACKETS, RATES, avoidance_elasticity_flat=2 - ) - pd.testing.assert_series_equal( - res_e2_avoidance_flat, pd.Series(EXPECTED_E2_AVOIDANCE_FLAT) - ) - # Ensure error when passing both rate and elasticity. - with pytest.raises(Exception): - mdf.tax_from_mtrs(INCOME, BRACKETS, RATES, 0.1, 2) diff --git a/microdf/tests/test_utils.py b/microdf/tests/test_utils.py deleted file mode 100644 index d09d798f..00000000 --- a/microdf/tests/test_utils.py +++ /dev/null @@ -1,39 +0,0 @@ -import pandas as pd - -import microdf as mdf - - -def test_cartesian_product() -> None: - """""" - res = mdf.cartesian_product( - {"a": [1, 2, 3], "b": ["val1", "val2"], "c": [100, 101]} - ) - EXPECTED = pd.DataFrame( - { - "a": [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3], - "b": [ - "val1", - "val1", - "val2", - "val2", - "val1", - "val1", - "val2", - "val2", - "val1", - "val1", - "val2", - "val2", - ], - "c": [100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101], - } - ) - pd.testing.assert_frame_equal(res, EXPECTED) - - -def test_flatten() -> None: - """""" - L = [[[1, 2, 3], [4, 5]], 6] - res = list(mdf.flatten(L)) - EXPECTED = [1, 2, 3, 4, 5, 6] - assert res == EXPECTED diff --git a/microdf/tests/test_weighted.py b/microdf/tests/test_weighted.py deleted file mode 100644 index 57e4b757..00000000 --- a/microdf/tests/test_weighted.py +++ /dev/null @@ -1,74 +0,0 @@ -import pandas as pd -import pytest - -import microdf as mdf - -X = [1, 5, 2] -Y = [0, -6, 3] -W = [4, 1, 1] -df = pd.DataFrame({"x": X, "y": Y, "w": W}) -ms = mdf.MicroSeries(X, weights=W) -md = mdf.MicroDataFrame(df[["x", "y"]], weights=W) -# Also make a version with groups. -df2 = df.copy(deep=True) -df2.x *= 2 -df2.y *= 1.5 -dfg = pd.concat([df, df2]) -dfg["g"] = ["a"] * 3 + ["b"] * 3 -mdg = mdf.MicroDataFrame(dfg[["x", "y", "g"]], weights=dfg["w"]) - - -def test_weighted_quantile() -> None: - Q = [0, 0.5, 1] - mdf.weighted_quantile(df, "x", "w", Q).tolist() - - -def test_weighted_median() -> None: - assert mdf.weighted_median(df, "x") == 2 - mdf.weighted_median(df, "x", "w") - # Test with groups. - mdf.weighted_median(dfg, "x", "w", "g") - - -def test_weighted_mean() -> None: - # Test unweighted. - assert mdf.weighted_mean(df, "x") == 8 / 3 - # Test weighted. - assert mdf.weighted_mean(df, "x", "w") == 11 / 6 - # Test weighted with multiple columns. - assert mdf.weighted_mean(df, ["x", "y"], "w").tolist() == [11 / 6, -3 / 6] - # Test grouped. - mdf.weighted_mean(dfg, "x", "w", "g") - mdf.weighted_mean(dfg, ["x", "y"], "w", "g") - - -def test_weighted_sum() -> None: - # Test unweighted. - assert mdf.weighted_sum(df, "x") == 8 - # Test weighted. - assert mdf.weighted_sum(df, "x", "w") == 11 - # Test weighted with multiple columns. - assert mdf.weighted_sum(df, ["x", "y"], "w").tolist() == [11, -3] - # Test grouped. - mdf.weighted_sum(dfg, "x", "w", "g") - mdf.weighted_sum(dfg, ["x", "y"], "w", "g") - - -def test_gini() -> None: - # Test nothing breaks. - ms.gini() - # Unweighted. - mdf.gini(df, "x") - # Weighted - mdf.gini(df, "x", "w") - # Unweighted, grouped - mdf.gini(dfg, "x", groupby="g") - # Weighted, grouped - mdf.gini(dfg, "x", "w", groupby="g") - # Test old and new match. - assert ms.gini() == mdf.gini(df, "x", "w") - - -def test_add_weighted_quantiles() -> None: - with pytest.deprecated_call(): - mdf.add_weighted_quantiles(df, "x", "w") diff --git a/microdf/ubi.py b/microdf/ubi.py deleted file mode 100644 index 2ff7fc29..00000000 --- a/microdf/ubi.py +++ /dev/null @@ -1,46 +0,0 @@ -from typing import List, Optional, Union - -import numpy as np -import pandas as pd - -import microdf as mdf - - -def ubi_or_bens( - df: pd.DataFrame, - ben_cols: Union[str, List[str]], - max_ubi: str = "max_ubi", - ubi: str = "ubi", - bens: str = "bens", - update_income_measures: Optional[List[str]] = None, -) -> None: - """Calculates whether a tax unit will take UBI or benefits, and adjusts - values accordingly. - - :param df: DataFrame. - :param ben_cols: List of columns for benefits. - :param max_ubi: Column name of the maximum UBI, before accounting for - benefits. Defaults to 'max_ubi'. - :param ubi: Column name to add representing the UBI. Defaults to 'ubi'. - :param bens: Column name to add representing total benefits (after - adjustment). Defaults to 'bens'. - :param update_income_measures: List of income measures to update. Defaults - to ['expanded_income', 'aftertax_income']. - :returns: Nothing. Benefits in ben_cols are adjusted, ubi and bens columns - are added, and expanded_income and aftertax_income are updated - according to the net difference. - """ - if update_income_measures is None: - update_income_measures = ["expanded_income", "aftertax_income"] - # Prep list args. - update_income_measures = mdf.listify(update_income_measures) - total_bens = df[ben_cols].sum(axis=1) - take_ubi = df[max_ubi] > total_bens - df[ubi] = np.where(take_ubi, df[max_ubi], 0) - for ben in ben_cols: - df[ben] *= np.where(take_ubi, 0, 1) - df[bens] = df[ben_cols].sum(axis=1) - # Update expanded and aftertax income. - diff = df.ubi + df.bens - total_bens - for i in update_income_measures: - df[i] += diff diff --git a/microdf/utils.py b/microdf/utils.py deleted file mode 100644 index 9246308a..00000000 --- a/microdf/utils.py +++ /dev/null @@ -1,73 +0,0 @@ -import collections -from typing import Dict, List, Optional, Union - -import pandas as pd - - -def ordinal_label(n: int) -> str: - """Creates ordinal label from number. - - Adapted from https://stackoverflow.com/a/20007730/1840471. - - :param n: Number. - :returns: Ordinal label, e.g., 1st, 3rd, 24th, etc. - """ - n = int(n) - ix = (n / 10 % 10 != 1) * (n % 10 < 4) * n % 10 - return "%d%s" % (n, "tsnrhtdd"[ix::4]) - - -def dedup_list(lst: List) -> List: - """Remove duplicate items from a list. - - :param lst: List. - :returns: List with duplicate items removed from lst. - """ - return list(set(lst)) - - -def listify( - x: Union[str, List[str]], dedup: Optional[bool] = True -) -> List[str]: - """Return x as a list, if it isn't one already. - - :param x: A single item or a list - :param dedup: Default value = True) - :returns: x if x is a list, otherwise [x]. Also flattens the list and - removes Nones. - """ - if not isinstance(x, list): - x = [x] - res = flatten(x) - res = [x for x in res if x is not None] - if dedup: - return dedup_list(res) - return res - - -def flatten(lst: List) -> None: - """Flatten list. From https://stackoverflow.com/a/2158532/1840471. - - :param lst: List. - :returns: Flattened version. - - """ - for el in lst: - if isinstance(el, collections.abc.Iterable) and not isinstance( - el, (str, bytes) - ): - yield from flatten(el) - else: - yield el - - -def cartesian_product(d: Dict) -> pd.DataFrame: - """Produces a DataFrame as a Cartesian product of dictionary keys and - values. - - :param d: Dictionary where each item's key corresponds to a column name, - and each value is a list of values. - :returns: DataFrame with a Cartesian product of each dictionary item. - """ - index = pd.MultiIndex.from_product(d.values(), names=d.keys()) - return pd.DataFrame(index=index).reset_index() diff --git a/microdf/weighted.py b/microdf/weighted.py deleted file mode 100644 index 383b2364..00000000 --- a/microdf/weighted.py +++ /dev/null @@ -1,235 +0,0 @@ -import warnings -from typing import List, Optional, Union - -import numpy as np -import pandas as pd - -import microdf as mdf - - -def weight( - df: pd.DataFrame, col: Union[str, List[str]], w: Optional[str] = None -) -> pd.Series: - """Calculates the weighted value of a column in a DataFrame. - - :param df: A pandas DataFrame. - :param col: A string indicating the column in the DataFrame to weight. Can - also be a list of column strings. - :param w: Weight column. - :returns: A pandas Series multiplying the column by its weight. - """ - if w is None: - return df[col] - return df[col].multiply(df[w], axis="index") - - -def weighted_sum( - df: pd.DataFrame, - col: Union[str, List[str]], - w: Optional[str] = None, - groupby: Optional[str] = None, -) -> pd.Series: - """Calculates the weighted sum of a column in a DataFrame. - - :param df: A pandas DataFrame. - :param col: A string indicating the column in the DataFrame. Can also be a - list of column strings. - :param w: Weight column. - :param groupby: Groupby column. - :returns: The weighted sum of a DataFrame's column. - """ - - def _weighted_sum( - df: pd.DataFrame, col: Union[str, List[str]], w: Optional[str] - ) -> float: - """For weighted sum with provided weight.""" - return weight(df, col, w).sum() - - if groupby is None: - if w is None: - return df[col].sum() - return _weighted_sum(df, col, w) - # If grouping. - if w is None: - return df.groupby(groupby)[col].sum() - return df.groupby(groupby).apply(lambda x: _weighted_sum(x, col, w)) - - -def weighted_mean( - df: pd.DataFrame, - col: Union[str, List[str]], - w: Optional[str] = None, - groupby: Optional[str] = None, -) -> pd.Series: - """Calculates the weighted mean of a column in a DataFrame. - - :param df: A pandas DataFrame. - :param col: A string indicating the column in the DataFrame. Can also be a - list of column strings. - :param w: Weight column. - :param groupby: Groupby column. - :returns: The weighted mean of a DataFrame's column. - """ - - def _weighted_mean( - df: pd.DataFrame, col: Union[str, List[str]], w: Optional[str] - ) -> float: - """For weighted mean with provided weight.""" - return weighted_sum(df, col, w) / df[w].sum() - - if groupby is None: - if w is None: - return df[col].mean() - return _weighted_mean(df, col, w) - # Group. - if w is None: - return df.groupby(groupby)[col].mean() - return df.groupby(groupby).apply(lambda x: _weighted_mean(x, col, w)) - - -def weighted_quantile( - df: pd.DataFrame, col: str, w: str, quantiles: np.array -) -> np.array: - """Calculates weighted quantiles of a set of values. - - Doesn't exactly match unweighted quantiles of stacked values. - See stackoverflow.com/q/21844024#comment102342137_29677616. - - :param df: DataFrame to calculate weighted quantiles from. - :type df: pd.DataFrame - :param col: Name of numeric column in df to calculate weighted quantiles - from. - :type col: str - :param w: Name of weight column in df. - :type w: str - :param quantiles: Array of quantiles to calculate. - :type quantiles: np.array - :return: Array of weighted quantiles. - :rtype: np.array - """ - values = np.array(df[col]) - quantiles = np.array(quantiles) - if w is None: - sample_weight = np.ones(len(values)) - else: - sample_weight = np.array(df[w]) - assert np.all(quantiles >= 0) and np.all( - quantiles <= 1 - ), "quantiles should be in [0, 1]" - sorter = np.argsort(values) - values = values[sorter] - sample_weight = sample_weight[sorter] - weighted_quantiles = np.cumsum(sample_weight) - 0.5 * sample_weight - weighted_quantiles /= np.sum(sample_weight) - return np.interp(quantiles, weighted_quantiles, values) - - -def weighted_median( - df: pd.DataFrame, - col: Union[str, List[str]], - w: Optional[str] = None, - groupby: Optional[str] = None, -) -> pd.Series: - """Calculates the weighted median of a column in a DataFrame. - - :param df: A pandas DataFrame containing Tax-Calculator data. - :param col: A string indicating the column in the DataFrame. - :param w: Weight column. - :returns: The weighted median of a DataFrame's column. - """ - - def _weighted_median( - df: pd.DataFrame, col: Union[str, List[str]], w: Optional[str] - ) -> float: - """For weighted median with provided weight.""" - return weighted_quantile(df, col, w, 0.5) - - if groupby is None: - if w is None: - return df[col].median() - return _weighted_median(df, col, w) - # Group. - if w is None: - return df.groupby(groupby)[col].median() - return df.groupby(groupby).apply(lambda x: _weighted_median(x, col, w)) - - -def add_weighted_quantiles( - df: pd.DataFrame, col: Union[str, List[str]], w: Optional[str] = None -) -> None: - """Adds weighted quantiles of a column to a DataFrame. This will be - deprecated in the next minor release. Please use MicroSeries.rank instead. - - Adds columns for each of these types of quantiles to a DataFrame: - * *_percentile_exact: Exact percentile. - * *_percentile: Integer percentile (ceiling). - * *_2percentile: Integer percentile (ceiling, for each two percentiles). - * *_ventile: Integer percentile (ceiling, for each five percentiles). - * *_decile: Integer decile. - * *_quintile: Integer quintile. - * *_quartile: Integer quartile. - - Negative values are assigned -1. - - :param df: A pandas DataFrame. - :param col: A string indicating the column in the DataFrame to calculate. - :param w: Weight column. - :returns: Nothing. Columns are added in place. Also sorts df by col. - """ - warnings.warn( - "This will be deprecated in the next minor release. " - "Please use MicroSeries.rank instead.", - DeprecationWarning, - ) - df.sort_values(by=col, inplace=True) - col_pctile = col + "_percentile_exact" - df[col_pctile] = 100 * df[w].cumsum() / df[w].sum() - # "Null out" negatives using -1, since integer arrays can't be NaN. - df[col_pctile] = np.where(df[col] >= 0, df[col_pctile], 0) - # Reduce top record, otherwise it's incorrectly rounded up. - df[col_pctile] = np.where( - df[col_pctile] >= 99.99999, 99.99999, df[col_pctile] - ) - df[col + "_percentile"] = np.ceil(df[col_pctile]).astype(int) - df[col + "_2percentile"] = 2 * np.ceil(df[col_pctile] / 2).astype(int) - df[col + "_ventile"] = 5 * np.ceil(df[col_pctile] / 5).astype(int) - df[col + "_decile"] = np.ceil(df[col_pctile] / 10).astype(int) - df[col + "_quintile"] = np.ceil(df[col_pctile] / 20).astype(int) - df[col + "_quartile"] = np.ceil(df[col_pctile] / 25).astype(int) - - -def quantile_chg( - df1: pd.DataFrame, - df2: pd.DataFrame, - col1: str, - col2: str, - w1: Optional[str] = None, - w2: Optional[str] = None, - q: Optional[np.ndarray] = None, -) -> pd.DataFrame: - """Create table with two sets of quantiles. - - :param df1: DataFrame with first set of values. - :param df2: DataFrame with second set of values. - :param col1: Name of columns with values in df1. - :param col2: Name of columns with values in df2. - :param w1: Name of weight column in df1. - :param w2: Name of weight column in df2. - :param q: Quantiles. Defaults to decile boundaries. - :returns: DataFrame with two rows and a column for each quantile. Column - labels are "xth percentile" and a label is added to the median. - """ - if q is None: - q = np.arange(0.1, 1, 0.1) - q1 = weighted_quantile(df1, col1, w1, q) - q2 = weighted_quantile(df2, col2, w2, q) - qdf = pd.DataFrame([q1, q2]) - # Set decile labels. - q_print = [mdf.ordinal_label((i * 100)) for i in q] - try: # List index throws an error if the value is not found. - median_index = q.tolist().index(0.5) - q_print[median_index] += " (median)" - except ValueError: - pass # Don't assign median to any label. - qdf.columns = q_print - return qdf From 9063cbd25c022ef9d059516100a74d09dfe25c1a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 12:55:41 -0400 Subject: [PATCH 2/6] Fix CI issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update gini.ipynb to use MicroSeries.gini() instead of removed mdf.gini() - Add missing newlines to __init__.py and changelog_entry.yaml 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- changelog_entry.yaml | 2 +- docs/gini.ipynb | 196 +++++++------------------------------------ microdf/__init__.py | 2 +- 3 files changed, 32 insertions(+), 168 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index f324d094..6473c3c3 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -6,4 +6,4 @@ - Remove agg, concat, constants, custom_taxes, income_measures, inequality (standalone functions), io, poverty (standalone functions), tax, ubi, utils, and weighted modules. - Remove _optional module as it's no longer needed. - Remove associated test files for deleted modules. - - Simplify package to focus on core weighted data structures used by PolicyEngine. \ No newline at end of file + - Simplify package to focus on core weighted data structures used by PolicyEngine. diff --git a/docs/gini.ipynb b/docs/gini.ipynb index 160ea6c2..7b5bffd3 100644 --- a/docs/gini.ipynb +++ b/docs/gini.ipynb @@ -4,27 +4,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# `gini` example" + "# `gini` example using MicroSeries" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import microdf as mdf\n", - "\n", - "import pandas as pd" + "import pandas as pd\n", + "import numpy as np" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "x = [-10, -1, 0, 5, 100]\n", + "# Create sample data\n", + "x = [10, 20, 30, 40, 100]\n", "w = [1, 2, 3, 4, 5]\n", "df = pd.DataFrame({'x': x, 'w': w})" ] @@ -33,180 +34,56 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Simple behavior" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.9617021276595745" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mdf.gini(df, 'x')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dealing with negatives" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This will be equivalent to `mdf.gini(pd.DataFrame({'x': [0, 0, 0, 5, 100]}))`." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.780952380952381" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mdf.gini(df, 'x', negatives='zero')" + "## Using MicroSeries.gini()" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.780952380952381" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "mdf.gini(pd.DataFrame({'x': [0, 0, 0, 5, 100]}), 'x')" + "# Create a MicroSeries with weights\n", + "ms = mdf.MicroSeries(df.x, weights=df.w)\n", + "print(f\"Gini coefficient: {ms.gini():.4f}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This will be equivalent to `mdf.gini(pd.DataFrame({'x': [0, 9, 10, 15, 110]}))`." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.6277777777777778" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mdf.gini(df, 'x', negatives='shift')" + "## Without weights" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.6277777777777778" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "mdf.gini(pd.DataFrame({'x': [0, 9, 10, 15, 110]}), 'x')" + "# Create a MicroSeries without weights (equal weights)\n", + "ms_unweighted = mdf.MicroSeries(df.x)\n", + "print(f\"Unweighted Gini coefficient: {ms_unweighted.gini():.4f}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Dealing with weights" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.6800524934383202" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mdf.gini(df, 'x', 'w')" + "## Working with MicroDataFrame" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.6800524934383202" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "mdf.gini(pd.DataFrame({'x': [-10,\n", - " -1, -1,\n", - " 0, 0, 0,\n", - " 5, 5, 5, 5,\n", - " 100, 100, 100, 100, 100]}),\n", - " 'x')" + "# Create a MicroDataFrame\n", + "mdf_df = mdf.MicroDataFrame(df, weights='w')\n", + "\n", + "# Access column as MicroSeries and calculate gini\n", + "print(f\"Gini from MicroDataFrame column: {mdf_df.x.gini():.4f}\")" ] } ], @@ -226,22 +103,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false + "version": "3.7.7" } }, "nbformat": 4, - "nbformat_minor": 2 -} + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/microdf/__init__.py b/microdf/__init__.py index 75612a42..6667956d 100644 --- a/microdf/__init__.py +++ b/microdf/__init__.py @@ -11,4 +11,4 @@ # microdataframe.py "MicroDataFrame", "MicroDataFrameGroupBy", -] \ No newline at end of file +] From ed14c0d31a846baecccdfb4253b460b9d6c8e232 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 12:59:01 -0400 Subject: [PATCH 3/6] Apply black formatting to test file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- microdf/tests/test_microseries_dataframe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/microdf/tests/test_microseries_dataframe.py b/microdf/tests/test_microseries_dataframe.py index 9d259664..6a22d83c 100644 --- a/microdf/tests/test_microseries_dataframe.py +++ b/microdf/tests/test_microseries_dataframe.py @@ -105,8 +105,6 @@ def test_multiple_groupby() -> None: assert (df.groupby(["x", "y"]).z.sum() == np.array([5, 6])).all() - - def test_set_index() -> None: d = mdf.MicroDataFrame(dict(x=[1, 2, 3]), weights=[4, 5, 6]) assert d.x.__class__ == MicroSeries From 77f3a337cf1c9e6e44fcbe4bd0b6193911b3a246 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 14:25:52 -0400 Subject: [PATCH 4/6] Update pyproject.toml and README for v1.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump version to 1.0.0 for major breaking change - Update package description to focus on weighted DataFrames/Series - Rewrite README with clearer overview and usage examples - Update badges to point to PolicyEngine organization - Update maintainer email to max@policyengine.org 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- README.md | 43 +++++++++++++++++++++++++++++++++++++------ changelog_entry.yaml | 4 ++++ pyproject.toml | 4 ++-- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8afd3d93..402b3c23 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,49 @@ -[![Build](https://github.com/PSLmodels/microdf/workflows/Build%20and%20test%20[Python%203.7,%203.8,%203.9]/badge.svg)](https://github.com/PSLmodels/microdf/actions?query=workflow%3A%22Build+and+test+%5BPython+3.7%2C+3.8%2C+3.9%5D%22) -[![Codecov](https://codecov.io/gh/PSLmodels/microdf/branch/master/graph/badge.svg)](https://codecov.io/gh/PSLmodels/microdf) +[![Build](https://github.com/PolicyEngine/microdf/workflows/Build%20and%20test%20[Python%203.9+]/badge.svg)](https://github.com/PolicyEngine/microdf/actions) +[![Codecov](https://codecov.io/gh/PolicyEngine/microdf/branch/master/graph/badge.svg)](https://codecov.io/gh/PolicyEngine/microdf) # microdf -Analysis tools for working with survey microdata as DataFrames. +Weighted pandas DataFrames and Series for survey microdata analysis. -*Disclaimer: `MicroSeries` and `MicroDataFrame` are experimental features and may not consider weights after performing some operations. See open issues.* +## Overview +microdf provides `MicroDataFrame` and `MicroSeries` classes that extend pandas functionality with integrated weighting support, essential for accurate survey data analysis. + +## Key Features +- **MicroDataFrame**: A pandas DataFrame with an integrated weight column +- **MicroSeries**: A pandas Series with integrated weights +- **Weighted operations**: All aggregations (sum, mean, median, etc.) automatically use weights +- **Inequality metrics**: Built-in Gini coefficient calculation +- **Poverty analysis**: Integrated poverty rate and gap calculations ## Installation Install with: - pip install git+git://github.com/PSLmodels/microdf.git + pip install microdf-python + +Or for development: + + pip install git+https://github.com/PolicyEngine/microdf.git + +## Usage +```python +import microdf as mdf +import pandas as pd + +# Create sample data with weights +df = pd.DataFrame({ + 'income': [10_000, 20_000, 30_000, 40_000, 50_000], + 'weights': [1, 2, 3, 2, 1] +}) + +# Create a MicroDataFrame +mdf_df = mdf.MicroDataFrame(df, weights='weights') + +# All operations are weight-aware +print(mdf_df.income.mean()) # Weighted mean +print(mdf_df.income.gini()) # Gini coefficient +``` ## Questions -Contact the maintainer, Max Ghenis (mghenis@gmail.com). +Contact the maintainer, Max Ghenis (max@policyengine.org). ## Citation You may cite the source of your analysis as "microdf release #.#.#, author's calculations." diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 6473c3c3..df3535d8 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -7,3 +7,7 @@ - Remove _optional module as it's no longer needed. - Remove associated test files for deleted modules. - Simplify package to focus on core weighted data structures used by PolicyEngine. + changed: + - Update package description to reflect focused scope on weighted DataFrames and Series. + - Update README with clearer documentation and usage examples. + - Version bumped to 1.0.0 to reflect major breaking changes. diff --git a/pyproject.toml b/pyproject.toml index 730d755b..642f428a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "microdf-python" -version = "0.4.5" -description = "Survey microdata as DataFrames" +version = "1.0.0" +description = "Weighted pandas DataFrames and Series for survey microdata" readme = "README.md" authors = [ { name = "Max Ghenis", email = "max@ubicenter.org" } From 336c99139a942535f7751dc4c27237a3a751f155 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 14:35:10 -0400 Subject: [PATCH 5/6] Remove pip from dev dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's a standard Python tool that doesn't need to be explicitly listed. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- changelog_entry.yaml | 1 + pyproject.toml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index df3535d8..73e723d2 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -11,3 +11,4 @@ - Update package description to reflect focused scope on weighted DataFrames and Series. - Update README with clearer documentation and usage examples. - Version bumped to 1.0.0 to reflect major breaking changes. + - Remove pip from dev dependencies as it's not needed. diff --git a/pyproject.toml b/pyproject.toml index 642f428a..1128d3e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ dev = [ "docformatter", "isort", "linecheck", - "pip", "pytest", "pytest-cov", "setuptools", From 447dfa4a78cc2c3af74be78c348e061a549f4b55 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 15:02:05 -0400 Subject: [PATCH 6/6] Update to Python 3.13 as main version, support 3.9+ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update black target-version to py313 - Update all CI workflows to use Python 3.13 as main version - Test against Python 3.9, 3.10, 3.11, 3.12, and 3.13 in CI - Remove taxcalc from CI install commands (already removed) - Python 3.13 is the current bugfix version (released Oct 2023) - Python 3.9 is the oldest version with security support This change is enabled by the removal of taxcalc dependency. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/master.yml | 10 +++++----- .github/workflows/pr.yaml | 10 +++++----- README.md | 2 +- changelog_entry.yaml | 2 ++ pyproject.toml | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5e075291..8b49f6ab 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -14,14 +14,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@v3 with: version: "latest" - name: Install dependencies run: | - uv pip install -e ".[dev,taxcalc]" --system + uv pip install -e ".[dev]" --system - name: Run tests run: make test @@ -37,14 +37,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@v3 with: version: "latest" - name: Install dependencies run: | - uv pip install -e ".[dev,docs,taxcalc]" --system + uv pip install -e ".[dev,docs]" --system - name: Build Jupyter Book shell: bash -l {0} run: | @@ -77,7 +77,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@v3 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 637f8fda..e838769f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@v3 with: @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: version: "latest" - name: Install dependencies run: | - uv pip install -e ".[dev,taxcalc]" --system + uv pip install -e ".[dev]" --system - name: Run tests with coverage run: make test - name: Upload coverage to Codecov @@ -64,14 +64,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' - name: Install uv uses: astral-sh/setup-uv@v3 with: version: "latest" - name: Install dependencies run: | - uv pip install -e ".[dev,docs,taxcalc]" --system + uv pip install -e ".[dev,docs]" --system - name: Build Jupyter Book run: | jb build docs/. diff --git a/README.md b/README.md index 402b3c23..366a06d7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build](https://github.com/PolicyEngine/microdf/workflows/Build%20and%20test%20[Python%203.9+]/badge.svg)](https://github.com/PolicyEngine/microdf/actions) +[![Build](https://github.com/PolicyEngine/microdf/workflows/Pull%20request/badge.svg)](https://github.com/PolicyEngine/microdf/actions) [![Codecov](https://codecov.io/gh/PolicyEngine/microdf/branch/master/graph/badge.svg)](https://codecov.io/gh/PolicyEngine/microdf) # microdf diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 73e723d2..daa140ec 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -12,3 +12,5 @@ - Update README with clearer documentation and usage examples. - Version bumped to 1.0.0 to reflect major breaking changes. - Remove pip from dev dependencies as it's not needed. + - Update to Python 3.13 as main version, supporting Python 3.9+. + - Test against Python 3.9, 3.10, 3.11, 3.12, and 3.13 in CI. diff --git a/pyproject.toml b/pyproject.toml index 1128d3e1..c09a97c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ line_length = 79 [tool.black] line-length = 79 -target-version = ["py311"] +target-version = ["py313"] [tool.flake8] max-line-length = 79