|
| 1 | +""" |
| 2 | +Missing values in a formula are dropped, whatever block of the formula has them. |
| 3 | +
|
| 4 | +The dependent variable, the exogenous regressors, the endogenous regressors and the |
| 5 | +instruments of a model made from a formula can have missing values, as the arrays that |
| 6 | +the models use. The observations are dropped with a MissingValueWarning, and the results |
| 7 | +are those of the model fit to the data after the rows with missing values were dropped. |
| 8 | +""" |
| 9 | + |
| 10 | +import numpy as np |
| 11 | +from numpy.testing import assert_allclose, assert_array_equal |
| 12 | +import pandas as pd |
| 13 | +from pandas.testing import assert_index_equal |
| 14 | +import pytest |
| 15 | + |
| 16 | +from linearmodels.iv import IV2SLS, IVGMM, IVGMMCUE, IVJIVE, IVLIML |
| 17 | +from linearmodels.shared.exceptions import MissingValueWarning |
| 18 | + |
| 19 | +MODELS = [IV2SLS, IVLIML, IVGMM, IVGMMCUE, IVJIVE] |
| 20 | +FORMULA = "y ~ 1 + x + w + [e ~ z1 + z2]" |
| 21 | +NOBS = 150 |
| 22 | +ROWS = [3, 40] |
| 23 | +BLOCKS = {"dependent": "y", "exog": "x", "endog": "e", "instruments": "z1"} |
| 24 | + |
| 25 | + |
| 26 | +@pytest.fixture(scope="module") |
| 27 | +def frame(): |
| 28 | + rng = np.random.default_rng(0) |
| 29 | + df = pd.DataFrame( |
| 30 | + rng.standard_normal((NOBS, 6)), |
| 31 | + columns=["y", "x", "w", "e", "z1", "z2"], |
| 32 | + index=pd.Index([f"obs{i}" for i in range(NOBS)]), |
| 33 | + ) |
| 34 | + df["e"] = df["e"] + df["z1"] + df["z2"] |
| 35 | + df["y"] = df["y"] + df["e"] + df["x"] |
| 36 | + df["weight"] = np.exp(0.3 * rng.standard_normal(NOBS)) |
| 37 | + return df |
| 38 | + |
| 39 | + |
| 40 | +def assert_same_fit(res, expected, cls): |
| 41 | + # The estimates of the continuously updated GMM come from an optimizer, and the |
| 42 | + # optimizer sees data in another memory layout |
| 43 | + rtol = 1e-5 if cls is IVGMMCUE else 1e-10 |
| 44 | + assert res.nobs == expected.nobs |
| 45 | + assert_index_equal(res.resids.index, expected.resids.index) |
| 46 | + assert_allclose(res.params, expected.params, rtol=rtol) |
| 47 | + assert_allclose(res.cov, expected.cov, rtol=rtol * 100) |
| 48 | + assert_allclose(res.resids, expected.resids, rtol=rtol * 100, atol=1e-12) |
| 49 | + |
| 50 | + |
| 51 | +@pytest.mark.parametrize("cls", MODELS) |
| 52 | +@pytest.mark.parametrize("block", list(BLOCKS)) |
| 53 | +def test_missing_value_in_each_block(frame, cls, block): |
| 54 | + data = frame.copy() |
| 55 | + data.iloc[ROWS, data.columns.get_loc(BLOCKS[block])] = np.nan |
| 56 | + with pytest.warns(MissingValueWarning, match="missing values"): |
| 57 | + mod = cls.from_formula(FORMULA, data) |
| 58 | + assert mod.notnull.sum() == NOBS - 2 |
| 59 | + assert not mod.notnull[ROWS].any() |
| 60 | + expected = cls.from_formula(FORMULA, frame.drop(index=frame.index[ROWS])).fit() |
| 61 | + assert_same_fit(mod.fit(), expected, cls) |
| 62 | + |
| 63 | + |
| 64 | +@pytest.mark.parametrize("cls", MODELS) |
| 65 | +def test_missing_values_in_every_block_in_different_rows(frame, cls): |
| 66 | + data = frame.copy() |
| 67 | + rows = {"y": 3, "x": 10, "e": 20, "z2": 30, "w": 31} |
| 68 | + for col, row in rows.items(): |
| 69 | + data.iloc[row, data.columns.get_loc(col)] = np.nan |
| 70 | + with pytest.warns(MissingValueWarning, match="missing values"): |
| 71 | + mod = cls.from_formula(FORMULA, data) |
| 72 | + keep = np.ones(NOBS, dtype=bool) |
| 73 | + keep[list(rows.values())] = False |
| 74 | + assert_array_equal(mod.notnull, keep) |
| 75 | + expected = cls.from_formula(FORMULA, frame[keep]).fit() |
| 76 | + assert_same_fit(mod.fit(), expected, cls) |
| 77 | + |
| 78 | + |
| 79 | +@pytest.mark.parametrize("cls", [IV2SLS, IVLIML, IVGMM]) |
| 80 | +@pytest.mark.parametrize("block", list(BLOCKS)) |
| 81 | +def test_missing_values_with_weights(frame, cls, block): |
| 82 | + data = frame.copy() |
| 83 | + data.iloc[ROWS, data.columns.get_loc(BLOCKS[block])] = np.nan |
| 84 | + with pytest.warns(MissingValueWarning, match="missing values"): |
| 85 | + mod = cls.from_formula(FORMULA, data, weights=data["weight"]) |
| 86 | + clean = frame.drop(index=frame.index[ROWS]) |
| 87 | + expected = cls.from_formula(FORMULA, clean, weights=clean["weight"]).fit() |
| 88 | + res = mod.fit() |
| 89 | + assert res.nobs == NOBS - 2 |
| 90 | + assert_allclose(res.params, expected.params, rtol=1e-10) |
| 91 | + assert_allclose(res.cov, expected.cov, rtol=1e-8) |
| 92 | + |
| 93 | + |
| 94 | +def test_no_missing_values_do_not_warn(frame, recwarn): |
| 95 | + IV2SLS.from_formula(FORMULA, frame).fit() |
| 96 | + assert not [w for w in recwarn if issubclass(w.category, MissingValueWarning)] |
| 97 | + |
| 98 | + |
| 99 | +def test_all_values_missing(frame): |
| 100 | + data = frame.copy() |
| 101 | + data["y"] = np.nan |
| 102 | + with pytest.raises(ValueError, match="All observations contain missing data"): |
| 103 | + IV2SLS.from_formula(FORMULA, data) |
| 104 | + |
| 105 | + |
| 106 | +@pytest.mark.parametrize("block", ["x", "e"]) |
| 107 | +def test_predict_with_missing_values(frame, block): |
| 108 | + res = IV2SLS.from_formula(FORMULA, frame).fit() |
| 109 | + new = frame.iloc[:7].copy() |
| 110 | + new.iloc[2, new.columns.get_loc(block)] = np.nan |
| 111 | + pred = res.model.predict(res.params, data=new) |
| 112 | + assert pred.shape == (7, 1) |
| 113 | + assert np.isnan(pred.iloc[2, 0]) |
| 114 | + assert pred.drop(index=new.index[2]).notnull().all().all() |
| 115 | + # The results drop the predictions that are missing, unless missing is True |
| 116 | + assert res.predict(data=new).shape == (6, 1) |
| 117 | + kept = res.predict(data=new, missing=True) |
| 118 | + assert kept.shape == (7, 1) |
| 119 | + assert_allclose(kept.drop(index=new.index[2]), pred.drop(index=new.index[2])) |
| 120 | + complete = res.predict(data=frame.iloc[:7]) |
| 121 | + assert_allclose( |
| 122 | + complete.drop(index=new.index[2]), pred.drop(index=new.index[2]), rtol=1e-12 |
| 123 | + ) |
0 commit comments