diff --git a/doc/source/utility.rst b/doc/source/utility.rst index 3be545bc5c..4828388841 100644 --- a/doc/source/utility.rst +++ b/doc/source/utility.rst @@ -43,3 +43,65 @@ Covariance Estimation cov_cluster cov_kernel group_debias_coefficient + + +Missing Values +-------------- + +.. currentmodule:: linearmodels.shared.exceptions + +A model drops the observations that have a missing value, such as ``NaN``, in any of +its inputs when it is created. It issues a :class:`MissingValueWarning` if it drops any +observation, and the results use the remaining observations. This is the same in every +model. + +The warning is on by default. It can be turned off in three ways. + +* Set ``linearmodels.WARN_ON_MISSING`` to ``False``. It applies to every model that is + created afterwards. + + .. code-block:: python + + import linearmodels + + linearmodels.WARN_ON_MISSING = False + +* Set the environment variable ``LINEARMODELS_WARN_ON_MISSING`` before importing + linearmodels, for example in the shell that starts Python. The values ``""`` (the + empty string), ``"0"``, ``"false"`` and ``"False"`` turn the warning off. Any other + value, or no variable, leaves it on. The variable is read when linearmodels is + imported, so setting it later has no effect. + + .. code-block:: bash + + export LINEARMODELS_WARN_ON_MISSING=0 + +* Use a warnings filter. A filter applies only where it is set, so it can turn off the + warning for part of a program. The warning is issued when the model is created, not + when it is fit. + + .. code-block:: python + + import warnings + + from linearmodels.iv import IV2SLS + from linearmodels.shared.exceptions import MissingValueWarning + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", MissingValueWarning) + mod = IV2SLS(dependent, exog, endog, instruments) + res = mod.fit() + + The action ``"error"`` instead of ``"ignore"`` raises an exception if a model drops + any observation. + +.. deprecated:: 7.1 + + The module attribute ``linearmodels.DROP_MISSING`` and the environment variable + ``LINEARMODELS_DROP_MISSING`` have never had an effect, because observations with + missing values are always dropped. Reading ``linearmodels.DROP_MISSING``, and + importing linearmodels when the environment variable is set, issue a + ``FutureWarning``. Both will be removed in linearmodels 8.0. Use + ``WARN_ON_MISSING`` to turn off the warning. + +.. autoclass:: linearmodels.shared.exceptions.MissingValueWarning diff --git a/linearmodels/__init__.py b/linearmodels/__init__.py index 32d53c8791..bac783ac07 100644 --- a/linearmodels/__init__.py +++ b/linearmodels/__init__.py @@ -37,12 +37,36 @@ - System Estimation using Generalized Method of Moments (GMM) Designed to work equally well with NumPy, Pandas or xarray data. + +Notes +----- +A model drops the observations that have a missing value in any of its inputs when +it is created, and issues a ``MissingValueWarning`` if it drops any. The warning +can be turned off in three ways: + +- Set ``linearmodels.WARN_ON_MISSING = False``. It applies to every model that is + created afterwards. +- Set the environment variable ``LINEARMODELS_WARN_ON_MISSING`` before importing + linearmodels. The values ``""``, ``"0"``, ``"false"`` and ``"False"`` turn the + warning off. Any other value leaves it on. +- Use a warnings filter, for example + ``warnings.simplefilter("ignore", MissingValueWarning)``, where + ``MissingValueWarning`` is imported from ``linearmodels.shared.exceptions``. The + filter applies where it is set, for example inside ``warnings.catch_warnings()``. + +``linearmodels.DROP_MISSING`` and the environment variable +``LINEARMODELS_DROP_MISSING`` have never had any effect, because observations with +missing values are always dropped. They are deprecated and will be removed in +linearmodels 8.0. Reading ``DROP_MISSING``, and importing linearmodels when the +environment variable is set, issue a ``FutureWarning``. """ from __future__ import annotations import os import sys +from typing import TYPE_CHECKING +import warnings from ._version import version as __version__, version_tuple from .asset_pricing.model import ( @@ -65,11 +89,22 @@ OLS = _OLS _WARN_ON_MISSING = os.environ.get("LINEARMODELS_WARN_ON_MISSING", "1") WARN_ON_MISSING = False if _WARN_ON_MISSING in ("", "0", "false", "False") else True -_DROP_MISSING = os.environ.get("LINEARMODELS_DROP_MISSING", "1") -DROP_MISSING = False if _DROP_MISSING in ("", "0", "false", "False") else True + +# DROP_MISSING and LINEARMODELS_DROP_MISSING have never had an effect. They are +# deprecated and will be removed in 8.0, together with __getattr__ and the tests of +# the deprecation in tests/test_deprecations.py. +_DROP_MISSING = os.environ.get("LINEARMODELS_DROP_MISSING") +if _DROP_MISSING is not None: + warnings.warn( + "The environment variable LINEARMODELS_DROP_MISSING has no effect and will " + "be removed in linearmodels 8.0. Observations with missing values are always " + "dropped. Use LINEARMODELS_WARN_ON_MISSING to turn off the " + "MissingValueWarning.", + FutureWarning, + stacklevel=2, + ) __all__ = [ - "DROP_MISSING", "IV2SLS", "IV3SLS", "IVGMM", @@ -94,6 +129,30 @@ "version_tuple", ] +if TYPE_CHECKING: + # Type checkers must still see DROP_MISSING, and must not accept any other name + # because of the __getattr__ below + DROP_MISSING: bool +else: + + def __getattr__(name: str) -> bool: + # Python calls this function only for names that the module does not define. + # DROP_MISSING is deliberately not defined, so that reading it warns, and it + # is not in __all__, so that import * does not read it. Assigning to + # linearmodels.DROP_MISSING defines it, and then it does not warn. + if name == "DROP_MISSING": + warnings.warn( + "linearmodels.DROP_MISSING has no effect and will be removed in " + "linearmodels 8.0. Observations with missing values are always " + "dropped. Use linearmodels.WARN_ON_MISSING to turn off the " + "MissingValueWarning.", + FutureWarning, + stacklevel=2, + ) + # What 7.0 gave: True, unless the environment variable turned it off + return _DROP_MISSING not in ("", "0", "false", "False") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + def test( extra_args: str | list[str] | None = None, diff --git a/linearmodels/shared/exceptions.py b/linearmodels/shared/exceptions.py index 70744758db..318ff50d91 100644 --- a/linearmodels/shared/exceptions.py +++ b/linearmodels/shared/exceptions.py @@ -6,7 +6,18 @@ class MissingValueWarning(Warning): - pass + """ + Warning that observations with missing values were dropped. + + Notes + ----- + A model drops the observations that have a missing value in any of its inputs + when it is created, and issues this warning if it drops any. The warning is on + by default. Set ``linearmodels.WARN_ON_MISSING = False`` or the environment + variable ``LINEARMODELS_WARN_ON_MISSING=0`` to turn it off for every model. To + change it for part of a program, use a warnings filter for this class, with the + action ``"ignore"`` or ``"error"``. + """ missing_value_warning_msg = """ diff --git a/linearmodels/tests/shared/test_missing_values.py b/linearmodels/tests/shared/test_missing_values.py new file mode 100644 index 0000000000..6cc1c9688a --- /dev/null +++ b/linearmodels/tests/shared/test_missing_values.py @@ -0,0 +1,229 @@ +""" +Tests of how the models treat missing values, and of how to turn off the warning + +A model drops the observations that have a missing value in any of its inputs and +issues a MissingValueWarning. The warning is controlled by +linearmodels.WARN_ON_MISSING, by the environment variable +LINEARMODELS_WARN_ON_MISSING, which sets it when linearmodels is imported, and by +warnings filters. +""" + +import json +import os +import subprocess +import sys +import warnings + +import numpy as np +import pandas as pd +import pytest + +import linearmodels +from linearmodels import ( + IV2SLS, + SUR, + AbsorbingLS, + FamaMacBeth, + LinearFactorModel, + PanelOLS, +) +from linearmodels.shared.exceptions import MissingValueWarning + +NOBS = 120 +# Each model has a missing value in three different rows +MISSING_ROWS = 3 +MESSAGE = "Dropping rows with missing observations" + + +def iv_model(rng, missing): + instruments = rng.standard_normal((NOBS, 2)) + endog = instruments @ [1.0, 0.5] + rng.standard_normal(NOBS) + exog = np.column_stack([np.ones(NOBS), rng.standard_normal(NOBS)]) + y = exog @ [1.0, 0.5] + endog + rng.standard_normal(NOBS) + if missing: + y[3] = np.nan + endog[40] = np.nan + instruments[77, 0] = np.nan + return IV2SLS(y, exog, endog, instruments) + + +def absorbing_model(rng, missing): + cats = pd.DataFrame({"a": pd.Categorical(rng.integers(0, 6, NOBS))}) + x = rng.standard_normal((NOBS, 2)) + weights = rng.uniform(0.5, 1.5, NOBS) + y = x @ [1.0, 0.5] + rng.standard_normal(NOBS) + if missing: + y[3] = np.nan + x[40, 1] = np.nan + weights[77] = np.nan + return AbsorbingLS(y, x, absorb=cats, weights=weights) + + +def panel_data(rng, missing): + index = pd.MultiIndex.from_product([range(12), range(10)], names=["i", "t"]) + x = pd.DataFrame(rng.standard_normal((NOBS, 2)), index=index, columns=["a", "b"]) + y = pd.Series(x.to_numpy() @ [1.0, 0.5] + rng.standard_normal(NOBS), index=index) + if missing: + y.iloc[3] = np.nan + x.iloc[40, 0] = np.nan + x.iloc[77, 1] = np.nan + return y, x + + +def panel_model(rng, missing): + return PanelOLS(*panel_data(rng, missing), entity_effects=True) + + +def fama_macbeth_model(rng, missing): + return FamaMacBeth(*panel_data(rng, missing)) + + +def system_model(rng, missing): + y = rng.standard_normal((2, NOBS)) + x = rng.standard_normal((2, NOBS, 2)) + if missing: + y[0, 3] = np.nan + x[1, 40, 0] = np.nan + y[1, 77] = np.nan + equations = {f"eq{i}": {"dependent": y[i], "exog": x[i]} for i in range(2)} + return SUR(equations) + + +def factor_model(rng, missing): + factors = rng.standard_normal((NOBS, 2)) + portfolios = factors @ rng.standard_normal((2, 4)) + rng.standard_normal((NOBS, 4)) + if missing: + portfolios[3, 1] = np.nan + factors[40, 0] = np.nan + portfolios[77, 3] = np.nan + return LinearFactorModel(portfolios, factors) + + +MODELS = { + "IV2SLS": iv_model, + "AbsorbingLS": absorbing_model, + "PanelOLS": panel_model, + "FamaMacBeth": fama_macbeth_model, + "SUR": system_model, + "LinearFactorModel": factor_model, +} + + +def build(name, missing=True): + rng = np.random.default_rng(20261012) + return MODELS[name](rng, missing) + + +def nobs_after_dropping(name, missing=True): + """The observations that are left, which the system results sum over equations""" + nobs = NOBS - MISSING_ROWS if missing else NOBS + return 2 * nobs if name == "SUR" else nobs + + +@pytest.fixture(autouse=True) +def warn_on_missing(monkeypatch): + # The tests do not depend on the environment variable, and the setting is + # restored after each test + monkeypatch.setattr(linearmodels, "WARN_ON_MISSING", True) + + +@pytest.mark.parametrize("name", MODELS) +def test_missing_values_are_dropped_with_a_warning(name): + with pytest.warns(MissingValueWarning, match=MESSAGE): + model = build(name) + assert model.fit().nobs == nobs_after_dropping(name) + + +@pytest.mark.parametrize("name", MODELS) +def test_no_warning_if_nothing_is_missing(name): + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingValueWarning) + model = build(name, missing=False) + assert model.fit().nobs == nobs_after_dropping(name, missing=False) + + +@pytest.mark.parametrize("name", MODELS) +def test_warn_on_missing_false_turns_the_warning_off(name, monkeypatch): + monkeypatch.setattr(linearmodels, "WARN_ON_MISSING", False) + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingValueWarning) + model = build(name) + # The observations are dropped as before + assert model.fit().nobs == nobs_after_dropping(name) + + +@pytest.mark.parametrize("name", MODELS) +def test_warnings_filter_ignore(name): + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + warnings.simplefilter("ignore", MissingValueWarning) + model = build(name) + assert not [w for w in record if issubclass(w.category, MissingValueWarning)] + assert model.fit().nobs == nobs_after_dropping(name) + + +@pytest.mark.parametrize("name", MODELS) +def test_warnings_filter_error(name): + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingValueWarning) + with pytest.raises(MissingValueWarning, match=MESSAGE): + build(name) + + +# Imports linearmodels with each of the values in argv[1] of the environment variable +# in turn, and prints WARN_ON_MISSING for each as JSON. A null value means that the +# variable is not set. +ENVIRONMENT_SCRIPT = """ +import importlib +import json +import os +import sys + +import linearmodels + +result = {} +for value in json.loads(sys.argv[1]): + if value is None: + os.environ.pop("LINEARMODELS_WARN_ON_MISSING", None) + else: + os.environ["LINEARMODELS_WARN_ON_MISSING"] = value + importlib.reload(linearmodels) + result[json.dumps(value)] = linearmodels.WARN_ON_MISSING +print(json.dumps(result)) +""" + +# The warning is off for the values "", "0", "false" and "False", and on for any other +# value and if the variable is not set +WARN_ON_MISSING_FROM_ENVIRONMENT = { + None: True, + "": False, + "0": False, + "false": False, + "False": False, + "1": True, + "true": True, + "True": True, + "FALSE": True, + "no": True, + "off": True, +} + + +def test_warn_on_missing_environment_variable(): + env = {k: v for k, v in os.environ.items() if not k.startswith("LINEARMODELS_")} + values = json.dumps(list(WARN_ON_MISSING_FROM_ENVIRONMENT)) + cmd = [sys.executable, "-c", ENVIRONMENT_SCRIPT, values] + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=300, check=False, env=env + ) + assert proc.returncode == 0, proc.stderr + result = json.loads(proc.stdout.strip().splitlines()[-1]) + expected = {json.dumps(k): v for k, v in WARN_ON_MISSING_FROM_ENVIRONMENT.items()} + assert result == expected + + +def test_environment_variable_is_read_when_linearmodels_is_imported(monkeypatch): + # Setting it later has no effect + monkeypatch.setenv("LINEARMODELS_WARN_ON_MISSING", "0") + with pytest.warns(MissingValueWarning, match=MESSAGE): + build("IV2SLS") diff --git a/linearmodels/tests/test_deprecations.py b/linearmodels/tests/test_deprecations.py index 7155d0618b..c8c5473792 100644 --- a/linearmodels/tests/test_deprecations.py +++ b/linearmodels/tests/test_deprecations.py @@ -5,17 +5,23 @@ """ import importlib +import json +import os from pathlib import Path import re import subprocess import sys +import warnings import numpy as np from numpy.testing import assert_allclose import pandas as pd import pytest +import linearmodels +from linearmodels.iv import IV2SLS from linearmodels.iv.absorbing import AbsorbingLS +from linearmodels.shared.exceptions import MissingValueWarning LSMR_OPTIONS_MESSAGE = ( "lsmr_options is deprecated and will be removed in linearmodels 8.0" @@ -25,6 +31,13 @@ "linearmodels.__future__.ordering has no effect and will be removed in " "linearmodels 8.0" ) +DROP_MISSING_MESSAGE = ( + "linearmodels.DROP_MISSING has no effect and will be removed in linearmodels 8.0" +) +DROP_MISSING_ENV_MESSAGE = ( + "The environment variable LINEARMODELS_DROP_MISSING has no effect and will be " + "removed in linearmodels 8.0" +) @pytest.fixture @@ -127,3 +140,198 @@ def test_ordering_warns_in_a_new_interpreter(): proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300, check=False) assert proc.returncode == 0, proc.stderr assert f":2: FutureWarning: {ORDERING_MESSAGE}" in proc.stderr + + +@pytest.fixture +def restore_drop_missing(): + """Leave DROP_MISSING as it was: undefined, so that reading it warns""" + namespace = vars(linearmodels) + defined = "DROP_MISSING" in namespace + original = namespace.get("DROP_MISSING") + yield + if defined: + namespace["DROP_MISSING"] = original + else: + namespace.pop("DROP_MISSING", None) + + +def test_drop_missing_warns_when_it_is_read(): + with pytest.warns(FutureWarning, match=re.escape(DROP_MISSING_MESSAGE)) as record: + value = linearmodels.DROP_MISSING + # What 7.0 gave: the setting of the environment variable, True if it is not set + legacy = os.environ.get("LINEARMODELS_DROP_MISSING", "1") + assert value is (legacy not in ("", "0", "false", "False")) + matches = [w for w in record if DROP_MISSING_MESSAGE in str(w.message)] + assert len(matches) == 1 + # The warning points at the line that reads it + assert Path(matches[0].filename).resolve() == Path(__file__).resolve() + assert "WARN_ON_MISSING" in str(matches[0].message) + + +def test_drop_missing_warns_when_it_is_imported(): + # Run the statement as if it was in another file, on line 2, to check where the + # warning points + code = compile("\nfrom linearmodels import DROP_MISSING\n", "importing.py", "exec") + with pytest.warns(FutureWarning, match=re.escape(DROP_MISSING_MESSAGE)) as record: + exec(code, {}) # noqa: S102 + matches = [w for w in record if DROP_MISSING_MESSAGE in str(w.message)] + assert matches + assert all(w.filename == "importing.py" and w.lineno == 2 for w in matches) + + +def test_drop_missing_is_not_exported(): + # A name in __all__ would be read, and so warn, by import * + assert "DROP_MISSING" not in linearmodels.__all__ + namespace = {} + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + exec("from linearmodels import *", namespace) # noqa: S102 + assert "DROP_MISSING" not in namespace + assert namespace["IV2SLS"] is IV2SLS + + +def test_assigning_drop_missing_does_not_warn(restore_drop_missing): + vars(linearmodels).pop("DROP_MISSING", None) + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + linearmodels.DROP_MISSING = False + assert linearmodels.DROP_MISSING is False + # Deleting what was assigned makes reading it warn again + del linearmodels.DROP_MISSING + with pytest.warns(FutureWarning, match=re.escape(DROP_MISSING_MESSAGE)): + value = linearmodels.DROP_MISSING + assert isinstance(value, bool) + + +@pytest.mark.parametrize("name", ["DROP_MISSINGS", "drop_missing", "not_an_attribute"]) +def test_other_unknown_attributes_still_raise_attribute_error(name): + message = f"module 'linearmodels' has no attribute '{name}'" + with pytest.raises(AttributeError, match=re.escape(message)): + getattr(linearmodels, name) + assert not hasattr(linearmodels, name) + assert getattr(linearmodels, name, 7) == 7 + + +def two_stage_least_squares(y, exog, endog, instruments): + x = np.column_stack([exog, endog]) + z = np.column_stack([exog, instruments]) + fitted = z @ np.linalg.lstsq(z, x, rcond=None)[0] + return np.linalg.solve(fitted.T @ x, fitted.T @ y) + + +@pytest.mark.parametrize("drop_missing", [True, False]) +def test_drop_missing_has_no_effect(restore_drop_missing, drop_missing): + rng = np.random.default_rng(531) + nobs = 200 + instruments = rng.standard_normal((nobs, 2)) + endog = instruments @ [1.0, 0.5] + rng.standard_normal(nobs) + exog = np.column_stack([np.ones(nobs), rng.standard_normal(nobs)]) + y = exog @ [1.0, 0.5] + endog + rng.standard_normal(nobs) + complete = (y, exog, endog, instruments) + y, endog, instruments = y.copy(), endog.copy(), instruments.copy() + y[3] = np.nan + endog[40] = np.nan + instruments[77, 0] = np.nan + keep = np.ones(nobs, dtype=bool) + keep[[3, 40, 77]] = False + expected = two_stage_least_squares(*(c[keep] for c in complete)) + + linearmodels.DROP_MISSING = drop_missing + with pytest.warns(MissingValueWarning): + res = IV2SLS(y, exog, endog, instruments).fit() + # The rows with a missing value are dropped, whatever DROP_MISSING is + assert res.nobs == nobs - 3 + assert_allclose(res.params, expected, rtol=1e-10) + + +def run_python(*args, env_vars=None): + """Run a new interpreter without the settings of linearmodels in its environment""" + env = {k: v for k, v in os.environ.items() if not k.startswith("LINEARMODELS_")} + env.update(env_vars or {}) + cmd = [sys.executable, *args] + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=300, check=False, env=env + ) + assert proc.returncode == 0, proc.stderr + return proc + + +def test_drop_missing_environment_variable_warns_in_a_new_interpreter(): + # The warning happens when linearmodels is first imported in a process, so use a + # new interpreter to check what a user sees. The code is on line 2. -W makes this + # independent of PYTHONWARNINGS + code = "\nimport linearmodels\n" + proc = run_python( + "-W", + "default::FutureWarning", + "-c", + code, + env_vars={"LINEARMODELS_DROP_MISSING": "0"}, + ) + assert f":2: FutureWarning: {DROP_MISSING_ENV_MESSAGE}" in proc.stderr + + +# Imports linearmodels with the environment variable unset, then imports it again +# with each of the values in argv[1]. Prints the warnings about DROP_MISSING, and +# what reading DROP_MISSING returns, as JSON. +ENVIRONMENT_SCRIPT = """ +import importlib +import json +import os +import sys +import warnings + + +def drop_missing_warnings(record): + return [str(w.message) for w in record if "DROP_MISSING" in str(w.message)] + + +result = {} +with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + import linearmodels + # from-imports of submodules rely on __getattr__ raising AttributeError + from linearmodels import datasets + namespace = {} + exec("from linearmodels import *", namespace) +result["plain import"] = drop_missing_warnings(record) +result["import * defines DROP_MISSING"] = "DROP_MISSING" in namespace + +for value in json.loads(sys.argv[1]): + os.environ["LINEARMODELS_DROP_MISSING"] = value + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + importlib.reload(linearmodels) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + drop_missing = linearmodels.DROP_MISSING + result[value] = [drop_missing_warnings(record), drop_missing] +print(json.dumps(result)) +""" + +# What DROP_MISSING was in 7.0 for each value of LINEARMODELS_DROP_MISSING +LEGACY_DROP_MISSING = { + "": False, + "0": False, + "false": False, + "False": False, + "1": True, + "true": True, + "no": True, +} + + +def test_drop_missing_environment_variable_in_a_new_interpreter(): + proc = run_python("-c", ENVIRONMENT_SCRIPT, json.dumps(list(LEGACY_DROP_MISSING))) + result = json.loads(proc.stdout.strip().splitlines()[-1]) + # No warning without the environment variable, also for import * + assert result.pop("plain import") == [] + assert result.pop("import * defines DROP_MISSING") is False + # Any value, including the ones that turned it off, warns once and keeps what it + # gave in 7.0 + for value, expected in LEGACY_DROP_MISSING.items(): + messages, drop_missing = result.pop(value) + assert len(messages) == 1 + assert DROP_MISSING_ENV_MESSAGE in messages[0] + assert drop_missing is expected + assert not result