From a8c2853dfd722097cfc48f8957981e27c34b5393 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Mon, 12 Oct 2026 12:36:14 +0100 Subject: [PATCH 1/2] MAINT: Deprecate linearmodels.DROP_MISSING, which has no effect linearmodels.DROP_MISSING and the environment variable LINEARMODELS_DROP_MISSING have never done anything. They were added together with WARN_ON_MISSING in 3.0, and nothing in the package has ever read DROP_MISSING: observations with missing values are always dropped, whatever it is set to. Setting it to False neither keeps the incomplete observations nor silences the MissingValueWarning, which is what #531 asked for. WARN_ON_MISSING does that. Both are deprecated and will be removed in linearmodels 8.0, as lsmr_options and linearmodels.__future__.ordering are: * Reading linearmodels.DROP_MISSING emits a FutureWarning that points at the line that reads it, also for "from linearmodels import DROP_MISSING". It returns what it returned before: True, unless the environment variable turned it off. * Importing linearmodels with LINEARMODELS_DROP_MISSING set, to any value, emits a FutureWarning that points at the import. A plain import is silent. DROP_MISSING is no longer defined in the module, since Python calls a module-level __getattr__ only for names that the module does not define. Assigning linearmodels.DROP_MISSING = value defines it again and does not warn. That is the one use that cannot warn. DROP_MISSING is removed from __all__. A name in __all__ is read by "from linearmodels import *", which would then warn every user of a star import, and by anything else that walks __all__, such as test_all_is_valid. The cost is that a star import no longer binds the name, and the only thing that it could be used for is to read a setting that does nothing. __getattr__ is hidden from type checkers. With it, they accept every attribute of linearmodels, typed bool, so that a typo such as linearmodels.IV2SLSS passes a type check. They see "DROP_MISSING: bool" instead, so code that reads it still type checks while it is deprecated. The tests are in test_deprecations.py, with the other deprecations of 8.0: * Reading the attribute, and importing it, warn at the line that does it, and return the value of 7.0. * The name is not in __all__, and a star import neither warns nor defines it. * Assigning the attribute does not warn, and deleting it makes reading it warn again. The fixture puts the module back in the state in which it found it. * Other unknown attributes still raise AttributeError, which hasattr, getattr with a default and "from linearmodels import " rely on. * A fit with missing values gives the same number of observations and the same 2SLS parameters for DROP_MISSING = True and False, and they equal 2SLS on the complete observations computed with NumPy. This is what "has no effect" means. * In new interpreters: the environment variable warns at the import with the default warning filter, a plain import is silent, and the value of DROP_MISSING for each value of the variable equals that of 7.0. Six of these tests fail on main, which neither warns nor leaves the name out of __all__. The others check that what is not deprecated is unchanged. Refs #531 --- linearmodels/__init__.py | 51 +++++- linearmodels/tests/test_deprecations.py | 208 ++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 3 deletions(-) diff --git a/linearmodels/__init__.py b/linearmodels/__init__.py index 32d53c8791..e4659dd4df 100644 --- a/linearmodels/__init__.py +++ b/linearmodels/__init__.py @@ -37,12 +37,22 @@ - System Estimation using Generalized Method of Moments (GMM) Designed to work equally well with NumPy, Pandas or xarray data. + +Notes +----- +``linearmodels.DROP_MISSING`` and the environment variable +``LINEARMODELS_DROP_MISSING`` have never had any effect, because observations with +missing values are always dropped. They are deprecated and will be removed in +linearmodels 8.0. Reading ``DROP_MISSING``, and importing linearmodels when the +environment variable is set, issue a ``FutureWarning``. """ from __future__ import annotations import os import sys +from typing import TYPE_CHECKING +import warnings from ._version import version as __version__, version_tuple from .asset_pricing.model import ( @@ -65,11 +75,22 @@ OLS = _OLS _WARN_ON_MISSING = os.environ.get("LINEARMODELS_WARN_ON_MISSING", "1") WARN_ON_MISSING = False if _WARN_ON_MISSING in ("", "0", "false", "False") else True -_DROP_MISSING = os.environ.get("LINEARMODELS_DROP_MISSING", "1") -DROP_MISSING = False if _DROP_MISSING in ("", "0", "false", "False") else True + +# DROP_MISSING and LINEARMODELS_DROP_MISSING have never had an effect. They are +# deprecated and will be removed in 8.0, together with __getattr__ and the tests of +# the deprecation in tests/test_deprecations.py. +_DROP_MISSING = os.environ.get("LINEARMODELS_DROP_MISSING") +if _DROP_MISSING is not None: + warnings.warn( + "The environment variable LINEARMODELS_DROP_MISSING has no effect and will " + "be removed in linearmodels 8.0. Observations with missing values are always " + "dropped. Use LINEARMODELS_WARN_ON_MISSING to turn off the " + "MissingValueWarning.", + FutureWarning, + stacklevel=2, + ) __all__ = [ - "DROP_MISSING", "IV2SLS", "IV3SLS", "IVGMM", @@ -94,6 +115,30 @@ "version_tuple", ] +if TYPE_CHECKING: + # Type checkers must still see DROP_MISSING, and must not accept any other name + # because of the __getattr__ below + DROP_MISSING: bool +else: + + def __getattr__(name: str) -> bool: + # Python calls this function only for names that the module does not define. + # DROP_MISSING is deliberately not defined, so that reading it warns, and it + # is not in __all__, so that import * does not read it. Assigning to + # linearmodels.DROP_MISSING defines it, and then it does not warn. + if name == "DROP_MISSING": + warnings.warn( + "linearmodels.DROP_MISSING has no effect and will be removed in " + "linearmodels 8.0. Observations with missing values are always " + "dropped. Use linearmodels.WARN_ON_MISSING to turn off the " + "MissingValueWarning.", + FutureWarning, + stacklevel=2, + ) + # What 7.0 gave: True, unless the environment variable turned it off + return _DROP_MISSING not in ("", "0", "false", "False") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + def test( extra_args: str | list[str] | None = None, diff --git a/linearmodels/tests/test_deprecations.py b/linearmodels/tests/test_deprecations.py index 7155d0618b..c8c5473792 100644 --- a/linearmodels/tests/test_deprecations.py +++ b/linearmodels/tests/test_deprecations.py @@ -5,17 +5,23 @@ """ import importlib +import json +import os from pathlib import Path import re import subprocess import sys +import warnings import numpy as np from numpy.testing import assert_allclose import pandas as pd import pytest +import linearmodels +from linearmodels.iv import IV2SLS from linearmodels.iv.absorbing import AbsorbingLS +from linearmodels.shared.exceptions import MissingValueWarning LSMR_OPTIONS_MESSAGE = ( "lsmr_options is deprecated and will be removed in linearmodels 8.0" @@ -25,6 +31,13 @@ "linearmodels.__future__.ordering has no effect and will be removed in " "linearmodels 8.0" ) +DROP_MISSING_MESSAGE = ( + "linearmodels.DROP_MISSING has no effect and will be removed in linearmodels 8.0" +) +DROP_MISSING_ENV_MESSAGE = ( + "The environment variable LINEARMODELS_DROP_MISSING has no effect and will be " + "removed in linearmodels 8.0" +) @pytest.fixture @@ -127,3 +140,198 @@ def test_ordering_warns_in_a_new_interpreter(): proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300, check=False) assert proc.returncode == 0, proc.stderr assert f":2: FutureWarning: {ORDERING_MESSAGE}" in proc.stderr + + +@pytest.fixture +def restore_drop_missing(): + """Leave DROP_MISSING as it was: undefined, so that reading it warns""" + namespace = vars(linearmodels) + defined = "DROP_MISSING" in namespace + original = namespace.get("DROP_MISSING") + yield + if defined: + namespace["DROP_MISSING"] = original + else: + namespace.pop("DROP_MISSING", None) + + +def test_drop_missing_warns_when_it_is_read(): + with pytest.warns(FutureWarning, match=re.escape(DROP_MISSING_MESSAGE)) as record: + value = linearmodels.DROP_MISSING + # What 7.0 gave: the setting of the environment variable, True if it is not set + legacy = os.environ.get("LINEARMODELS_DROP_MISSING", "1") + assert value is (legacy not in ("", "0", "false", "False")) + matches = [w for w in record if DROP_MISSING_MESSAGE in str(w.message)] + assert len(matches) == 1 + # The warning points at the line that reads it + assert Path(matches[0].filename).resolve() == Path(__file__).resolve() + assert "WARN_ON_MISSING" in str(matches[0].message) + + +def test_drop_missing_warns_when_it_is_imported(): + # Run the statement as if it was in another file, on line 2, to check where the + # warning points + code = compile("\nfrom linearmodels import DROP_MISSING\n", "importing.py", "exec") + with pytest.warns(FutureWarning, match=re.escape(DROP_MISSING_MESSAGE)) as record: + exec(code, {}) # noqa: S102 + matches = [w for w in record if DROP_MISSING_MESSAGE in str(w.message)] + assert matches + assert all(w.filename == "importing.py" and w.lineno == 2 for w in matches) + + +def test_drop_missing_is_not_exported(): + # A name in __all__ would be read, and so warn, by import * + assert "DROP_MISSING" not in linearmodels.__all__ + namespace = {} + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + exec("from linearmodels import *", namespace) # noqa: S102 + assert "DROP_MISSING" not in namespace + assert namespace["IV2SLS"] is IV2SLS + + +def test_assigning_drop_missing_does_not_warn(restore_drop_missing): + vars(linearmodels).pop("DROP_MISSING", None) + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + linearmodels.DROP_MISSING = False + assert linearmodels.DROP_MISSING is False + # Deleting what was assigned makes reading it warn again + del linearmodels.DROP_MISSING + with pytest.warns(FutureWarning, match=re.escape(DROP_MISSING_MESSAGE)): + value = linearmodels.DROP_MISSING + assert isinstance(value, bool) + + +@pytest.mark.parametrize("name", ["DROP_MISSINGS", "drop_missing", "not_an_attribute"]) +def test_other_unknown_attributes_still_raise_attribute_error(name): + message = f"module 'linearmodels' has no attribute '{name}'" + with pytest.raises(AttributeError, match=re.escape(message)): + getattr(linearmodels, name) + assert not hasattr(linearmodels, name) + assert getattr(linearmodels, name, 7) == 7 + + +def two_stage_least_squares(y, exog, endog, instruments): + x = np.column_stack([exog, endog]) + z = np.column_stack([exog, instruments]) + fitted = z @ np.linalg.lstsq(z, x, rcond=None)[0] + return np.linalg.solve(fitted.T @ x, fitted.T @ y) + + +@pytest.mark.parametrize("drop_missing", [True, False]) +def test_drop_missing_has_no_effect(restore_drop_missing, drop_missing): + rng = np.random.default_rng(531) + nobs = 200 + instruments = rng.standard_normal((nobs, 2)) + endog = instruments @ [1.0, 0.5] + rng.standard_normal(nobs) + exog = np.column_stack([np.ones(nobs), rng.standard_normal(nobs)]) + y = exog @ [1.0, 0.5] + endog + rng.standard_normal(nobs) + complete = (y, exog, endog, instruments) + y, endog, instruments = y.copy(), endog.copy(), instruments.copy() + y[3] = np.nan + endog[40] = np.nan + instruments[77, 0] = np.nan + keep = np.ones(nobs, dtype=bool) + keep[[3, 40, 77]] = False + expected = two_stage_least_squares(*(c[keep] for c in complete)) + + linearmodels.DROP_MISSING = drop_missing + with pytest.warns(MissingValueWarning): + res = IV2SLS(y, exog, endog, instruments).fit() + # The rows with a missing value are dropped, whatever DROP_MISSING is + assert res.nobs == nobs - 3 + assert_allclose(res.params, expected, rtol=1e-10) + + +def run_python(*args, env_vars=None): + """Run a new interpreter without the settings of linearmodels in its environment""" + env = {k: v for k, v in os.environ.items() if not k.startswith("LINEARMODELS_")} + env.update(env_vars or {}) + cmd = [sys.executable, *args] + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=300, check=False, env=env + ) + assert proc.returncode == 0, proc.stderr + return proc + + +def test_drop_missing_environment_variable_warns_in_a_new_interpreter(): + # The warning happens when linearmodels is first imported in a process, so use a + # new interpreter to check what a user sees. The code is on line 2. -W makes this + # independent of PYTHONWARNINGS + code = "\nimport linearmodels\n" + proc = run_python( + "-W", + "default::FutureWarning", + "-c", + code, + env_vars={"LINEARMODELS_DROP_MISSING": "0"}, + ) + assert f":2: FutureWarning: {DROP_MISSING_ENV_MESSAGE}" in proc.stderr + + +# Imports linearmodels with the environment variable unset, then imports it again +# with each of the values in argv[1]. Prints the warnings about DROP_MISSING, and +# what reading DROP_MISSING returns, as JSON. +ENVIRONMENT_SCRIPT = """ +import importlib +import json +import os +import sys +import warnings + + +def drop_missing_warnings(record): + return [str(w.message) for w in record if "DROP_MISSING" in str(w.message)] + + +result = {} +with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + import linearmodels + # from-imports of submodules rely on __getattr__ raising AttributeError + from linearmodels import datasets + namespace = {} + exec("from linearmodels import *", namespace) +result["plain import"] = drop_missing_warnings(record) +result["import * defines DROP_MISSING"] = "DROP_MISSING" in namespace + +for value in json.loads(sys.argv[1]): + os.environ["LINEARMODELS_DROP_MISSING"] = value + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + importlib.reload(linearmodels) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + drop_missing = linearmodels.DROP_MISSING + result[value] = [drop_missing_warnings(record), drop_missing] +print(json.dumps(result)) +""" + +# What DROP_MISSING was in 7.0 for each value of LINEARMODELS_DROP_MISSING +LEGACY_DROP_MISSING = { + "": False, + "0": False, + "false": False, + "False": False, + "1": True, + "true": True, + "no": True, +} + + +def test_drop_missing_environment_variable_in_a_new_interpreter(): + proc = run_python("-c", ENVIRONMENT_SCRIPT, json.dumps(list(LEGACY_DROP_MISSING))) + result = json.loads(proc.stdout.strip().splitlines()[-1]) + # No warning without the environment variable, also for import * + assert result.pop("plain import") == [] + assert result.pop("import * defines DROP_MISSING") is False + # Any value, including the ones that turned it off, warns once and keeps what it + # gave in 7.0 + for value, expected in LEGACY_DROP_MISSING.items(): + messages, drop_missing = result.pop(value) + assert len(messages) == 1 + assert DROP_MISSING_ENV_MESSAGE in messages[0] + assert drop_missing is expected + assert not result From 65dfa2e530c25ad22547d2e272790cda6546cccf Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Mon, 12 Oct 2026 12:37:07 +0100 Subject: [PATCH 2/2] DOC: Document WARN_ON_MISSING and the ways to turn off MissingValueWarning linearmodels.WARN_ON_MISSING and the environment variable LINEARMODELS_WARN_ON_MISSING turn off the MissingValueWarning that a model issues when it drops observations with missing values. They have worked since 3.0, but nothing documented them, so #531 asked for an option that exists. They are now in the package docstring (help(linearmodels)), in the docstring of MissingValueWarning, and in a new "Missing Values" section of the Utilities page, which also describes the deprecation of DROP_MISSING. Three ways to turn the warning off are documented: * linearmodels.WARN_ON_MISSING = False, for every model that is created afterwards. * The environment variable LINEARMODELS_WARN_ON_MISSING, which is read when linearmodels is imported, so it has to be set before the import. The values "", "0", "false" and "False" turn the warning off. Any other value, including "FALSE", "no" and "off", and no variable, leave it on. * A warnings filter for MissingValueWarning, which applies where it is set, so it can turn the warning off for part of a program, or turn it into an error. The warning is issued when the model is created, not when it is fit. tests/shared/test_missing_values.py checks what the documentation says. This is behavior that main already has, so the tests pass on main. For IV2SLS, AbsorbingLS, PanelOLS, FamaMacBeth, SUR and LinearFactorModel, which are one model of each kind that calls missing_warning, with a missing value in three different inputs and rows: * The model warns, and the fit has the number of observations without those rows. For SUR the results count the observations of both equations. * There is no warning if nothing is missing. * With WARN_ON_MISSING = False the model does not warn and drops the same rows. * A filter with the action "ignore" removes the warning, and the action "error" raises it. * The environment variable gives WARN_ON_MISSING = False for "", "0", "false" and "False", and True for the other values and if it is not set. This runs in a new interpreter, which imports linearmodels again for each value. * Setting the variable after the import has no effect. The tests do not depend on the environment of the test run, because a fixture sets WARN_ON_MISSING and pytest puts it back after every test. The Sphinx build adds no warnings. The page has no spelling errors. Refs #531 --- doc/source/utility.rst | 62 +++++ linearmodels/__init__.py | 14 ++ linearmodels/shared/exceptions.py | 13 +- .../tests/shared/test_missing_values.py | 229 ++++++++++++++++++ 4 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 linearmodels/tests/shared/test_missing_values.py diff --git a/doc/source/utility.rst b/doc/source/utility.rst index 3be545bc5c..4828388841 100644 --- a/doc/source/utility.rst +++ b/doc/source/utility.rst @@ -43,3 +43,65 @@ Covariance Estimation cov_cluster cov_kernel group_debias_coefficient + + +Missing Values +-------------- + +.. currentmodule:: linearmodels.shared.exceptions + +A model drops the observations that have a missing value, such as ``NaN``, in any of +its inputs when it is created. It issues a :class:`MissingValueWarning` if it drops any +observation, and the results use the remaining observations. This is the same in every +model. + +The warning is on by default. It can be turned off in three ways. + +* Set ``linearmodels.WARN_ON_MISSING`` to ``False``. It applies to every model that is + created afterwards. + + .. code-block:: python + + import linearmodels + + linearmodels.WARN_ON_MISSING = False + +* Set the environment variable ``LINEARMODELS_WARN_ON_MISSING`` before importing + linearmodels, for example in the shell that starts Python. The values ``""`` (the + empty string), ``"0"``, ``"false"`` and ``"False"`` turn the warning off. Any other + value, or no variable, leaves it on. The variable is read when linearmodels is + imported, so setting it later has no effect. + + .. code-block:: bash + + export LINEARMODELS_WARN_ON_MISSING=0 + +* Use a warnings filter. A filter applies only where it is set, so it can turn off the + warning for part of a program. The warning is issued when the model is created, not + when it is fit. + + .. code-block:: python + + import warnings + + from linearmodels.iv import IV2SLS + from linearmodels.shared.exceptions import MissingValueWarning + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", MissingValueWarning) + mod = IV2SLS(dependent, exog, endog, instruments) + res = mod.fit() + + The action ``"error"`` instead of ``"ignore"`` raises an exception if a model drops + any observation. + +.. deprecated:: 7.1 + + The module attribute ``linearmodels.DROP_MISSING`` and the environment variable + ``LINEARMODELS_DROP_MISSING`` have never had an effect, because observations with + missing values are always dropped. Reading ``linearmodels.DROP_MISSING``, and + importing linearmodels when the environment variable is set, issue a + ``FutureWarning``. Both will be removed in linearmodels 8.0. Use + ``WARN_ON_MISSING`` to turn off the warning. + +.. autoclass:: linearmodels.shared.exceptions.MissingValueWarning diff --git a/linearmodels/__init__.py b/linearmodels/__init__.py index e4659dd4df..bac783ac07 100644 --- a/linearmodels/__init__.py +++ b/linearmodels/__init__.py @@ -40,6 +40,20 @@ Notes ----- +A model drops the observations that have a missing value in any of its inputs when +it is created, and issues a ``MissingValueWarning`` if it drops any. The warning +can be turned off in three ways: + +- Set ``linearmodels.WARN_ON_MISSING = False``. It applies to every model that is + created afterwards. +- Set the environment variable ``LINEARMODELS_WARN_ON_MISSING`` before importing + linearmodels. The values ``""``, ``"0"``, ``"false"`` and ``"False"`` turn the + warning off. Any other value leaves it on. +- Use a warnings filter, for example + ``warnings.simplefilter("ignore", MissingValueWarning)``, where + ``MissingValueWarning`` is imported from ``linearmodels.shared.exceptions``. The + filter applies where it is set, for example inside ``warnings.catch_warnings()``. + ``linearmodels.DROP_MISSING`` and the environment variable ``LINEARMODELS_DROP_MISSING`` have never had any effect, because observations with missing values are always dropped. They are deprecated and will be removed in diff --git a/linearmodels/shared/exceptions.py b/linearmodels/shared/exceptions.py index 70744758db..318ff50d91 100644 --- a/linearmodels/shared/exceptions.py +++ b/linearmodels/shared/exceptions.py @@ -6,7 +6,18 @@ class MissingValueWarning(Warning): - pass + """ + Warning that observations with missing values were dropped. + + Notes + ----- + A model drops the observations that have a missing value in any of its inputs + when it is created, and issues this warning if it drops any. The warning is on + by default. Set ``linearmodels.WARN_ON_MISSING = False`` or the environment + variable ``LINEARMODELS_WARN_ON_MISSING=0`` to turn it off for every model. To + change it for part of a program, use a warnings filter for this class, with the + action ``"ignore"`` or ``"error"``. + """ missing_value_warning_msg = """ diff --git a/linearmodels/tests/shared/test_missing_values.py b/linearmodels/tests/shared/test_missing_values.py new file mode 100644 index 0000000000..6cc1c9688a --- /dev/null +++ b/linearmodels/tests/shared/test_missing_values.py @@ -0,0 +1,229 @@ +""" +Tests of how the models treat missing values, and of how to turn off the warning + +A model drops the observations that have a missing value in any of its inputs and +issues a MissingValueWarning. The warning is controlled by +linearmodels.WARN_ON_MISSING, by the environment variable +LINEARMODELS_WARN_ON_MISSING, which sets it when linearmodels is imported, and by +warnings filters. +""" + +import json +import os +import subprocess +import sys +import warnings + +import numpy as np +import pandas as pd +import pytest + +import linearmodels +from linearmodels import ( + IV2SLS, + SUR, + AbsorbingLS, + FamaMacBeth, + LinearFactorModel, + PanelOLS, +) +from linearmodels.shared.exceptions import MissingValueWarning + +NOBS = 120 +# Each model has a missing value in three different rows +MISSING_ROWS = 3 +MESSAGE = "Dropping rows with missing observations" + + +def iv_model(rng, missing): + instruments = rng.standard_normal((NOBS, 2)) + endog = instruments @ [1.0, 0.5] + rng.standard_normal(NOBS) + exog = np.column_stack([np.ones(NOBS), rng.standard_normal(NOBS)]) + y = exog @ [1.0, 0.5] + endog + rng.standard_normal(NOBS) + if missing: + y[3] = np.nan + endog[40] = np.nan + instruments[77, 0] = np.nan + return IV2SLS(y, exog, endog, instruments) + + +def absorbing_model(rng, missing): + cats = pd.DataFrame({"a": pd.Categorical(rng.integers(0, 6, NOBS))}) + x = rng.standard_normal((NOBS, 2)) + weights = rng.uniform(0.5, 1.5, NOBS) + y = x @ [1.0, 0.5] + rng.standard_normal(NOBS) + if missing: + y[3] = np.nan + x[40, 1] = np.nan + weights[77] = np.nan + return AbsorbingLS(y, x, absorb=cats, weights=weights) + + +def panel_data(rng, missing): + index = pd.MultiIndex.from_product([range(12), range(10)], names=["i", "t"]) + x = pd.DataFrame(rng.standard_normal((NOBS, 2)), index=index, columns=["a", "b"]) + y = pd.Series(x.to_numpy() @ [1.0, 0.5] + rng.standard_normal(NOBS), index=index) + if missing: + y.iloc[3] = np.nan + x.iloc[40, 0] = np.nan + x.iloc[77, 1] = np.nan + return y, x + + +def panel_model(rng, missing): + return PanelOLS(*panel_data(rng, missing), entity_effects=True) + + +def fama_macbeth_model(rng, missing): + return FamaMacBeth(*panel_data(rng, missing)) + + +def system_model(rng, missing): + y = rng.standard_normal((2, NOBS)) + x = rng.standard_normal((2, NOBS, 2)) + if missing: + y[0, 3] = np.nan + x[1, 40, 0] = np.nan + y[1, 77] = np.nan + equations = {f"eq{i}": {"dependent": y[i], "exog": x[i]} for i in range(2)} + return SUR(equations) + + +def factor_model(rng, missing): + factors = rng.standard_normal((NOBS, 2)) + portfolios = factors @ rng.standard_normal((2, 4)) + rng.standard_normal((NOBS, 4)) + if missing: + portfolios[3, 1] = np.nan + factors[40, 0] = np.nan + portfolios[77, 3] = np.nan + return LinearFactorModel(portfolios, factors) + + +MODELS = { + "IV2SLS": iv_model, + "AbsorbingLS": absorbing_model, + "PanelOLS": panel_model, + "FamaMacBeth": fama_macbeth_model, + "SUR": system_model, + "LinearFactorModel": factor_model, +} + + +def build(name, missing=True): + rng = np.random.default_rng(20261012) + return MODELS[name](rng, missing) + + +def nobs_after_dropping(name, missing=True): + """The observations that are left, which the system results sum over equations""" + nobs = NOBS - MISSING_ROWS if missing else NOBS + return 2 * nobs if name == "SUR" else nobs + + +@pytest.fixture(autouse=True) +def warn_on_missing(monkeypatch): + # The tests do not depend on the environment variable, and the setting is + # restored after each test + monkeypatch.setattr(linearmodels, "WARN_ON_MISSING", True) + + +@pytest.mark.parametrize("name", MODELS) +def test_missing_values_are_dropped_with_a_warning(name): + with pytest.warns(MissingValueWarning, match=MESSAGE): + model = build(name) + assert model.fit().nobs == nobs_after_dropping(name) + + +@pytest.mark.parametrize("name", MODELS) +def test_no_warning_if_nothing_is_missing(name): + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingValueWarning) + model = build(name, missing=False) + assert model.fit().nobs == nobs_after_dropping(name, missing=False) + + +@pytest.mark.parametrize("name", MODELS) +def test_warn_on_missing_false_turns_the_warning_off(name, monkeypatch): + monkeypatch.setattr(linearmodels, "WARN_ON_MISSING", False) + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingValueWarning) + model = build(name) + # The observations are dropped as before + assert model.fit().nobs == nobs_after_dropping(name) + + +@pytest.mark.parametrize("name", MODELS) +def test_warnings_filter_ignore(name): + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + warnings.simplefilter("ignore", MissingValueWarning) + model = build(name) + assert not [w for w in record if issubclass(w.category, MissingValueWarning)] + assert model.fit().nobs == nobs_after_dropping(name) + + +@pytest.mark.parametrize("name", MODELS) +def test_warnings_filter_error(name): + with warnings.catch_warnings(): + warnings.simplefilter("error", MissingValueWarning) + with pytest.raises(MissingValueWarning, match=MESSAGE): + build(name) + + +# Imports linearmodels with each of the values in argv[1] of the environment variable +# in turn, and prints WARN_ON_MISSING for each as JSON. A null value means that the +# variable is not set. +ENVIRONMENT_SCRIPT = """ +import importlib +import json +import os +import sys + +import linearmodels + +result = {} +for value in json.loads(sys.argv[1]): + if value is None: + os.environ.pop("LINEARMODELS_WARN_ON_MISSING", None) + else: + os.environ["LINEARMODELS_WARN_ON_MISSING"] = value + importlib.reload(linearmodels) + result[json.dumps(value)] = linearmodels.WARN_ON_MISSING +print(json.dumps(result)) +""" + +# The warning is off for the values "", "0", "false" and "False", and on for any other +# value and if the variable is not set +WARN_ON_MISSING_FROM_ENVIRONMENT = { + None: True, + "": False, + "0": False, + "false": False, + "False": False, + "1": True, + "true": True, + "True": True, + "FALSE": True, + "no": True, + "off": True, +} + + +def test_warn_on_missing_environment_variable(): + env = {k: v for k, v in os.environ.items() if not k.startswith("LINEARMODELS_")} + values = json.dumps(list(WARN_ON_MISSING_FROM_ENVIRONMENT)) + cmd = [sys.executable, "-c", ENVIRONMENT_SCRIPT, values] + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=300, check=False, env=env + ) + assert proc.returncode == 0, proc.stderr + result = json.loads(proc.stdout.strip().splitlines()[-1]) + expected = {json.dumps(k): v for k, v in WARN_ON_MISSING_FROM_ENVIRONMENT.items()} + assert result == expected + + +def test_environment_variable_is_read_when_linearmodels_is_imported(monkeypatch): + # Setting it later has no effect + monkeypatch.setenv("LINEARMODELS_WARN_ON_MISSING", "0") + with pytest.warns(MissingValueWarning, match=MESSAGE): + build("IV2SLS")