From 064dcd5f2161e88af38ba13e9fd82d6e3b5d0824 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Mon, 12 Oct 2026 20:51:13 +0100 Subject: [PATCH 1/2] BUG: Evaluate the asset pricing formulas in the namespace of the caller The from_formula methods of TradedFactorModel, LinearFactorModel and LinearFactorModelGMM passed context=0 to formulaic, with a TODO to pass the evaluation environment. The formula was then evaluated in the namespace of an internal function, and a variable or a function of the code that calls from_formula was not found: mod = TradedFactorModel.from_formula("p1 + p2 ~ double(f1) + f2", data) FactorEvaluationError: Unable to evaluate factor `double(f1)`. [NameError: name 'double' is not defined] The system models, SUR, IV3SLS and IVSystemGMM, find it. A local variable, as in I(f1 * scale_factor) inside a function, failed in the same way, and a variable that has the name of a transform of formulaic, such as scale, resolved to the transform. from_formula now captures the namespace of its caller with capture_context(1), as the system models do, and passes it to the formulas of the portfolios and of the factors. A column of data takes precedence over a variable of the caller, which takes precedence over the transforms of formulaic, which are still available without an import (np.exp, center, and the others). test_formula_context.py checks the three models, with the portfolios in the formula and in portfolios, with functions of the module and of the function that calls from_formula, local variables, functions in the portfolios and the precedence, and that the results are those of the model built from the transformed arrays. 14 of its 23 tests fail before the change. The docstrings of the from_formula methods say that the formulas can use the variables and functions of the caller. --- linearmodels/asset_pricing/model.py | 50 +++++-- .../asset_pricing/test_formula_context.py | 135 ++++++++++++++++++ 2 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 linearmodels/tests/asset_pricing/test_formula_context.py diff --git a/linearmodels/asset_pricing/model.py b/linearmodels/asset_pricing/model.py index aaf0f5c296..d503607332 100644 --- a/linearmodels/asset_pricing/model.py +++ b/linearmodels/asset_pricing/model.py @@ -4,11 +4,12 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Any, cast from formulaic import model_matrix from formulaic.materializers.types import NAAction +from formulaic.utils.context import capture_context import numpy as np from numpy.linalg import lstsq from pandas import DataFrame @@ -158,7 +159,10 @@ def formula(self, value: str | None) -> None: @staticmethod def _prepare_data_from_formula( - formula: str, data: DataFrame, portfolios: DataFrame | None + formula: str, + data: DataFrame, + portfolios: DataFrame | None, + context: Mapping[str, Any] | None, ) -> tuple[DataFrame, DataFrame, str]: orig_formula = formula # Missing values are kept so that the model drops the observations with a @@ -168,7 +172,7 @@ def _prepare_data_from_formula( factors_mm = model_matrix( formula + " + 0", data, - context=0, # TODO: self._eval_env, + context=context, ensure_full_rank=True, na_action=na_action, ) @@ -178,7 +182,7 @@ def _prepare_data_from_formula( portfolios_mm = model_matrix( formula_components[0].strip() + " + 0", data, - context=0, # TODO: self._eval_env, + context=context, ensure_full_rank=False, na_action=na_action, ) @@ -186,7 +190,7 @@ def _prepare_data_from_formula( factors_mm = model_matrix( formula_components[1].strip() + " + 0", data, - context=0, # TODO: self._eval_env, + context=context, ensure_full_rank=False, na_action=na_action, ) @@ -269,6 +273,15 @@ def from_formula( The second specified the portfolio using ``+`` to separate the test portfolios and ``~`` to separate the test portfolios from the factors. + The formula can use the variables and functions that are available + where ``from_formula`` is called, in addition to the columns of ``data`` + and the transforms of formulaic. + + .. versionchanged:: 7.1 + + Variables and functions of the caller could not be used in the + formula, and raised a ``FactorEvaluationError``. + Examples -------- >>> from linearmodels.datasets import french @@ -283,8 +296,9 @@ def from_formula( >>> formula = "MktRF + SMB + HML" >>> mod = TradedFactorModel.from_formula(formula, data, portfolios=portfolios) """ + context = capture_context(1) factors, portfolios, formula = cls._prepare_data_from_formula( - formula, data, portfolios + formula, data, portfolios, context ) mod = cls(portfolios, factors) mod.formula = formula @@ -606,6 +620,15 @@ def from_formula( The second specified the portfolio using ``+`` to separate the test portfolios and ``~`` to separate the test portfolios from the factors. + The formula can use the variables and functions that are available + where ``from_formula`` is called, in addition to the columns of ``data`` + and the transforms of formulaic. + + .. versionchanged:: 7.1 + + Variables and functions of the caller could not be used in the + formula, and raised a ``FactorEvaluationError``. + Examples -------- >>> from linearmodels.datasets import french @@ -620,8 +643,9 @@ def from_formula( >>> formula = "MktRF + SMB + HML" >>> mod = LinearFactorModel.from_formula(formula, data, portfolios=portfolios) """ + context = capture_context(1) factors, portfolios, formula = cls._prepare_data_from_formula( - formula, data, portfolios + formula, data, portfolios, context ) mod = cls(portfolios, factors, risk_free=risk_free, sigma=sigma) mod.formula = formula @@ -919,6 +943,15 @@ def from_formula( The second specified the portfolio using ``+`` to separate the test portfolios and ``~`` to separate the test portfolios from the factors. + The formula can use the variables and functions that are available + where ``from_formula`` is called, in addition to the columns of ``data`` + and the transforms of formulaic. + + .. versionchanged:: 7.1 + + Variables and functions of the caller could not be used in the + formula, and raised a ``FactorEvaluationError``. + Examples -------- >>> from linearmodels.datasets import french @@ -933,8 +966,9 @@ def from_formula( >>> formula = "MktRF + SMB + HML" >>> mod = LinearFactorModel.from_formula(formula, data, portfolios=portfolios) """ + context = capture_context(1) factors, portfolios, formula = cls._prepare_data_from_formula( - formula, data, portfolios + formula, data, portfolios, context ) mod = cls(portfolios, factors, risk_free=risk_free) mod.formula = formula diff --git a/linearmodels/tests/asset_pricing/test_formula_context.py b/linearmodels/tests/asset_pricing/test_formula_context.py new file mode 100644 index 0000000000..6be4854849 --- /dev/null +++ b/linearmodels/tests/asset_pricing/test_formula_context.py @@ -0,0 +1,135 @@ +""" +Tests that the formulas of the asset pricing models use the variables and functions +of the code that calls from_formula, as the formulas of the system models do +""" + +from formulaic.errors import FactorEvaluationError +import numpy as np +from numpy.testing import assert_allclose +import pandas as pd +import pytest + +from linearmodels.asset_pricing.model import ( + LinearFactorModel, + LinearFactorModelGMM, + TradedFactorModel, +) +from linearmodels.system import SUR + +MODELS = [TradedFactorModel, LinearFactorModel, LinearFactorModelGMM] +PORTFOLIOS = "port_01 + port_02 + port_03 + port_04" + + +@pytest.fixture(scope="module") +def data(): + rng = np.random.default_rng(2) + factors = 0.05 * rng.standard_normal((120, 2)) + 0.01 + portfolios = factors @ rng.uniform(0.5, 1.5, (2, 4)) + portfolios += 0.03 * rng.standard_normal((120, 4)) + frame = pd.DataFrame(portfolios, columns=PORTFOLIOS.split(" + ")) + frame["factor_01"] = factors[:, 0] + frame["factor_02"] = factors[:, 1] + return frame + + +def double(x): + return 2 * x + + +def check(mod, data, first_factor, name): + """The model has the transformed first factor and the other, as the arrays""" + expected = np.c_[first_factor, data["factor_02"]] + assert list(mod.factors.pandas.columns) == [name, "factor_02"] + assert_allclose(mod.factors.ndarray, expected) + assert_allclose(mod.portfolios.ndarray, data[PORTFOLIOS.split(" + ")]) + assert mod.formula is not None + + +@pytest.mark.parametrize("model", MODELS) +@pytest.mark.parametrize("portfolios_in_formula", [True, False]) +def test_function_of_the_module(model, data, portfolios_in_formula): + factors = "double(factor_01) + factor_02" + if portfolios_in_formula: + mod = model.from_formula(f"{PORTFOLIOS} ~ {factors}", data) + else: + portfolios = data[PORTFOLIOS.split(" + ")] + mod = model.from_formula(factors, data, portfolios=portfolios) + check(mod, data, 2 * data["factor_01"], "double(factor_01)") + + +@pytest.mark.parametrize("model", MODELS) +def test_local_variable_and_function(model, data): + scale_factor = 3.0 + + def triple(x): + return scale_factor * x + + formula = f"{PORTFOLIOS} ~ I(factor_01 * scale_factor) + factor_02" + mod = model.from_formula(formula, data) + check(mod, data, 3 * data["factor_01"], "I(factor_01 * scale_factor)") + + formula = f"{PORTFOLIOS} ~ triple(factor_01) + factor_02" + mod = model.from_formula(formula, data) + check(mod, data, 3 * data["factor_01"], "triple(factor_01)") + + +@pytest.mark.parametrize("model", MODELS) +def test_function_in_the_portfolios(model, data): + formula = "double(port_01) + port_02 + port_03 + port_04 ~ factor_01 + factor_02" + mod = model.from_formula(formula, data) + assert mod.portfolios.pandas.columns[0] == "double(port_01)" + assert_allclose(mod.portfolios.ndarray[:, 0], 2 * data["port_01"]) + assert_allclose(mod.portfolios.ndarray[:, 1:], data[PORTFOLIOS.split(" + ")[1:]]) + + +@pytest.mark.parametrize("model", MODELS) +def test_built_in_transforms_do_not_need_an_import(model, data): + # np is a transform of formulaic, and center is its own + formula = f"{PORTFOLIOS} ~ np.exp(factor_01) + center(factor_02)" + mod = model.from_formula(formula, data) + assert_allclose(mod.factors.ndarray[:, 0], np.exp(data["factor_01"])) + assert_allclose( + mod.factors.ndarray[:, 1], data["factor_02"] - data["factor_02"].mean() + ) + + +@pytest.mark.parametrize("model", MODELS) +def test_data_before_context(model, data): + # A column of the data hides a variable with the same name + factor_01 = 12345.0 + assert factor_01 > 0 + mod = model.from_formula(f"{PORTFOLIOS} ~ factor_01 + factor_02", data) + assert_allclose(mod.factors.ndarray[:, 0], data["factor_01"]) + + +@pytest.mark.parametrize("model", MODELS) +def test_unknown_names_still_raise(model, data): + with pytest.raises(FactorEvaluationError, match="not_defined_anywhere"): + model.from_formula(f"{PORTFOLIOS} ~ not_defined_anywhere(factor_01)", data) + + +def test_results_use_the_transformed_factor(data): + multiplier = 0.5 + formula = f"{PORTFOLIOS} ~ I(factor_01 * multiplier) + factor_02" + for model in (TradedFactorModel, LinearFactorModel): + res = model.from_formula(formula, data).fit() + direct = model( + data[PORTFOLIOS.split(" + ")], + np.c_[multiplier * data["factor_01"], data["factor_02"]], + ).fit() + assert_allclose(res.params, direct.params) + assert_allclose(res.cov, direct.cov) + assert_allclose(res.risk_premia, direct.risk_premia) + + +def test_same_as_the_system_models(data): + # The system models capture the context of the caller in the same way + frame = data.rename(columns={"port_01": "y1", "port_02": "y2"}) + res = SUR.from_formula( + {"a": "y1 ~ 1 + double(factor_01)", "b": "y2 ~ 1 + factor_02"}, frame + ).fit() + assert res.params.shape == (4,) + mod = TradedFactorModel.from_formula( + "y1 + y2 ~ double(factor_01) + factor_02", frame + ) + assert mod.factors.shape == (120, 2) From 5830fb6270152409b0e91e97cf967831b238e421 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Mon, 12 Oct 2026 21:21:13 +0100 Subject: [PATCH 2/2] MAINT: Pass an integer context to formulaic if the caller's namespace is unavailable capture_context returns None where the frames of the caller cannot be read, and model_matrix takes an integer or a mapping. The asset pricing formulas then use the integer 0, the namespace of the function that evaluates the formula, which is what they did before the context was captured. There is no change in behavior where the namespace can be captured. --- linearmodels/asset_pricing/model.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/linearmodels/asset_pricing/model.py b/linearmodels/asset_pricing/model.py index d503607332..0fbb394d35 100644 --- a/linearmodels/asset_pricing/model.py +++ b/linearmodels/asset_pricing/model.py @@ -168,11 +168,14 @@ def _prepare_data_from_formula( # Missing values are kept so that the model drops the observations with a # missing value in the portfolios or the factors, as it does for arrays na_action = NAAction("ignore") + # capture_context returns None if the frames of the caller are unavailable, + # when formulaic looks in the namespace of this function + eval_context: int | Mapping[str, Any] = 0 if context is None else context if portfolios is not None: factors_mm = model_matrix( formula + " + 0", data, - context=context, + context=eval_context, ensure_full_rank=True, na_action=na_action, ) @@ -182,7 +185,7 @@ def _prepare_data_from_formula( portfolios_mm = model_matrix( formula_components[0].strip() + " + 0", data, - context=context, + context=eval_context, ensure_full_rank=False, na_action=na_action, ) @@ -190,7 +193,7 @@ def _prepare_data_from_formula( factors_mm = model_matrix( formula_components[1].strip() + " + 0", data, - context=context, + context=eval_context, ensure_full_rank=False, na_action=na_action, )