Skip to content

Commit 05b7903

Browse files
authored
Merge pull request #802 from bashtage/lm71-fix-system-asset-pricing-small
BUG: Raise a ValueError for a debiased factor model without residual …
2 parents 8f137ee + 2366ab2 commit 05b7903

7 files changed

Lines changed: 427 additions & 10 deletions

File tree

‎linearmodels/asset_pricing/model.py‎

Lines changed: 66 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -136,9 +136,20 @@ def _validate_data(self) -> None:
136136
"equivalent and must not have rank\n"
137137
"less than the dimension of the smaller shape."
138138
)
139-
if has_constant(f)[0]:
139+
rank = np.linalg.matrix_rank(f)
140+
constant = has_constant(f, rank)[0]
141+
if constant and rank < f.shape[1]:
142+
# has_constant is also True if the columns are collinear. Collinear
143+
# factors have a constant if a column is constant, or, for a matrix
144+
# with more rows than columns, if a constant is in the span of the
145+
# columns
146+
nobs = f.shape[0]
147+
aug_rank = np.linalg.matrix_rank(np.c_[np.ones((nobs, 1)), f])
148+
constant_column = np.any((np.ptp(f, axis=0) == 0) & ~np.all(f == 0, axis=0))
149+
constant = bool(constant_column or (nobs > f.shape[1] and aug_rank == rank))
150+
if constant:
140151
raise ValueError("factors must not contain a constant or equivalent.")
141-
if np.linalg.matrix_rank(f) < f.shape[1]:
152+
if rank < f.shape[1]:
142153
raise ValueError(
143154
"Model cannot be estimated. factors do not have full column rank."
144155
)
@@ -148,6 +159,18 @@ def _validate_data(self) -> None:
148159
"more returns to\nestimate the model parameters."
149160
)
150161

162+
def _check_debiased(self, debiased: bool) -> None:
163+
"""Debiasing needs more observations than regressors in the regressions"""
164+
nobs, nfactor = self.factors.shape
165+
if debiased and nobs <= nfactor + 1:
166+
raise ValueError(
167+
"The covariance cannot be debiased when the number of observations "
168+
f"({nobs}) is not larger than the number of factors plus one "
169+
f"({nfactor + 1}), the number of regressors of each time-series "
170+
"regression, since there are no residual degrees of freedom. Use "
171+
"debiased=False or more observations."
172+
)
173+
151174
@property
152175
def formula(self) -> str | None:
153176
return self._formula
@@ -326,6 +349,7 @@ def fit(
326349
``kernel``, one of "bartlett", "parzen" or "qs" (quadratic spectral)
327350
and ``bandwidth`` (a positive integer).
328351
"""
352+
self._check_debiased(debiased)
329353
p = self.portfolios.ndarray
330354
f = self.factors.ndarray
331355
nportfolio = p.shape[1]
@@ -459,11 +483,33 @@ def __init__(
459483
self.portfolios.shape[1]
460484
)
461485
else:
462-
self._sigma = np.asarray(sigma)
463-
vals, vecs = np.linalg.eigh(sigma)
486+
self._sigma = self._validate_sigma(sigma)
487+
vals, vecs = np.linalg.eigh(self._sigma)
464488
self._sigma_m12 = vecs @ np.diag(1.0 / np.sqrt(vals)) @ vecs.T
465489
self._sigma_inv = np.linalg.inv(self._sigma).astype(float, copy=False)
466490

491+
def _validate_sigma(
492+
self, sigma: linearmodels.typing.data.ArrayLike
493+
) -> linearmodels.typing.data.Float64Array:
494+
"""Check that sigma is a symmetric positive definite matrix of the right size"""
495+
nportfolio = self.portfolios.shape[1]
496+
sigma_arr = np.asarray(sigma, dtype=float)
497+
if sigma_arr.shape != (nportfolio, nportfolio):
498+
raise ValueError(
499+
f"sigma must be a square array with shape ({nportfolio}, "
500+
f"{nportfolio}), where {nportfolio} is the number of test portfolios. "
501+
f"The shape of sigma is {sigma_arr.shape}."
502+
)
503+
if not np.all(np.isfinite(sigma_arr)):
504+
raise ValueError("sigma must not contain missing or infinite values.")
505+
scale = np.max(np.abs(sigma_arr))
506+
if np.max(np.abs(sigma_arr - sigma_arr.T)) > 1e-8 * scale:
507+
raise ValueError("sigma must be symmetric.")
508+
vals = np.linalg.eigvalsh((sigma_arr + sigma_arr.T) / 2)
509+
if not vals[0] > nportfolio * np.finfo(float).eps * vals[-1]:
510+
raise ValueError("sigma must be positive definite.")
511+
return sigma_arr
512+
467513
def __str__(self) -> str:
468514
out = super().__str__()
469515
if np.any(self._sigma != np.eye(self.portfolios.shape[1])):
@@ -509,8 +555,12 @@ class LinearFactorModel(_LinearFactorModelBase):
509555
Flag indicating whether the risk-free rate should be estimated
510556
from returns along other risk premia. If False, the returns are
511557
assumed to be excess returns using the correct risk-free rate.
512-
sigma : array_like
513-
Positive definite residual covariance (nportfolio by nportfolio)
558+
sigma : array_like, optional
559+
Positive definite residual covariance (nportfolio by nportfolio) that
560+
weights the test portfolios in the second stage. A ``ValueError`` is
561+
raised if it does not have this shape, contains missing or infinite
562+
values, or is not symmetric and positive definite. If not provided, the
563+
identity matrix is used.
514564
515565
Notes
516566
-----
@@ -591,8 +641,12 @@ def from_formula(
591641
Flag indicating whether the risk-free rate should be estimated
592642
from returns along other risk premia. If False, the returns are
593643
assumed to be excess returns using the correct risk-free rate.
594-
sigma : array_like
595-
Positive definite residual covariance (nportfolio by nportfolio)
644+
sigma : array_like, optional
645+
Positive definite residual covariance (nportfolio by nportfolio) that
646+
weights the test portfolios in the second stage. A ``ValueError`` is
647+
raised if it does not have this shape, contains missing or infinite
648+
values, or is not symmetric and positive definite. If not provided,
649+
the identity matrix is used.
596650
597651
Returns
598652
-------
@@ -659,6 +713,7 @@ def fit(
659713
"""
660714
nobs, nf, nport, nrf, s1, s2, s3 = self._boundaries()
661715
excess_returns = not self._risk_free
716+
self._check_debiased(debiased)
662717
f = self.factors.ndarray
663718
p = self.portfolios.ndarray
664719
nport = p.shape[1]
@@ -1009,7 +1064,9 @@ def fit(
10091064
mod = LinearFactorModel(
10101065
self.portfolios, self.factors, risk_free=self._risk_free
10111066
)
1012-
res = mod.fit()
1067+
# The starting values do not use the covariance, which cannot be debiased
1068+
# if there are no residual degrees of freedom in the first stage
1069+
res = mod.fit(debiased=False)
10131070
betas_1d = np.asarray(res.betas).ravel()
10141071
lam = np.asarray(res.risk_premia)
10151072
mu = self.factors.ndarray.mean(0)

‎linearmodels/shared/io.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ def _str(v: float) -> str:
1010
"""Preferred basic formatter"""
1111
if np.isnan(v):
1212
return " "
13+
if np.isinf(v):
14+
return f"{v:8.4g}"
1315
av = abs(v)
1416
digits = 0
1517
if av != 0:
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
"""
2+
Tests of the messages for factors that are collinear or contain a constant
3+
4+
has_constant is also True for a matrix that does not have full column rank, so that
5+
collinear factors were reported as containing a constant and the message for factors
6+
without full column rank could not be reached.
7+
"""
8+
9+
import numpy as np
10+
import pandas as pd
11+
import pytest
12+
13+
from linearmodels.asset_pricing.model import (
14+
LinearFactorModel,
15+
LinearFactorModelGMM,
16+
TradedFactorModel,
17+
)
18+
19+
MODELS = [TradedFactorModel, LinearFactorModel, LinearFactorModelGMM]
20+
RANK = r"Model cannot be estimated\. factors do not have full column rank\."
21+
CONSTANT = r"factors must not contain a constant or equivalent\."
22+
23+
24+
@pytest.fixture(scope="module")
25+
def data():
26+
rng = np.random.default_rng(0)
27+
factors = rng.standard_normal((100, 3))
28+
portfolios = factors @ rng.standard_normal((3, 6)) + rng.standard_normal((100, 6))
29+
return portfolios, factors
30+
31+
32+
def collinear_factors(kind, factors):
33+
x1, x2, x3 = factors.T
34+
ones = np.ones_like(x1)
35+
cases = {
36+
"duplicate": np.c_[factors, x1],
37+
"combination": np.c_[factors, x1 - 2 * x2],
38+
"combination first": np.c_[x1 + x2, factors],
39+
# A constant column, or a constant in the span, makes the factors contain one
40+
"duplicate constant": np.c_[factors, ones, ones],
41+
"constant in span": np.c_[x1, x2, 1 - x1 - x2],
42+
"constant in span, collinear": np.c_[x1, x2, 1 - x1 - x2, x1 + x2],
43+
"constant and duplicate": np.c_[factors, x1, ones],
44+
"constant not one": np.c_[factors, 3 * ones],
45+
"constant not one and duplicate": np.c_[factors, 3 * ones, x1],
46+
"scaled constant in span": np.c_[x1, x2, 5 - 2 * x1 - 2 * x2],
47+
"x3 only": x3[:, None],
48+
}
49+
return cases[kind]
50+
51+
52+
@pytest.mark.parametrize("model", MODELS)
53+
@pytest.mark.parametrize("kind", ["duplicate", "combination", "combination first"])
54+
@pytest.mark.parametrize("pandas", [False, True])
55+
def test_collinear_factors(data, model, kind, pandas):
56+
portfolios, factors = data
57+
collinear = collinear_factors(kind, factors)
58+
if pandas:
59+
collinear = pd.DataFrame(collinear)
60+
with pytest.raises(ValueError, match=RANK):
61+
model(portfolios, collinear)
62+
63+
64+
@pytest.mark.parametrize("model", MODELS)
65+
@pytest.mark.parametrize(
66+
"kind",
67+
[
68+
"duplicate constant",
69+
"constant in span",
70+
"constant in span, collinear",
71+
"constant and duplicate",
72+
"constant not one",
73+
"constant not one and duplicate",
74+
"scaled constant in span",
75+
],
76+
)
77+
def test_constant_is_reported_as_a_constant(data, model, kind):
78+
portfolios, factors = data
79+
with pytest.raises(ValueError, match=CONSTANT):
80+
model(portfolios, collinear_factors(kind, factors))
81+
82+
83+
@pytest.mark.parametrize("model", MODELS)
84+
def test_valid_factors(data, model):
85+
portfolios, factors = data
86+
mod = model(portfolios, factors)
87+
assert mod.factors.shape == (100, 3)
88+
mod = model(portfolios, collinear_factors("x3 only", factors))
89+
assert mod.factors.shape == (100, 1)
90+
91+
92+
def test_fewer_observations_than_factors(data):
93+
# A wide matrix does not have full column rank, whatever it contains
94+
portfolios, factors = data
95+
with pytest.raises(ValueError, match=RANK):
96+
TradedFactorModel(portfolios[:2], factors[:2])

‎linearmodels/tests/asset_pricing/test_model.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ def test_errors(data):
169169
TradedFactorModel(p3, data.factors)
170170
with pytest.raises(ValueError, match=r"factors must not contain"):
171171
TradedFactorModel(data.portfolios, f)
172-
with pytest.raises(ValueError, match=r"factors must not contain"):
172+
with pytest.raises(ValueError, match=r"factors do not have full column rank"):
173173
TradedFactorModel(data.portfolios, f2)
174174
with pytest.raises(ValueError, match=r"Model cannot be estimated"):
175175
TradedFactorModel(p5, f5)
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"""
2+
Tests of models with as many observations as regressors in the time-series regressions
3+
4+
With nobs equal to nfactor + 1 the regressions have no residual degrees of freedom,
5+
so the covariance cannot be debiased.
6+
"""
7+
8+
import numpy as np
9+
import pytest
10+
11+
from linearmodels.asset_pricing.model import (
12+
LinearFactorModel,
13+
LinearFactorModelGMM,
14+
TradedFactorModel,
15+
)
16+
17+
NFACTOR = 2
18+
NPORTFOLIO = 5
19+
20+
21+
def make_data(nobs):
22+
rng = np.random.default_rng(0)
23+
factors = rng.standard_normal((nobs, NFACTOR))
24+
betas = rng.standard_normal((NFACTOR, NPORTFOLIO))
25+
portfolios = factors @ betas + 0.1 * rng.standard_normal((nobs, NPORTFOLIO))
26+
return portfolios, factors
27+
28+
29+
@pytest.mark.parametrize("model", [TradedFactorModel, LinearFactorModel])
30+
@pytest.mark.parametrize("fit_options", [{}, {"debiased": True}])
31+
def test_saturated_model_cannot_be_debiased(model, fit_options):
32+
portfolios, factors = make_data(NFACTOR + 1)
33+
mod = model(portfolios, factors)
34+
match = r"cannot be debiased.*\(3\).*\(3\).*Use debiased=False"
35+
with pytest.raises(ValueError, match=match):
36+
mod.fit(**fit_options)
37+
with pytest.raises(ValueError, match=match):
38+
mod.fit(cov_type="kernel", bandwidth=1, **fit_options)
39+
40+
41+
@pytest.mark.parametrize("model", [TradedFactorModel, LinearFactorModel])
42+
def test_saturated_model_without_debiasing(model):
43+
portfolios, factors = make_data(NFACTOR + 1)
44+
res = model(portfolios, factors).fit(debiased=False)
45+
assert res.nobs == NFACTOR + 1
46+
assert np.all(np.isfinite(res.params))
47+
48+
49+
@pytest.mark.parametrize("model", [TradedFactorModel, LinearFactorModel])
50+
def test_one_more_observation_can_be_debiased(model):
51+
portfolios, factors = make_data(NFACTOR + 2)
52+
res = model(portfolios, factors).fit()
53+
assert np.all(np.isfinite(res.cov))
54+
55+
56+
def test_gmm_starting_values_are_not_debiased(monkeypatch):
57+
# The starting values come from LinearFactorModel, and do not use the covariance,
58+
# so that they must not fail if the first stage has no degrees of freedom
59+
portfolios, factors = make_data(40)
60+
calls = []
61+
fit = LinearFactorModel.fit
62+
63+
def record(self, *args, **kwargs):
64+
calls.append(kwargs.get("debiased", True))
65+
return fit(self, *args, **kwargs)
66+
67+
monkeypatch.setattr(LinearFactorModel, "fit", record)
68+
LinearFactorModelGMM(portfolios, factors).fit(disp=0)
69+
assert calls == [False]

0 commit comments

Comments
 (0)