BUG: Raise a clear error when a panel model has no residual degrees o…

linearmodels/panel/covariance.py

@@ -105,6 +105,13 @@ def __init__(
         self._nobs_eff = self._nobs - extra_df
         if debiased:
             self._nobs_eff -= self._nvar
+        if self._nobs_eff <= 0:
+            regressors = f"{self._nvar} regressors and " if debiased else ""
+            raise ValueError(
+                "The covariance of the model cannot be estimated since the model has "
+                f"no residual degrees of freedom. It has {self._nobs} observations "
+                f"and uses {regressors}{extra_df} degrees of freedom for the effects."
+            )
         self._scale = self._nobs / self._nobs_eff
         self._name = "Unadjusted"
 

linearmodels/panel/data.py

@@ -442,6 +442,9 @@ def _demean_both(self, weights: PanelData | None) -> PanelData:
         e = self.demean(group, weights=weights)
         assert isinstance(e, PanelData)
         d = self.dummies(dummy, drop_first=True)
+        if d.shape[1] == 0:
+            # A single entity or period: there are no dummy variables to remove
+            return e
         d.index = e.index
         d_pd = PanelData(d).demean(group, weights=weights)
         d_arr = d_pd.values2d

linearmodels/panel/model.py

@@ -523,9 +523,10 @@ def _grid_to_rows(self, grid: PanelData, labelled: bool) -> PanelData:
         return PanelData(rows)
 
     def _check_exog_rank(self) -> int:
-        if not self._check_rank:
-            return self.exog.shape[1]
         x = cast("linearmodels.typing.data.Float64Array", self.exog.values2d)
+        if not self._check_rank:
+            # The shape of the PanelData is (variables, periods, entities)
+            return x.shape[1]
         _, _, rank_of_x, _ = _lstsq(x, np.ones(x.shape[0]))
         if rank_of_x < x.shape[1]:
             raise ValueError(
@@ -574,8 +575,9 @@ def _validate_data(self) -> None:
             self._not_null = cast("BoolArray", np.asarray(~missing))
 
         w_df = self.weights.dataframe
-        if np.any(np.asarray(w_df) <= 0):
-            raise ValueError("weights must be strictly positive.")
+        w_values = np.asarray(w_df)
+        if np.any(w_values <= 0) or np.any(np.isinf(w_values)):
+            raise ValueError("weights must be strictly positive and finite.")
         w_df = w_df / w_df.mean()
         self.weights = PanelData(w_df)
         rank_of_x = self._check_exog_rank()
@@ -607,6 +609,10 @@ def _f_statistic(
         weps_const = y
         num_df = x.shape[1]
         name = "Model F-statistic (homoskedastic)"
+        if df_resid <= 0:
+            return InvalidTestStatistic(
+                "The model has no residual degrees of freedom", name=name
+            )
         if self.has_constant:
             if num_df == 1:
                 return InvalidTestStatistic("Model contains only a constant", name=name)
@@ -2205,17 +2211,29 @@ def fit(
                 df_num -= 1
             weps_pooled = wy - wx @ _lstsq(wx, wy, rcond=None)[0]
             resid_ss_pooled = float(np.squeeze(weps_pooled.T @ weps_pooled))
-            num = (resid_ss_pooled - resid_ss) / df_num
-
-            denom = resid_ss / df_denom
-            stat = num / denom
-            f_pooled = WaldTestStatistic(
-                stat,
-                "Effects are zero",
-                df_num,
-                df_denom=df_denom,
-                name="Pooled F-statistic",
+            name = "Pooled F-statistic"
+            f_pooled: (
+                WaldTestStatistic | InapplicableTestStatistic | InvalidTestStatistic
             )
+            if df_num <= 0:
+                # A single entity (or period) with a constant: the effect is the constant
+                f_pooled = InapplicableTestStatistic(
+                    reason="The effects do not use any degrees of freedom", name=name
+                )
+            elif df_denom <= 0:
+                f_pooled = InvalidTestStatistic(
+                    "The model has no residual degrees of freedom", name=name
+                )
+            else:
+                num = (resid_ss_pooled - resid_ss) / df_num
+                denom = resid_ss / df_denom
+                f_pooled = WaldTestStatistic(
+                    num / denom,
+                    "Effects are zero",
+                    df_num,
+                    df_denom=df_denom,
+                    name=name,
+                )
             res.update(f_pooled=f_pooled)
             effects = DataFrame(
                 eps_effects - eps,
@@ -2421,10 +2439,20 @@ def fit(
 
         wx = root_w * x
         wy = root_w * y
-        params = _lstsq(wx, wy, rcond=None)[0]
+        params, _, rank, _ = _lstsq(wx, wy, rcond=None)
+        if self._check_rank and rank < x.shape[1]:
+            raise ValueError(
+                "The regressors are collinear in the means of the entities, which "
+                f"are the {y.shape[0]} observations of the regression, so that the "
+                "parameters are not identified. This happens if there are fewer "
+                "entities than regressors, or if a regressor is a combination of "
+                "the others only in the means, e.g., the dummy variables of the "
+                "periods and the constant in a balanced panel. If you wish to "
+                "proceed, you can set check_rank=False."
+            )
 
         df_resid = y.shape[0] - x.shape[1]
-        df_model = (x.shape[1],)
+        df_model = x.shape[1]
         nobs = y.shape[0]
         cov_config = self._setup_clusters(cov_config)
         extra_df = 0
@@ -3311,17 +3339,16 @@ def _mr(ex: linearmodels.typing.data.Float64Array) -> int:
         if valid_blocks.sum() < exog.shape[1]:
 
             warnings.warn(
-                "The number of time-series observation available to estimate "
-                "cross-sectional\nregressions, {}, is less than the number of "
-                "parameters in the model. Parameter\ninference is not "
-                "available.".format(valid_blocks.sum()),
+                "The number of cross-sectional regressions that can be estimated, "
+                "{}, is less than the number of parameters in the model. Parameter "
+                "inference is not available.".format(valid_blocks.sum()),
                 InferenceUnavailableWarning,
                 stacklevel=3,
             )
         elif valid_blocks.sum() < valid_blocks.shape[0]:
 
             warnings.warn(
-                "{} of the time-series regressions cannot be estimated due to "
+                "{} of the cross-sectional regressions cannot be estimated due to "
                 "deficient rank.".format(valid_blocks.shape[0] - valid_blocks.sum()),
                 MissingValueWarning,
                 stacklevel=3,

linearmodels/panel/utility.py

@@ -642,6 +642,17 @@ class PanelModelData(NamedTuple):
     clusters: DataFrame
 
 
+def _integers(
+    rng: numpy.random.RandomState | numpy.random.Generator,
+    high: int,
+    size: tuple[int, ...],
+) -> linearmodels.typing.data.IntArray:
+    """Random integers in [0, high) from a RandomState or a Generator"""
+    if isinstance(rng, numpy.random.Generator):
+        return rng.integers(0, high, size)
+    return rng.randint(0, high, size)
+
+
 def generate_panel_data(
     nentity: int = 971,
     ntime: int = 7,
@@ -650,7 +661,7 @@ def generate_panel_data(
     missing: float = 0,
     other_effects: int = 2,
     ncats: int | list[int] = 4,
-    rng: numpy.random.RandomState | None = None,
+    rng: numpy.random.RandomState | numpy.random.Generator | None = None,
 ) -> PanelModelData:
     """
     Simulate panel data for testing
@@ -673,9 +684,9 @@ def generate_panel_data(
         The number of categories to use in other_effects and variance
         clusters. If list-like, then it must have as many elements
         as other_effects.
-    rng : RandomState
-        A NumPy RandomState instance. If not provided, one is initialized
-        using a fixed seed.
+    rng : RandomState or Generator
+        A NumPy RandomState or Generator instance. If not provided, a
+        RandomState is initialized using a fixed seed.
 
     Returns
     -------
@@ -721,12 +732,11 @@ def generate_panel_data(
         _c = []
         for i in range(other_effects):
             nc = ncats[i]
-            _c.append(rng.randint(0, nc, (1, t, n)))
+            _c.append(_integers(rng, nc, (1, t, n)))
         c = np.concatenate(_c, 0)
 
     vcats = [f"varcat.{i}" for i in range(2)]
-    vc2 = np.ones((2, t, 1)) @ rng.randint(0, n // 2, (2, 1, n))
-    vc1 = vc2[[0]]
+    vc2 = np.ones((2, t, 1)) @ _integers(rng, n // 2, (2, 1, n))
 
     if const:
         x[0] = 1.0
@@ -760,14 +770,9 @@ def generate_panel_data(
         c, items=cats, major_axis=time, minor_axis=entities, swap=True
     )
     other_eff = c_df.reindex(index)
-    vc1_df = panel_to_frame(
-        vc1, items=vcats[:1], major_axis=time, minor_axis=entities, swap=True
-    )
-    vc1_df = vc1_df.reindex(index)
-    vc2_df = panel_to_frame(
+    clusters = panel_to_frame(
         vc2, items=vcats, major_axis=time, minor_axis=entities, swap=True
     )
-    vc2_df = vc2_df.reindex(index)
-    clusters = concat([vc1_df, vc2_df], sort=False)
+    clusters = clusters.reindex(index)
     data = concat([y_df, x_df], axis=1, sort=False)
     return PanelModelData(data, w_df, other_eff, clusters)

linearmodels/tests/panel/test_between_means.py

@@ -0,0 +1,60 @@
+"""
+BetweenOLS where the means of the regressors are collinear, and df_model
+
+The regression of BetweenOLS uses the means of the entities. The regressors can be
+collinear in the means when they are not collinear in the data: there are fewer
+entities than regressors, or the dummy variables of the periods are the constant 1/T in
+the means of a balanced panel and so they are collinear with the constant. The
+covariance raised numpy's "Singular matrix" error when it was used (and the parameters
+were not identified), and the error has to say what is wrong.
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from linearmodels.datasets import wage_panel
+from linearmodels.panel import BetweenOLS
+
+
+def make_panel(nentity, nperiod, nvar, seed=0):
+    rg = np.random.default_rng(seed)
+    index = pd.MultiIndex.from_product(
+        [np.arange(nentity), np.arange(nperiod)], names=["entity", "time"]
+    )
+    nobs = nentity * nperiod
+    x = pd.DataFrame(
+        rg.standard_normal((nobs, nvar)),
+        index=index,
+        columns=[f"x{i}" for i in range(nvar)],
+    )
+    y = pd.Series(rg.standard_normal(nobs), index=index, name="y")
+    return y, x
+
+
+def test_between_with_more_regressors_than_entities():
+    y, x = make_panel(2, 3, 3)
+    with pytest.raises(ValueError, match="collinear in the means of the entities"):
+        BetweenOLS(y, x).fit()
+    # The rank is not checked if the user does not want it
+    res = BetweenOLS(y, x, check_rank=False).fit(debiased=False)
+    assert res.params.shape == (3,)
+
+
+def test_between_with_time_dummies_in_a_balanced_panel():
+    data = wage_panel.load()
+    year = pd.Categorical(data.year)
+    data = data.set_index(["nr", "year"])
+    data["year"] = year
+    data["const"] = 1.0
+    with pytest.raises(ValueError, match="collinear in the means of the entities"):
+        BetweenOLS(data.lwage, data[["const", "expersq", "union", "year"]]).fit()
+    res = BetweenOLS(data.lwage, data[["const", "expersq", "union"]]).fit()
+    assert res.df_resid == 545 - 3
+
+
+def test_between_df_model_is_the_number_of_regressors():
+    y, x = make_panel(10, 3, 2)
+    res = BetweenOLS(y, x).fit()
+    assert res.df_model == 2
+    assert isinstance(res.df_model, int)

linearmodels/tests/panel/test_both_effects_degenerate.py

@@ -0,0 +1,49 @@
+"""
+Entity and time effects in a panel with a single entity or a single period
+
+The effects remove all of the variation of the regressors. The error has to say that the
+effects absorbed the variables. The default algorithm that removes both effects
+raised "No objects to concatenate" from pandas, since it uses the dummy variables of the
+shorter dimension, and there are none.
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from linearmodels.panel import PanelOLS
+from linearmodels.panel.data import PanelData
+from linearmodels.panel.utility import AbsorbingEffectError
+
+
+def make_panel(nentity, nperiod, nvar, seed=0):
+    rg = np.random.default_rng(seed)
+    index = pd.MultiIndex.from_product(
+        [np.arange(nentity), np.arange(nperiod)], names=["entity", "time"]
+    )
+    nobs = nentity * nperiod
+    x = pd.DataFrame(
+        rg.standard_normal((nobs, nvar)),
+        index=index,
+        columns=[f"x{i}" for i in range(nvar)],
+    )
+    y = pd.Series(rg.standard_normal(nobs), index=index, name="y")
+    return y, x
+
+
+@pytest.mark.parametrize("low_memory", [None, False, True])
+@pytest.mark.parametrize("shape", [(1, 6), (6, 1)])
+def test_entity_and_time_effects_absorb_a_single_entity_or_period(shape, low_memory):
+    y, x = make_panel(*shape, 2)
+    mod = PanelOLS(y, x, entity_effects=True, time_effects=True)
+    with pytest.raises(AbsorbingEffectError, match="absorbed"):
+        mod.fit(low_memory=low_memory)
+
+
+@pytest.mark.parametrize("shape", [(1, 6), (6, 1)])
+def test_demean_both_of_a_single_entity_or_period(shape):
+    # There is nothing to remove besides the means of the group with the observations
+    _, x = make_panel(*shape, 2)
+    demeaned = PanelData(x).demean("both")
+    assert demeaned.dataframe.shape == x.shape
+    np.testing.assert_allclose(demeaned.values2d, 0.0, atol=1e-14)