Merge pull request #400 from CCPBioSim/398-bug-regression-tests-intermittently-fail

Improve Regression Test Reproducibility in CI

Author: harryswift01

.github/workflows/pr.yaml

@@ -93,9 +93,7 @@ jobs:
         run: |
           python -m pytest tests/regression \
             -m "not slow" \
-            -n auto \
-            --dist=loadscope \
-            -k "${{ matrix.system }}" \
+            --system "${{ matrix.system }}" \
             -vv \
             --durations=20
 

.github/workflows/weekly-regression.yaml

@@ -64,11 +64,9 @@ jobs:
 
       - name: Run regression tests (slow, per system)
         run: |
-          pytest tests/regression \
-            -k "${{ matrix.system }}" \
+          python -m pytest tests/regression \
+            --system "${{ matrix.system }}" \
             --run-slow \
-            -n auto \
-            --dist=loadscope \
             -vv \
             --durations=20
 

CodeEntropy/levels/nodes/accumulators.py

@@ -86,17 +86,24 @@ def run(self, shared_data: dict[str, Any]) -> dict[str, Any]:
 
     @staticmethod
     def _build_group_index(groups: dict[int, Any]) -> GroupIndex:
-        """Build group id <-> index mappings.
+        """Build deterministic mappings between group IDs and contiguous indices.
+
+        Group IDs are sorted before indices are assigned, ensuring that the
+        resulting mappings do not depend on dictionary insertion order.
 
         Args:
-            groups: Mapping of group id to group members.
+            groups: Mapping of group IDs to their members.
 
         Returns:
-            GroupIndex mapping object.
+            Bidirectional mappings between group IDs and contiguous indices.
         """
-        group_ids = list(groups.keys())
-        gid2i = {gid: i for i, gid in enumerate(group_ids)}
-        return GroupIndex(group_id_to_index=gid2i, index_to_group_id=list(group_ids))
+        group_ids = sorted(groups)
+        gid2i = {gid: index for index, gid in enumerate(group_ids)}
+
+        return GroupIndex(
+            group_id_to_index=gid2i,
+            index_to_group_id=group_ids,
+        )
 
     @staticmethod
     def _build_accumulators(n_groups: int) -> CovarianceAccumulators:

conda-recipe/meta.yaml

@@ -38,7 +38,6 @@ requirements:
     - dask >=2026.1.2,<2026.2.0
     - distributed >=2026.1.2,<2026.2.0
     - dask-jobqueue >=0.9,<0.10
-    - pytest-xdist >=3.8, <3.9
     - numba >=0.65.1, <0.70
 
 test:

pyproject.toml

@@ -64,7 +64,6 @@ testing = [
     "pytest>=9.0,<10.0",
     "pytest-cov>=7.0,<8.0",
     "pytest-sugar>=1.1,<2.0",
-    "pytest-xdist"
 ]
 pre-commit = [
     "pre-commit>=4.5,<5.0",
@@ -89,7 +88,7 @@ CodeEntropy = "CodeEntropy.cli:main"
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 
-addopts = "-n auto -vv -ra"
+addopts = "-vv -ra"
 
 markers = [
   "regression: end-to-end regression tests against baselines",

tests/regression/conftest.py

@@ -1,9 +1,5 @@
 from __future__ import annotations
 
-import os
-import random
-
-import numpy as np
 import pytest
 
 
@@ -39,17 +35,11 @@ def pytest_addoption(parser: pytest.Parser) -> None:
 
 def pytest_configure(config: pytest.Config) -> None:
     """
-    Register markers and enforce deterministic behavior.
+    Register regression test markers.
     """
     config.addinivalue_line("markers", "regression: end-to-end regression tests")
     config.addinivalue_line("markers", "slow: long-running tests (20-30+ minutes)")
 
-    seed = 0
-    random.seed(seed)
-    np.random.seed(seed)
-
-    os.environ["PYTHONHASHSEED"] = "0"
-
 
 def pytest_collection_modifyitems(
     config: pytest.Config, items: list[pytest.Item]

tests/regression/helpers.py

@@ -3,6 +3,7 @@
 import json
 import os
 import subprocess
+import sys
 import tarfile
 import urllib.request
 from dataclasses import dataclass
@@ -366,12 +367,25 @@ def run_codeentropy_with_config(*, workdir: Path, config_src: Path) -> RunResult
 
     (workdir / "config.yaml").write_text(yaml.safe_dump(cooked, sort_keys=False))
 
+    env = os.environ.copy()
+    env.update(
+        {
+            "PYTHONHASHSEED": "0",
+            "OMP_NUM_THREADS": "1",
+            "OPENBLAS_NUM_THREADS": "1",
+            "MKL_NUM_THREADS": "1",
+            "NUMEXPR_NUM_THREADS": "1",
+            "BLIS_NUM_THREADS": "1",
+            "VECLIB_MAXIMUM_THREADS": "1",
+        }
+    )
+
     proc = subprocess.run(
-        ["python", "-m", "CodeEntropy"],
+        [sys.executable, "-m", "CodeEntropy"],
         cwd=str(workdir),
         capture_output=True,
         text=True,
-        env={**os.environ},
+        env=env,
     )
 
     (workdir / "codeentropy_stdout.txt").write_text(proc.stdout or "")

tests/regression/test_regression.py

@@ -10,27 +10,6 @@
 from tests.regression.cases import discover_cases
 from tests.regression.helpers import run_codeentropy_with_config
 
-CASES = discover_cases()
-
-
-def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
-    """
-    Dynamically parametrize regression test cases.
-
-    This hook enables pytest-xdist to distribute individual regression cases
-    across multiple workers by generating test parametrization at collection time.
-
-    Args:
-        metafunc (pytest.Metafunc): Pytest metafunction object used to inspect
-            and modify test function parametrization.
-    """
-    if "case" in metafunc.fixturenames:
-        metafunc.parametrize(
-            "case",
-            CASES,
-            ids=[c.id for c in CASES],
-        )
-
 
 def _group_index(payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """
@@ -139,6 +118,7 @@ def _compare_grouped(
 
 
 @pytest.mark.regression
+@pytest.mark.parametrize("case", discover_cases())
 def test_regression_matches_baseline(
     tmp_path: Path,
     case,