Merge pull request #560 from propcgamer20-png/fix/significance-confidence-affects-verdict

fix: significance_report's confidence parameter now affects the significant flag

Author: yakew7

faircode/significance.py

@@ -93,6 +93,15 @@ def permutation_test(group_a, group_b, n_permutations=10000, random_state=42):
 def significance_report(group_a, group_b, n_resamples=10000,
                         n_permutations=10000, confidence=0.95,
                         random_state=42):
+    """Bootstrap CI + permutation p-value for the gap mean(a) - mean(b).
+
+    `confidence` sets both the width of the returned CI *and* the
+    significance threshold: `significant` is `p_value < (1 - confidence)`.
+    At the default `confidence=0.95` that is the usual `p < 0.05`; a caller
+    passing `confidence=0.99` gets the correspondingly stricter `p < 0.01`,
+    so the CI and the verdict move together instead of the verdict being
+    pinned at 0.05 regardless.
+    """
     a = _as_array(group_a)
     b = _as_array(group_b)
     gap, ci_low, ci_high = bootstrap_ci(a, b, n_resamples, confidence,
@@ -105,7 +114,7 @@ def significance_report(group_a, group_b, n_resamples=10000,
         "ci_low": ci_low,
         "ci_high": ci_high,
         "p_value": p_value,
-        "significant": p_value < 0.05,
+        "significant": p_value < (1.0 - confidence),
         "n_a": n_a,
         "n_b": n_b,
         "small_sample_warning": n_a < 30 or n_b < 30,

tests/test_significance.py

@@ -37,6 +37,27 @@ def test_separated_groups_are_significant_and_ci_excludes_zero():
     assert rep["ci_low"] <= rep["gap"] <= rep["ci_high"]
 
 
+def test_confidence_tightens_the_significance_threshold_not_just_the_ci():
+    # A gap with 0.01 < p < 0.05 is significant at the default confidence
+    # (0.95 -> p < 0.05) but not at confidence=0.99 (-> p < 0.01). Before
+    # #548 the `significant` flag was pinned to p < 0.05 regardless of
+    # `confidence`, which only widened/narrowed the CI.
+    rng = np.random.default_rng(0)
+    for _ in range(5):
+        a = rng.binomial(1, 0.55, size=60).astype(float)
+        b = rng.binomial(1, 0.35, size=60).astype(float)
+
+    r95 = significance_report(a, b, n_resamples=3000, n_permutations=3000,
+                              confidence=0.95, random_state=4)
+    r99 = significance_report(a, b, n_resamples=3000, n_permutations=3000,
+                              confidence=0.99, random_state=4)
+
+    assert r95["p_value"] == r99["p_value"]          # same permutation test
+    assert 0.01 < r95["p_value"] < 0.05
+    assert r95["significant"] is True
+    assert r99["significant"] is False
+
+
 # ── Determinism ──────────────────────────────────────────────────────────────
 def test_random_state_makes_results_deterministic():
     a = [1] * 40 + [0] * 60