feat(review): extend debate to Stages 14/18 and the tournament to Stage 9

config.researchclaw.example.yaml

@@ -71,6 +71,61 @@ llm:
   # fallback_models:
   #   - "mistral"
 
+  # --- Independent reviewer / judge model (opt-in) ---
+  # By default the model that produced an artifact also reviews and judges it.
+  # Set reviewer_model to have a DIFFERENT model answer Stage 18 (peer review)
+  # and the Stage 20 quality gate; generation and revision stay with the author
+  # model. Leaving it empty keeps the legacy same-model behaviour.
+  # Set reviewer_model alone to judge with another model on the SAME endpoint,
+  # or add the reviewer_* fields below for a fully independent provider
+  # (e.g. generator = GPT, reviewer = Claude). Stage 18 records which model
+  # judged the paper in review_provenance.json.
+  reviewer_model: ""          # e.g. "claude-sonnet-4-6" — empty disables
+  reviewer_provider: ""       # e.g. "anthropic" — defaults to provider above
+  reviewer_base_url: ""       # defaults to base_url above
+  reviewer_api_key_env: ""    # e.g. "ANTHROPIC_API_KEY" — defaults to api_key_env
+  reviewer_api_key: ""        # inline key (optional; prefer the env var)
+
+  # --- Multi-model debate: Stages 8, 14 and 18 (opt-in) ---
+  # Stage 8 = hypothesis generation, Stage 14 = result analysis,
+  # Stage 18 = peer review (three reviewer roles, each on a different model).
+  # Builds a panel from primary_model + reviewer_model + fallback_models
+  # (deduplicated) and binds each debate role to a different model, so the
+  # perspectives genuinely disagree instead of one model arguing with itself.
+  # After `debate_rounds` rebuttal round(s), reviewer_model RANKS the positions.
+  # On Stages 8 and 14 primary_model then writes the final text from that
+  # ranking — ranking and writing are split so the disagreements are not
+  # averaged into consensus. Stage 18 is the deliberate exception: the reviewer
+  # writes the report itself, because handing the write-up back to the author
+  # model would defeat the purpose of peer review.
+  # Records debate_record.json. Multiplies LLM calls.
+  #
+  # To enable it you need BOTH of the following, plus a real panel:
+  #   debate_enabled: true       # otherwise the single-model path is used
+  #   tournament_enabled: false  # the tournament takes priority on Stage 8
+  # and primary_model + reviewer_model + fallback_models must dedupe to >= 2
+  # distinct models. A reviewer_model different from primary_model keeps the
+  # ranking independent.
+  # Env overrides: ARC_DEBATE_ROUNDS=<n>, ARC_ABL_DISABLE_DEBATE=1 (ablation:
+  # collapse the panel to a single role).
+  # debate_enabled: false
+  # debate_rounds: 1           # rebuttal rounds; 0 = opening statements only
+
+  # --- Best-of-N tournament: Stages 8 and 9 (opt-in) ---
+  # Stage 8 = hypothesis sets, Stage 9 = experiment designs (candidates are
+  # generated from three fixed stances: ambitious / robust / compute-efficient).
+  # Generates N candidate artifacts from diverse stances (round-robin over
+  # the debate panel when one is available), then reviewer_model scores and ranks
+  # them and only the winner proceeds — the pipeline stays linear, with one
+  # canonical artifact per stage. Blank generations are dropped before judging,
+  # so an empty candidate cannot be handed a fabricated score. Records
+  # tournament_record.json. Multiplies LLM calls (~N generations + 1 judge).
+  # Takes priority over debate when both are enabled.
+  # Env overrides: ARC_TOURNAMENT_CANDIDATES=<n>, ARC_ABL_DISABLE_TOURNAMENT=1
+  # (ablation: collapse to a single candidate).
+  # tournament_enabled: false
+  # tournament_candidates: 3   # < 2 disables the tournament
+
 literature_search:
   # Stage 4 academic search backends. Defaults preserve the current
   # OpenAlex -> Semantic Scholar -> arXiv order.

researchclaw/pipeline/stage_impls/_analysis.py

@@ -10,8 +10,10 @@
 
 from researchclaw.adapters import AdapterBundle
 from researchclaw.config import RCConfig
+from researchclaw.llm import build_panel_llms, build_reviewer_llm
 from researchclaw.llm.client import LLMClient
 from researchclaw.pipeline._domain import _detect_domain, _is_ml_domain
+from researchclaw.pipeline.debate import run_debate
 from researchclaw.pipeline._helpers import (
     StageResult,
     _build_context_preamble,
@@ -601,20 +603,39 @@ def _get_best_sandbox(it: dict) -> dict:
         # stays in the same vocabulary as the rest of the pipeline.
         _analysis_roles = _pm.debate_roles_analysis()
 
-        # --- Multi-perspective debate ---
         perspectives_dir = stage_dir / "perspectives"
         variables = {
             "preamble": preamble,
             "data_context": data_context,
             "context": context,
         }
-        perspectives = _multi_perspective_generate(
-            llm, _analysis_roles, variables, perspectives_dir
-        )
-        # --- Synthesize into unified analysis ---
-        analysis = _synthesize_perspectives(
-            llm, perspectives, "analysis_synthesize", _pm
-        )
+        _panel = build_panel_llms(config)
+        if _panel:
+            # --- Multi-model debate: distinct models argue per role, rebuttal
+            # round(s), then an independent judge ranks and the author model
+            # synthesizes the final analysis from that ranking. ---
+            _judge = build_reviewer_llm(config) or llm
+            analysis, _ = run_debate(
+                _panel,
+                _judge,
+                _analysis_roles,
+                variables,
+                rounds=config.llm.debate_rounds,
+                synth_prompt="analysis_synthesize",
+                out_dir=perspectives_dir,
+                prompts=_pm,
+                author_model=getattr(llm.config, "primary_model", ""),
+                synthesizer=llm,
+            )
+        else:
+            # --- Legacy multi-perspective (single model, no judge) ---
+            perspectives = _multi_perspective_generate(
+                llm, _analysis_roles, variables, perspectives_dir
+            )
+            # --- Synthesize into unified analysis ---
+            analysis = _synthesize_perspectives(
+                llm, perspectives, "analysis_synthesize", _pm
+            )
     else:
         # Template with real data if available
         ms = exp_data["metrics_summary"]

researchclaw/pipeline/stage_impls/_experiment_design.py

@@ -29,6 +29,18 @@
 
 logger = logging.getLogger(__name__)
 
+# Stances for the Stage 9 tournament. Candidates are asked for genuinely
+# different plans rather than N samples of the same one, so the judge has
+# something to choose between.
+_DESIGN_ANGLES = (
+    "Be ambitious: prioritize high-ceiling, novel methods that could yield a "
+    "strong result, accepting higher risk.",
+    "Be robust: prioritize strong, well-known baselines and a clean, defensible "
+    "comparison over novelty.",
+    "Be compute-efficient: design the most decisive experiment that fits a tight "
+    "compute budget — minimal but conclusive.",
+)
+
 
 def _normalize_plan_field(value: Any) -> list:
     """Normalize a plan field (baselines, proposed_methods, ablations, datasets)
@@ -212,13 +224,50 @@ def _execute_experiment_design(
             per_condition_budget_sec=_per_condition_sec,
             available_tier1_datasets=_tier1,
         )
-        resp = _chat_with_prompt(
-            llm,
-            sp.system,
-            sp.user,
-            json_mode=sp.json_mode,
-            max_tokens=sp.max_tokens,
-        )
+        if config.llm.tournament_enabled and config.llm.tournament_candidates >= 2:
+            # --- Best-of-N tournament: generate N candidate plans from diverse
+            # stances, then an independent judge picks the winner. Everything
+            # downstream (YAML parsing, normalization, caps) runs on the winner
+            # alone, so the stage still produces one canonical plan. ---
+            from types import SimpleNamespace
+
+            from researchclaw.llm import build_panel_llms, build_reviewer_llm
+            from researchclaw.pipeline.tournament import (
+                effective_candidates,
+                run_tournament,
+            )
+
+            _gens = build_panel_llms(config) or [llm]
+            _judge = build_reviewer_llm(config) or llm
+            _n = effective_candidates(config.llm.tournament_candidates)
+            _cps = [
+                (
+                    sp.system,
+                    sp.user
+                    + "\n\n## Exploration stance\n"
+                    + _DESIGN_ANGLES[i % len(_DESIGN_ANGLES)],
+                )
+                for i in range(_n)
+            ]
+            _winner, _ = run_tournament(
+                _gens,
+                _judge,
+                _cps,
+                rank_prompt="tournament_rank",
+                out_dir=stage_dir / "tournament",
+                prompts=_pm,
+                author_model=getattr(llm.config, "primary_model", ""),
+                label="plan",
+            )
+            resp = SimpleNamespace(content=_winner)
+        else:
+            resp = _chat_with_prompt(
+                llm,
+                sp.system,
+                sp.user,
+                json_mode=sp.json_mode,
+                max_tokens=sp.max_tokens,
+            )
         raw_yaml = _extract_yaml_block(resp.content)
         try:
             parsed = yaml.safe_load(raw_yaml)

researchclaw/pipeline/stage_impls/_review_publish.py

@@ -14,8 +14,10 @@
 
 from researchclaw.adapters import AdapterBundle
 from researchclaw.config import RCConfig
+from researchclaw.llm import build_panel_llms
 from researchclaw.llm.client import LLMClient
 from researchclaw.pipeline._domain import _detect_domain  # noqa: F401
+from researchclaw.pipeline.debate import run_debate
 from researchclaw.pipeline._helpers import (
     StageResult,
     _build_context_preamble,
@@ -161,6 +163,47 @@ def _build_reviewer_or_generator(config, generator_llm):
 # Stage 18: Peer Review
 # ---------------------------------------------------------------------------
 
+# Reviewer roles for the multi-model peer-review debate. Each is bound to a
+# different panel model so the three reviews are not one model's opinion
+# restated three times.
+_REVIEW_DEBATE_ROLES: dict[str, dict[str, str]] = {
+    "methodology_reviewer": {
+        "system": (
+            "You are a peer reviewer focused on METHODOLOGY and experimental "
+            "design rigor."
+        ),
+        "user": (
+            "Review this paper draft for methodological soundness, baselines, "
+            "ablations, and validity threats.\n\nTopic: {topic}\n\n"
+            "Evidence:\n{experiment_evidence}\n\nDraft:\n{draft}"
+        ),
+    },
+    "domain_reviewer": {
+        "system": (
+            "You are a peer reviewer who is a DOMAIN EXPERT judging novelty "
+            "and significance."
+        ),
+        "user": (
+            "Review this paper draft for novelty, related-work positioning, "
+            "and contribution significance.\n\nTopic: {topic}\n\n"
+            "Draft:\n{draft}"
+        ),
+    },
+    "rigor_reviewer": {
+        "system": (
+            "You are a peer reviewer focused on STATISTICS, reproducibility, "
+            "and claim-evidence consistency."
+        ),
+        "user": (
+            "Review this paper draft for statistical rigor, reproducibility, "
+            "and whether every claim is supported by the evidence. Flag "
+            "unsupported numbers.\n\nEvidence:\n{experiment_evidence}\n\n"
+            "Draft:\n{draft}"
+        ),
+    },
+}
+
+
 def _execute_peer_review(
     stage_dir: Path,
     run_dir: Path,
@@ -196,7 +239,31 @@ def _execute_peer_review(
         except Exception:  # noqa: BLE001
             pass
 
-    if _review_llm is not None:
+    _panel = build_panel_llms(config)
+    if _panel and _review_llm is not None:
+        # --- Multi-model peer-review debate: distinct models each play an
+        # independent reviewer role (methodology / domain / rigor) and rebut
+        # each other, then the independent reviewer synthesizes the report.
+        # No `synthesizer` is passed here on purpose: handing the write-up back
+        # to the author model would defeat the point of the stage. ---
+        _pm = prompts or PromptManager()
+        _variables = {
+            "topic": config.research.topic,
+            "draft": draft + _quality_suffix,
+            "experiment_evidence": experiment_evidence,
+        }
+        reviews, _ = run_debate(
+            _panel,
+            _review_llm,
+            _REVIEW_DEBATE_ROLES,
+            _variables,
+            rounds=config.llm.debate_rounds,
+            synth_prompt="review_synthesize",
+            out_dir=stage_dir / "debate",
+            prompts=_pm,
+            author_model=_author_model,
+        )
+    elif _review_llm is not None:
         _pm = prompts or PromptManager()
         _overlay = _get_evolution_overlay(run_dir, "peer_review")
         sp = _pm.for_stage(

researchclaw/prompts/shared.py

@@ -790,6 +790,25 @@
         ),
         "max_tokens": 8192,
     },
+    "review_synthesize": {
+        "system": (
+            "You are the area chair synthesizing several independent peer "
+            "reviews into one decision-oriented review report. Do not flatten "
+            "disagreement — surface the most serious concerns prominently and "
+            "do not soften them."
+        ),
+        "user": (
+            "Below are independent reviews of the paper from different "
+            "reviewers.\n"
+            "Synthesize them into a final review report with these sections:\n"
+            "## Summary\n## Strengths\n## Weaknesses (most serious first)\n"
+            "## Actionable Revisions (numbered, specific)\n"
+            "## Recommendation (ACCEPT / MINOR REVISION / MAJOR REVISION / "
+            "REJECT)\n\n"
+            "{perspectives}"
+        ),
+        "max_tokens": 6144,
+    },
     "tournament_rank": {
         "system": (
             "You score and rank competing research artifacts, distinct from the "