diff --git a/config.researchclaw.example.yaml b/config.researchclaw.example.yaml index 16bf8fd9f..8d0d0f5ff 100644 --- a/config.researchclaw.example.yaml +++ b/config.researchclaw.example.yaml @@ -71,6 +71,61 @@ llm: # fallback_models: # - "mistral" + # --- Independent reviewer / judge model (opt-in) --- + # By default the model that produced an artifact also reviews and judges it. + # Set reviewer_model to have a DIFFERENT model answer Stage 18 (peer review) + # and the Stage 20 quality gate; generation and revision stay with the author + # model. Leaving it empty keeps the legacy same-model behaviour. + # Set reviewer_model alone to judge with another model on the SAME endpoint, + # or add the reviewer_* fields below for a fully independent provider + # (e.g. generator = GPT, reviewer = Claude). Stage 18 records which model + # judged the paper in review_provenance.json. + reviewer_model: "" # e.g. "claude-sonnet-4-6" — empty disables + reviewer_provider: "" # e.g. "anthropic" — defaults to provider above + reviewer_base_url: "" # defaults to base_url above + reviewer_api_key_env: "" # e.g. "ANTHROPIC_API_KEY" — defaults to api_key_env + reviewer_api_key: "" # inline key (optional; prefer the env var) + + # --- Multi-model debate: Stages 8, 14 and 18 (opt-in) --- + # Stage 8 = hypothesis generation, Stage 14 = result analysis, + # Stage 18 = peer review (three reviewer roles, each on a different model). + # Builds a panel from primary_model + reviewer_model + fallback_models + # (deduplicated) and binds each debate role to a different model, so the + # perspectives genuinely disagree instead of one model arguing with itself. + # After `debate_rounds` rebuttal round(s), reviewer_model RANKS the positions. + # On Stages 8 and 14 primary_model then writes the final text from that + # ranking — ranking and writing are split so the disagreements are not + # averaged into consensus. Stage 18 is the deliberate exception: the reviewer + # writes the report itself, because handing the write-up back to the author + # model would defeat the purpose of peer review. + # Records debate_record.json. Multiplies LLM calls. + # + # To enable it you need BOTH of the following, plus a real panel: + # debate_enabled: true # otherwise the single-model path is used + # tournament_enabled: false # the tournament takes priority on Stage 8 + # and primary_model + reviewer_model + fallback_models must dedupe to >= 2 + # distinct models. A reviewer_model different from primary_model keeps the + # ranking independent. + # Env overrides: ARC_DEBATE_ROUNDS=, ARC_ABL_DISABLE_DEBATE=1 (ablation: + # collapse the panel to a single role). + # debate_enabled: false + # debate_rounds: 1 # rebuttal rounds; 0 = opening statements only + + # --- Best-of-N tournament: Stages 8 and 9 (opt-in) --- + # Stage 8 = hypothesis sets, Stage 9 = experiment designs (candidates are + # generated from three fixed stances: ambitious / robust / compute-efficient). + # Generates N candidate artifacts from diverse stances (round-robin over + # the debate panel when one is available), then reviewer_model scores and ranks + # them and only the winner proceeds — the pipeline stays linear, with one + # canonical artifact per stage. Blank generations are dropped before judging, + # so an empty candidate cannot be handed a fabricated score. Records + # tournament_record.json. Multiplies LLM calls (~N generations + 1 judge). + # Takes priority over debate when both are enabled. + # Env overrides: ARC_TOURNAMENT_CANDIDATES=, ARC_ABL_DISABLE_TOURNAMENT=1 + # (ablation: collapse to a single candidate). + # tournament_enabled: false + # tournament_candidates: 3 # < 2 disables the tournament + literature_search: # Stage 4 academic search backends. Defaults preserve the current # OpenAlex -> Semantic Scholar -> arXiv order. diff --git a/researchclaw/pipeline/stage_impls/_analysis.py b/researchclaw/pipeline/stage_impls/_analysis.py index 5d4b4ee0f..f6aee9720 100644 --- a/researchclaw/pipeline/stage_impls/_analysis.py +++ b/researchclaw/pipeline/stage_impls/_analysis.py @@ -10,8 +10,10 @@ from researchclaw.adapters import AdapterBundle from researchclaw.config import RCConfig +from researchclaw.llm import build_panel_llms, build_reviewer_llm from researchclaw.llm.client import LLMClient from researchclaw.pipeline._domain import _detect_domain, _is_ml_domain +from researchclaw.pipeline.debate import run_debate from researchclaw.pipeline._helpers import ( StageResult, _build_context_preamble, @@ -601,20 +603,39 @@ def _get_best_sandbox(it: dict) -> dict: # stays in the same vocabulary as the rest of the pipeline. _analysis_roles = _pm.debate_roles_analysis() - # --- Multi-perspective debate --- perspectives_dir = stage_dir / "perspectives" variables = { "preamble": preamble, "data_context": data_context, "context": context, } - perspectives = _multi_perspective_generate( - llm, _analysis_roles, variables, perspectives_dir - ) - # --- Synthesize into unified analysis --- - analysis = _synthesize_perspectives( - llm, perspectives, "analysis_synthesize", _pm - ) + _panel = build_panel_llms(config) + if _panel: + # --- Multi-model debate: distinct models argue per role, rebuttal + # round(s), then an independent judge ranks and the author model + # synthesizes the final analysis from that ranking. --- + _judge = build_reviewer_llm(config) or llm + analysis, _ = run_debate( + _panel, + _judge, + _analysis_roles, + variables, + rounds=config.llm.debate_rounds, + synth_prompt="analysis_synthesize", + out_dir=perspectives_dir, + prompts=_pm, + author_model=getattr(llm.config, "primary_model", ""), + synthesizer=llm, + ) + else: + # --- Legacy multi-perspective (single model, no judge) --- + perspectives = _multi_perspective_generate( + llm, _analysis_roles, variables, perspectives_dir + ) + # --- Synthesize into unified analysis --- + analysis = _synthesize_perspectives( + llm, perspectives, "analysis_synthesize", _pm + ) else: # Template with real data if available ms = exp_data["metrics_summary"] diff --git a/researchclaw/pipeline/stage_impls/_experiment_design.py b/researchclaw/pipeline/stage_impls/_experiment_design.py index d14fd63c7..8738d5eec 100644 --- a/researchclaw/pipeline/stage_impls/_experiment_design.py +++ b/researchclaw/pipeline/stage_impls/_experiment_design.py @@ -29,6 +29,18 @@ logger = logging.getLogger(__name__) +# Stances for the Stage 9 tournament. Candidates are asked for genuinely +# different plans rather than N samples of the same one, so the judge has +# something to choose between. +_DESIGN_ANGLES = ( + "Be ambitious: prioritize high-ceiling, novel methods that could yield a " + "strong result, accepting higher risk.", + "Be robust: prioritize strong, well-known baselines and a clean, defensible " + "comparison over novelty.", + "Be compute-efficient: design the most decisive experiment that fits a tight " + "compute budget — minimal but conclusive.", +) + def _normalize_plan_field(value: Any) -> list: """Normalize a plan field (baselines, proposed_methods, ablations, datasets) @@ -212,13 +224,50 @@ def _execute_experiment_design( per_condition_budget_sec=_per_condition_sec, available_tier1_datasets=_tier1, ) - resp = _chat_with_prompt( - llm, - sp.system, - sp.user, - json_mode=sp.json_mode, - max_tokens=sp.max_tokens, - ) + if config.llm.tournament_enabled and config.llm.tournament_candidates >= 2: + # --- Best-of-N tournament: generate N candidate plans from diverse + # stances, then an independent judge picks the winner. Everything + # downstream (YAML parsing, normalization, caps) runs on the winner + # alone, so the stage still produces one canonical plan. --- + from types import SimpleNamespace + + from researchclaw.llm import build_panel_llms, build_reviewer_llm + from researchclaw.pipeline.tournament import ( + effective_candidates, + run_tournament, + ) + + _gens = build_panel_llms(config) or [llm] + _judge = build_reviewer_llm(config) or llm + _n = effective_candidates(config.llm.tournament_candidates) + _cps = [ + ( + sp.system, + sp.user + + "\n\n## Exploration stance\n" + + _DESIGN_ANGLES[i % len(_DESIGN_ANGLES)], + ) + for i in range(_n) + ] + _winner, _ = run_tournament( + _gens, + _judge, + _cps, + rank_prompt="tournament_rank", + out_dir=stage_dir / "tournament", + prompts=_pm, + author_model=getattr(llm.config, "primary_model", ""), + label="plan", + ) + resp = SimpleNamespace(content=_winner) + else: + resp = _chat_with_prompt( + llm, + sp.system, + sp.user, + json_mode=sp.json_mode, + max_tokens=sp.max_tokens, + ) raw_yaml = _extract_yaml_block(resp.content) try: parsed = yaml.safe_load(raw_yaml) diff --git a/researchclaw/pipeline/stage_impls/_review_publish.py b/researchclaw/pipeline/stage_impls/_review_publish.py index 9091ab87b..7f7c096a7 100644 --- a/researchclaw/pipeline/stage_impls/_review_publish.py +++ b/researchclaw/pipeline/stage_impls/_review_publish.py @@ -14,8 +14,10 @@ from researchclaw.adapters import AdapterBundle from researchclaw.config import RCConfig +from researchclaw.llm import build_panel_llms from researchclaw.llm.client import LLMClient from researchclaw.pipeline._domain import _detect_domain # noqa: F401 +from researchclaw.pipeline.debate import run_debate from researchclaw.pipeline._helpers import ( StageResult, _build_context_preamble, @@ -161,6 +163,47 @@ def _build_reviewer_or_generator(config, generator_llm): # Stage 18: Peer Review # --------------------------------------------------------------------------- +# Reviewer roles for the multi-model peer-review debate. Each is bound to a +# different panel model so the three reviews are not one model's opinion +# restated three times. +_REVIEW_DEBATE_ROLES: dict[str, dict[str, str]] = { + "methodology_reviewer": { + "system": ( + "You are a peer reviewer focused on METHODOLOGY and experimental " + "design rigor." + ), + "user": ( + "Review this paper draft for methodological soundness, baselines, " + "ablations, and validity threats.\n\nTopic: {topic}\n\n" + "Evidence:\n{experiment_evidence}\n\nDraft:\n{draft}" + ), + }, + "domain_reviewer": { + "system": ( + "You are a peer reviewer who is a DOMAIN EXPERT judging novelty " + "and significance." + ), + "user": ( + "Review this paper draft for novelty, related-work positioning, " + "and contribution significance.\n\nTopic: {topic}\n\n" + "Draft:\n{draft}" + ), + }, + "rigor_reviewer": { + "system": ( + "You are a peer reviewer focused on STATISTICS, reproducibility, " + "and claim-evidence consistency." + ), + "user": ( + "Review this paper draft for statistical rigor, reproducibility, " + "and whether every claim is supported by the evidence. Flag " + "unsupported numbers.\n\nEvidence:\n{experiment_evidence}\n\n" + "Draft:\n{draft}" + ), + }, +} + + def _execute_peer_review( stage_dir: Path, run_dir: Path, @@ -196,7 +239,31 @@ def _execute_peer_review( except Exception: # noqa: BLE001 pass - if _review_llm is not None: + _panel = build_panel_llms(config) + if _panel and _review_llm is not None: + # --- Multi-model peer-review debate: distinct models each play an + # independent reviewer role (methodology / domain / rigor) and rebut + # each other, then the independent reviewer synthesizes the report. + # No `synthesizer` is passed here on purpose: handing the write-up back + # to the author model would defeat the point of the stage. --- + _pm = prompts or PromptManager() + _variables = { + "topic": config.research.topic, + "draft": draft + _quality_suffix, + "experiment_evidence": experiment_evidence, + } + reviews, _ = run_debate( + _panel, + _review_llm, + _REVIEW_DEBATE_ROLES, + _variables, + rounds=config.llm.debate_rounds, + synth_prompt="review_synthesize", + out_dir=stage_dir / "debate", + prompts=_pm, + author_model=_author_model, + ) + elif _review_llm is not None: _pm = prompts or PromptManager() _overlay = _get_evolution_overlay(run_dir, "peer_review") sp = _pm.for_stage( diff --git a/researchclaw/prompts/shared.py b/researchclaw/prompts/shared.py index d5406951e..3008d562b 100644 --- a/researchclaw/prompts/shared.py +++ b/researchclaw/prompts/shared.py @@ -790,6 +790,25 @@ ), "max_tokens": 8192, }, + "review_synthesize": { + "system": ( + "You are the area chair synthesizing several independent peer " + "reviews into one decision-oriented review report. Do not flatten " + "disagreement — surface the most serious concerns prominently and " + "do not soften them." + ), + "user": ( + "Below are independent reviews of the paper from different " + "reviewers.\n" + "Synthesize them into a final review report with these sections:\n" + "## Summary\n## Strengths\n## Weaknesses (most serious first)\n" + "## Actionable Revisions (numbered, specific)\n" + "## Recommendation (ACCEPT / MINOR REVISION / MAJOR REVISION / " + "REJECT)\n\n" + "{perspectives}" + ), + "max_tokens": 6144, + }, "tournament_rank": { "system": ( "You score and rank competing research artifacts, distinct from the "