feat(research): 🔬 inject scientific rigor (deterministic temp 0.0, seed 42, epistemic grounding, LLM-as-a-Judge)

Author: Arena AI Agent

epistemic_forge/benchmark/llm_judge.py

@@ -0,0 +1,30 @@
+"""LLM-as-a-Judge for Automated, Scientific Epistemic Evaluation (G-Eval style)."""
+from pydantic import BaseModel, Field
+from typing import Dict, Any
+from epistemic_forge.llm import generate_structured
+
+class JudgeEvaluation(BaseModel):
+    """Strict schema for the AI Judge."""
+    logical_coherence_score: int = Field(ge=1, le=5, description="1-5 score on how well the premises support the conclusion.")
+    hallucination_detected: bool = Field(description="True if the text makes empirical claims without warrants.")
+    critique: str = Field(description="Academic peer-review style critique of the artifact.")
+
+def evaluate_artifact_quality(question: str, artifact_text: str) -> Dict[str, Any]:
+    """Uses a stronger model (e.g., GPT-4o) to judge the output of the cheaper pipeline."""
+    messages = [
+        {"role": "system", "content": "You are a highly critical, NeurIPS-level peer reviewer. Evaluate the following research artifact for logical coherence and hallucination."},
+        {"role": "user", "content": f"Research Question: {question}\n\nArtifact Output:\n{artifact_text}"}
+    ]
+    
+    # We use a heavier model for judging, but keep temp 0.0 for deterministic grading
+    evaluation = generate_structured(
+        messages=messages,
+        response_model=JudgeEvaluation,
+        model="gpt-4o-2024-08-06"
+    )
+    
+    return {
+        "score": evaluation.logical_coherence_score,
+        "hallucination": evaluation.hallucination_detected,
+        "critique": evaluation.critique
+    }

epistemic_forge/llm.py

@@ -1,4 +1,6 @@
-"""SOTA LLM Engine using Instructor and Pydantic for Strict Structured Outputs."""
+"""SOTA LLM Engine using Instructor and Pydantic for Strict Structured Outputs.
+ENFORCES: Reproducibility (Seed 42, Temp 0.0) for scientific benchmarks.
+"""
 import instructor
 from openai import OpenAI
 from pydantic import BaseModel
@@ -15,23 +17,25 @@
 def generate_structured(
     messages: list, 
     response_model: type[BaseModel], 
-    model: str = "gpt-4o-2024-08-06", 
-    temperature: float = 0.0
+    model: str = "gpt-4o-mini", 
+    temperature: float = 0.0,
+    seed: int = 42
 ) -> BaseModel:
     """
-    State-of-the-Art Structured Extraction.
-    Guarantees the output strictly matches the Pydantic schema using JSON Mode / Tool Calls.
+    Research-Grade Extraction.
+    Enforces Temperature=0.0 and Seed=42 to guarantee deterministic, reproducible scientific output.
     """
     if not client:
         raise ValueError("LLM Client is not initialized. Please set OPENAI_API_KEY.")
 
     try:
-        logger.debug(f"Initiating strict structured call to {model} for schema [{response_model.__name__}]...")
+        logger.debug(f"Initiating scientifically rigorous call to {model} [temp={temperature}, seed={seed}] for schema [{response_model.__name__}]...")
         response = client.chat.completions.create(
             model=model,
             messages=messages,
             response_model=response_model,
             temperature=temperature,
+            seed=seed
         )
         return response
     except Exception as e:

epistemic_forge/models.py

@@ -162,6 +162,8 @@ class HegelianDialecticOutput(BaseModel):
     steelmanned_antithesis: str = Field(description="The absolute strongest possible argument against the core thesis.")
     synthesis_resolution: str = Field(description="The nuanced truth that reconciles the thesis and the antithesis.")
     remaining_uncertainties: List[str] = Field(description="Questions that still lack sufficient evidence.")
+    epistemic_confidence: float = Field(ge=0.0, le=1.0, description="Confidence in the synthesis based on available evidence.")
+    source_warrant: str = Field(description="The exact logical warrant or grounded theory that justifies this synthesis.")
 
 class ChainOfDensityOutput(BaseModel):
     """Strict schema for the Chain of Density Architect."""