feat(evaluation): ⚖️ eradicate blind logical scoring and implement strict Toulmin-Model LLM-as-a-Judge (Warrant strength & Rebuttal validity)

Author: Arena AI Agent

epistemic_forge/benchmark/llm_judge.py

@@ -1,46 +1,35 @@
-"""LLM-as-a-Judge for Automated, Scientific Epistemic Evaluation (G-Eval style)."""
-
-from pydantic import BaseModel, Field
+"""LLM-as-a-Judge for Automated, Scientific Epistemic Evaluation (Toulmin Model)."""
 from typing import Dict, Any
+from epistemic_forge.models import JudgeEvaluation
 from epistemic_forge.llm import generate_structured
-
-
-class JudgeEvaluation(BaseModel):
-    """Strict schema for the AI Judge."""
-
-    logical_coherence_score: int = Field(
-        ge=1,
-        le=5,
-        description="1-5 score on how well the premises support the conclusion.",
-    )
-    hallucination_detected: bool = Field(
-        description="True if the text makes empirical claims without warrants."
-    )
-    critique: str = Field(
-        description="Academic peer-review style critique of the artifact."
-    )
-
+from loguru import logger
 
 def evaluate_artifact_quality(question: str, artifact_text: str) -> Dict[str, Any]:
-    """Uses a stronger model (e.g., GPT-4o) to judge the output of the cheaper pipeline."""
+    """Uses a stronger model to judge the output based strictly on Toulmin's Model of Argumentation."""
+    logger.info("⚖️ Initiating strict Toulmin-based evaluation of the final artifact...")
+    
     messages = [
         {
-            "role": "system",
-            "content": "You are a highly critical, NeurIPS-level peer reviewer. Evaluate the following research artifact for logical coherence and hallucination.",
-        },
-        {
-            "role": "user",
-            "content": f"Research Question: {question}\n\nArtifact Output:\n{artifact_text}",
+            "role": "system", 
+            "content": (
+                "You are an Elite Academic Peer Reviewer specializing in the Toulmin Model of Argumentation. "
+                "Do NOT judge the artifact based on prose or formatting. You must ONLY evaluate the strength of the 'Warrants' (do they bridge the data to the claim?) "
+                "and the validity of the 'Rebuttals/Falsifiers' (are they real weaknesses or just strawmen?)."
+            )
         },
+        {"role": "user", "content": f"Core Inquiry: {question}\n\nSubmitted Artifact:\n{artifact_text}\n\nExecute the Toulmin Evaluation."}
     ]
-
-    # We use a heavier model for judging, but keep temp 0.0 for deterministic grading
-    evaluation = generate_structured(
-        messages=messages, response_model=JudgeEvaluation, model="gpt-4o-2024-08-06"
+    
+    # We use a robust model for judging, maintaining temp 0.0 for deterministic grading
+    evaluation: JudgeEvaluation = generate_structured(
+        messages=messages,
+        response_model=JudgeEvaluation,
+        model="openai/gpt-4o-mini", # Standardizing to openrouter/openai model format
+        api_base="https://openrouter.ai/api/v1" # Enforce OpenRouter for testing consistency
     )
-
+    
     return {
         "score": evaluation.logical_coherence_score,
         "hallucination": evaluation.hallucination_detected,
-        "critique": evaluation.critique,
+        "critique": evaluation.critique
     }