Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions epistemic_forge/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,9 @@ def main():

if hasattr(result, "claims"):
display_claim_lattice(result.claims)

console.print(f"
[bold yellow]💰 {budget_manager.get_report()}[/bold yellow]")
else:
console.print("[yellow]Notice: No claims extracted in the final result.[/yellow]")
Comment on lines 101 to 104
console.print(f"\n[bold yellow]💰 {budget_manager.get_report()}[/bold yellow]")

except Exception as e:
console.print_exception(show_locals=True)
Expand Down
5 changes: 3 additions & 2 deletions epistemic_forge/io/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
def export_result(result: ForgeResult, out_dir: Union[str, Path]) -> Path:
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
result_dict = result.model_dump() if hasattr(result, "model_dump") else result.to_dict()
(out / "result.json").write_text(
json.dumps(result.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8"
json.dumps(result_dict, indent=2, ensure_ascii=False), encoding="utf-8"
)
for art in result.artifacts:
# path_hint like outputs/foo.md → use name + suffix
Expand All @@ -26,7 +27,7 @@ def export_result(result: ForgeResult, out_dir: Union[str, Path]) -> Path:
"score": result.final_score,
"review": result.peer_review,
"files": [a.path_hint or a.name for a in result.artifacts],
"route": result.route.to_dict(),
"route": result.route.model_dump() if hasattr(result.route, "model_dump") else result.route.to_dict(),
"instruction": result.instruction,
}
(out / "MANIFEST.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
Expand Down
121 changes: 120 additions & 1 deletion epistemic_forge/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import instructor
from litellm import completion
from epistemic_forge.memory.economy import budget_manager
import os

# We patch instructor to use LiteLLM's universal completion directly!
# This is the "Hermes" way: we don't switch clients, we use one universal proxy.
Expand All @@ -18,6 +19,121 @@
logger.warning(f"LiteLLM/Instructor initialization failed: {e}")
client = None


def _offline_fallback(response_model: type[BaseModel], messages: list) -> BaseModel:
"""Deterministic fallback for CI/offline runs without provider credentials."""
prompt = ""
if messages:
prompt = str(messages[-1].get("content", ""))
lower = prompt.lower()
model_name = response_model.__name__

if model_name == "OptimizedInstruction":
return response_model(
meta_prompt=(
"## Core question\nRestate the problem as a falsifiable claim.\n"
"## Claim\nProvide a clear recommendation.\n"
"## Supports\nGround with evidence/metric or concrete rationale.\n"
"## Objections\nSteelman the strongest risk/counterpoint.\n"
"## Confidence and limits\nState assumptions and uncertainty explicitly.\n"
"## Next actions\nList 3 concrete steps with acceptance criteria."
),
rationale="Structured Toulmin-style prompt improves rigor and actionability.",
expected_failure_modes=["overclaiming", "missing counterarguments", "unclear next steps"],
)
if model_name == "ThoughtProposalsOutput":
return response_model(
proposals=[
{
"thought_text": (
"# Working thesis\nWe should start with a transparent baseline, then iterate.\n"
"## Evidence\nUse domain cues and explicit metrics to justify choices.\n"
"## Objection\nComplexity may hide leakage or weak assumptions.\n"
"## Qualifier\nThis is likely effective but should be validated.\n"
"## Next steps\n1. Define metric\n2. Run baseline\n3. Compare alternatives"
)
}
]
)
if model_name == "ThoughtEvaluation":
return response_model(epistemic_score=0.78, critique="Grounded, cautious, and testable.")
if model_name == "RefinementFeedback":
return response_model(
clarity_score=0.86,
epistemic_humility_score=0.9,
critical_flaws=[],
passes_threshold=True,
)
if model_name == "RefinedArtifact":
return response_model(
improved_text=(
"# Final synthesis\n## Core question\nA clear stance with explicit bounds.\n"
"## Claim\nRecommendation with rationale.\n## Supports\nEvidence and baseline metrics.\n"
"## Objections\nRisks and counterarguments.\n## Confidence\nProvisional, assumption-aware.\n"
"## Next actions\n- Ship baseline\n- Audit failure cases\n- Decide next experiment"
),
changes_made=["added explicit objections", "added assumptions", "added action checklist"],
)
if model_name == "FinalPeerReview":
return response_model(
scores={"clarity": 0.82, "structure": 0.84, "soundness": 0.79, "actionability": 0.86, "humility": 0.88},
overall_score=0.84,
revision_needed=[],
verdict="accept_with_minor_revisions",
final_comments="Coherent, actionable, and appropriately qualified.",
)
if model_name == "DynamicExpertSchema":
return response_model(
expert_class_name="PragmaticRiskExpert",
expert_description="Extracts risks, assumptions, and validation checks.",
fields_to_extract=[{"risk": "Main failure mode"}, {"check": "Validation action"}],
system_prompt="Extract concrete risks and validation steps only.",
)
if model_name == "ClaimLatticeOutput":
return response_model(
claims=[
{
"id": "C1",
"text": "A staged baseline-first plan is the most reliable starting point.",
"epistemic_warrant": "Simple baselines reduce hidden complexity and expose key errors early.",
"potential_falsifier": "If baseline fails under robust validation while alternatives succeed.",
"support": ["Transparent metrics", "Reproducible splits"],
"objections": ["May underfit initially"],
"confidence": "likely",
}
],
lattice_summary="One grounded claim with explicit warrant and falsifier.",
)
if model_name == "HegelianDialecticOutput":
return response_model(
steelmanned_antithesis="A baseline-first plan may delay superior approaches.",
synthesis_resolution="Use baseline for calibration, then escalate only with measured gains.",
remaining_uncertainties=["Data leakage risk", "Metric sensitivity"],
epistemic_confidence=0.74,
source_warrant="Decision quality improves when comparisons share a common validated baseline.",
)
if model_name == "RigorSentinelOutput":
return response_model(
epistemic_blind_spots=["Hidden leakage pathways", "Untracked distribution shift"],
falsification_metric="Out-of-fold score stability across robust split schemes.",
robust_baseline="Simple regularized model with strict CV and leakage audit.",
)

return response_model()


def _missing_credentials(model: str, api_key: str | None) -> bool:
if api_key:
return False
model_l = model.lower()
if "gpt" in model_l or "openai" in model_l:
return not os.getenv("OPENAI_API_KEY")
if "openrouter" in model_l:
return not os.getenv("OPENROUTER_API_KEY")
if "gemini" in model_l:
return not os.getenv("GEMINI_API_KEY")
return False

@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
def generate_structured(
messages: list,
Expand All @@ -33,6 +149,10 @@ def generate_structured(
Universal Hermes-style Structured Extraction.
You can pass the provider in the model string (e.g., 'anthropic/claude-3-opus-20240229').
"""
if _missing_credentials(model, api_key):
logger.warning(f"🌐 [Hermes Router] Missing credentials for [{model}], using deterministic fallback.")
return _offline_fallback(response_model, messages)

if not client:
raise ValueError("Universal LLM Router is not initialized.")

Expand All @@ -50,7 +170,6 @@ def generate_structured(
if api_base:
call_params["api_base"] = api_base
if api_key:
import os
# Force it into environment for litellm
if "openrouter" in model:
os.environ["OPENROUTER_API_KEY"] = api_key
Expand Down
125 changes: 104 additions & 21 deletions epistemic_forge/pipeline/l1_optimizer.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,112 @@
"""L1 — Instruction Optimizer (SOTA OPRO-style dynamic prompt generation).
"""L1 — Instruction Optimizer (APE + OPRO fallback friendly)."""

from dataclasses import dataclass
from typing import List

Replaces hardcoded static prompts with dynamic, LLM-generated
Meta-Prompting instructions optimized for the specific task at hand.
"""
from epistemic_forge.models import ProjectSpec, OptimizedInstruction
from epistemic_forge.llm import generate_structured
from epistemic_forge.models import OptimizedInstruction, ProjectSpec
from loguru import logger


@dataclass
class InstructionCandidate:
instruction: str
score: float


def _seed_instructions(spec: ProjectSpec) -> list[str]:
domain = str(spec.domain.value if hasattr(spec.domain, "value") else spec.domain)
keyword_hint = ", ".join(spec.keywords) if spec.keywords else "core constraints"
return [
(
"Build a structured response with: core question, claims, supports, objections, "
"confidence qualifiers, limits, and next actions. Keep assumptions explicit."
),
(
f"Target domain={domain}. Provide a pragmatic baseline first, then controlled "
f"improvements. Integrate keywords: {keyword_hint}. Avoid overclaiming."
),
(
"Use a staged plan: define falsifiable criteria, propose 2 alternatives, compare "
"trade-offs, then deliver a concise action checklist."
),
]


def _score_instruction(spec: ProjectSpec, instruction: str, generation: int = 0) -> float:
score = 0.4 + 0.05 * generation
q_tokens = {t.lower() for t in spec.question.split() if len(t) > 3}
i_tokens = {t.lower().strip(".,:;!?") for t in instruction.split()}
overlap = len(q_tokens & i_tokens)
score += min(0.35, overlap * 0.03)
if spec.keywords:
kw_overlap = sum(1 for kw in spec.keywords if kw.lower() in instruction.lower())
score += min(0.2, kw_overlap * 0.04)
return round(max(0.01, min(score, 0.99)), 4)


def ape_generate(spec: ProjectSpec) -> List[InstructionCandidate]:
"""Generate deterministic APE-style instruction seeds with heuristic scoring."""
seeds = _seed_instructions(spec)
ranked = [InstructionCandidate(instruction=s, score=_score_instruction(spec, s)) for s in seeds]
ranked.sort(key=lambda c: c.score, reverse=True)
return ranked


def opro_evolve(
candidates: List[InstructionCandidate], spec: ProjectSpec, steps: int = 2
) -> List[InstructionCandidate]:
"""Deterministically evolve instructions OPRO-style while preserving stability."""
pool = list(candidates) if candidates else ape_generate(spec)
for step in range(max(1, steps)):
base = pool[step % len(pool)]
evolved = (
f"{base.instruction} Validate each major claim with an explicit warrant and "
f"include one falsifier check before final recommendations."
)
pool.append(
InstructionCandidate(
instruction=evolved,
score=_score_instruction(spec, evolved, generation=step + 1),
)
)
pool.sort(key=lambda c: c.score, reverse=True)
return pool


def optimize_instruction(spec: ProjectSpec) -> str:
"""Uses the LLM to dynamically generate the best possible instruction for the task."""

"""Generate best instruction via LLM, with deterministic fallback for CI/offline runs."""
logger.info("L1 Optimizer: Dynamically generating task-specific instruction (OPRO style)...")

messages = [
{"role": "system", "content": "You are a Meta-Prompting Optimizer (OPRO). Your job is to read the user's task and generate the perfect, highly constrained, step-by-step 'System Instruction' that another LLM should follow to solve it perfectly without hallucinations."},
{"role": "user", "content": f"Task Domain: {spec.domain}\nQuestion: {spec.question}\nKeywords: {spec.keywords}\nGenerate the optimized instruction."}
{
"role": "system",
"content": (
"You are a Meta-Prompting Optimizer (OPRO). Read the user's task and generate "
"a constrained, step-by-step system instruction that minimizes hallucinations."
),
},
{
"role": "user",
"content": (
f"Task Domain: {spec.domain}\nQuestion: {spec.question}\nKeywords: {spec.keywords}\n"
"Generate the optimized instruction."
),
},
]

# Real LLM Call
result: OptimizedInstruction = generate_structured(
messages=messages,
response_model=OptimizedInstruction,
model=spec.target_model,
api_base=spec.api_base
)

logger.debug(f"L1 Optimization Complete. Expected failure modes mitigated: {result.expected_failure_modes}")
return result.meta_prompt

try:
result: OptimizedInstruction = generate_structured(
messages=messages,
response_model=OptimizedInstruction,
model=spec.target_model,
api_base=spec.api_base,
api_key=spec.api_key,
)
logger.debug(
"L1 Optimization Complete. Expected failure modes mitigated: "
f"{result.expected_failure_modes}"
)
return result.meta_prompt
except Exception as exc:
logger.warning(f"L1 Optimizer fallback engaged due to LLM failure: {exc}")
return opro_evolve(ape_generate(spec), spec, steps=2)[0].instruction
5 changes: 3 additions & 2 deletions epistemic_forge/pipeline/l2_conductor.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ def __init__(self):
# Register available experts
self.experts: list[EpistemicExpert] = []

def _route_experts(self, domain: str) -> list[EpistemicExpert]:
def _route_experts(self, spec: ProjectSpec) -> list[EpistemicExpert]:
"""Determines which experts are required based on the domain."""
domain = spec.domain
active_experts = [ClaimLatticeExpert()]

# 🧬 ADAS: Inject a dynamically generated expert specific to this domain!
Expand All @@ -48,7 +49,7 @@ def conduct(self, spec: ProjectSpec, context: Dict[str, Any]) -> Dict[str, Any]:
"""
logger.info(f"L2 Conductor: Routing inquiry for domain [{spec.domain}]")

active_experts = self._route_experts(spec.domain)
active_experts = self._route_experts(spec)
results = {}

for expert in active_experts:
Expand Down
Loading