Evaluator.py

# Copyright (c) 2019-2020, INESC TEC (https://www.inesctec.pt)

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.


import json
import pandas as pd
from llm_utils import call_llm


# -----------------------------
# CONFIGURATION
# -----------------------------
DESIGN = 'base'#1

EVALUATOR_PROMPT_PATH = "Evaluator_prompt.txt"
EXPLANATIONS_PATH = f"test_actor_explanations_{DESIGN}.json"
SNAPSHOTS_PATH = "test_snapshots_case39_cf2.json"
OUTPUT_CSV = f"evaluation_scores_design_{DESIGN}.csv"


# -----------------------------
# LOAD PROMPT
# -----------------------------
def load_prompt(path):
    with open(path, "r", encoding="utf-8") as f:
        return f.read()


# -----------------------------
# RUN EVALUATOR LLM
# -----------------------------
def run_evaluator(evaluator_prompt, snapshot, explanation):

    prompt = f"""
{evaluator_prompt}

POWER SYSTEM SNAPSHOT:
{json.dumps(snapshot, indent=2)}

LLM EXPLANATION:
{explanation}

Return ONLY valid JSON according to the specified output format.
"""

    response = call_llm(prompt, role="Evaluator", temperature=0.1)

    try:
        scores = json.loads(response)
    except Exception:
        print("⚠ Failed to parse evaluator response")
        print(response)

        scores = {
            "section_scores": {},
            "question_scores": {},
            "total_score": None,
            "percentage": None,
            "evaluation_summary": None
        }

    return scores


# -----------------------------
# MAIN
# -----------------------------
def main():

    print("Loading evaluator prompt...")
    evaluator_prompt = load_prompt(EVALUATOR_PROMPT_PATH)

    print("Loading explanations...")
    with open(EXPLANATIONS_PATH, "r", encoding="utf-8") as f:
        explanations = json.load(f)

    print("Loading snapshots...")
    with open(SNAPSHOTS_PATH, "r", encoding="utf-8") as f:
        snapshots = json.load(f)

    results = []

    for snapshot_id, explanation in explanations.items():

        print(f"\nEvaluating snapshot: {snapshot_id}")

        snapshot = snapshots[snapshot_id]["opf_summary"]

        scores = run_evaluator(
            evaluator_prompt,
            snapshot,
            explanation
        )

        section = scores.get("section_scores", {})
        questions = scores.get("question_scores", {})

        row = {
            "snapshot_id": snapshot_id,

            # Section scores
            "redispatch_trigger_identification": section.get("redispatch_trigger_identification"),
            "operational_logic": section.get("operational_logic"),
            "ptdf_directional_reasoning": section.get("ptdf_directional_reasoning"),
            "counterfactual_reasoning": section.get("counterfactual_reasoning"),
            "numerical_grounding_and_report_discipline": section.get("numerical_grounding_and_report_discipline"),
            "economic_and_dual_interpretation": section.get("economic_and_dual_interpretation"),
            "security_screening": section.get("security_screening"),
            "language_and_operator_usefulness": section.get("language_and_operator_usefulness"),

            # Global metrics
            "total_score": scores.get("total_score"),
            "percentage": scores.get("percentage"),
            "evaluation_summary": scores.get("evaluation_summary"),
        }

        # Store question scores (Q1–Q20)
        for q in range(1, 21):
            row[f"Q{q}"] = questions.get(f"Q{q}")

        results.append(row)

    df = pd.DataFrame(results)

    df.to_csv(OUTPUT_CSV, index=False)

    print("\n✅ Evaluation completed")
    print(f"Scores saved to: {OUTPUT_CSV}")



# -----------------------------
# RUN SCRIPT
# -----------------------------
if __name__ == "__main__":
    main()