run_gepa_only_workflow.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<'USAGE'
Usage: ./run_gepa_only_workflow.sh <model>

Model must be one of: gpt-4o, gpt-5.1, gpt-5.1-reasoning, codex

Runs only:
  1) GEPA training (prompt search)
  2) GEPA inference run using best_prompt.txt
USAGE
}

MODEL="${1:-}"
if [[ -z "$MODEL" ]]; then
  usage
  exit 1
fi

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

case "$MODEL" in
  gpt-4o)
    ENDPOINT="gpt-4o"
    CONFIG_FILE="gpt_4o.json"
    ;;
  gpt-5.1)
    ENDPOINT="gpt-5.1"
    CONFIG_FILE="gpt_5_1.json"
    ;;
  gpt-5.1-reasoning)
    ENDPOINT="gpt-5.1"
    CONFIG_FILE="gpt_5_1_no_reasoning.json"
    ;;
  codex)
    ENDPOINT="codex"
    CONFIG_FILE="codex.json"
    ;;
  *)
    usage
    exit 1
    ;;
esac

CONFIG_PATH="${REPO_ROOT}/src/endpoints/configs/${CONFIG_FILE}"
if [[ ! -f "$CONFIG_PATH" ]]; then
  echo "Config not found: $CONFIG_PATH" >&2
  exit 1
fi

RUN_ID="$(date +%Y%m%d_%H%M%S)_$$"
RESULTS_ROOT="${REPO_ROOT}/run_results/${MODEL}/${RUN_ID}"
RESULTS_JSON="${RESULTS_ROOT}/results.json"
LOG_DIR="${RESULTS_ROOT}/logs"

GEPA_DIR="${REPO_ROOT}/gepa_outputs/GEPA_${MODEL}/${RUN_ID}"

mkdir -p "$LOG_DIR" "$GEPA_DIR"
printf "[]\n" > "$RESULTS_JSON"

activate_conda() {
  if ! command -v conda >/dev/null 2>&1; then
    echo "ERROR: conda not found; cannot activate 'tractor' environment." >&2
    exit 1
  fi
  local conda_base=""
  conda_base="$(conda info --base 2>/dev/null || true)"
  if [[ -z "$conda_base" || ! -f "${conda_base}/etc/profile.d/conda.sh" ]]; then
    echo "ERROR: could not locate conda.sh; cannot activate 'tractor'." >&2
    exit 1
  fi
  # shellcheck source=/dev/null
  source "${conda_base}/etc/profile.d/conda.sh"
  conda activate tractor
}

append_result() {
  local strategy="$1"
  local log_file="$2"
  local out_file="$3"
  python - "$MODEL" "$strategy" "$log_file" "$out_file" <<'PY'
import json
import re
import sys
from pathlib import Path

model = sys.argv[1]
strategy = sys.argv[2]
log_path = Path(sys.argv[3])
out_file = Path(sys.argv[4])

text = log_path.read_text(errors="ignore")
match = re.search(r"Overall stats:\s*(\{[\s\S]*?\})", text)
if not match:
    raise SystemExit(f"ERROR: could not find 'Overall stats' in {log_path}")

stats = json.loads(match.group(1))
record = {
    "model": model,
    "strategy": strategy,
    "stats": {
        "All tests pass": stats.get("All tests pass", 0),
        "Compile": stats.get("Compile", 0),
    },
}
out_file.write_text(json.dumps(record, indent=2))
print(json.dumps(record, indent=2))
PY
}

run_gepa_build() {
  echo "==> Building GEPA prompt (${MODEL})"
  GEPA_LOG="${LOG_DIR}/gepa_build.log"
  activate_conda
  (
    cd "${REPO_ROOT}/src"
    python gepa.py \
      --benchmark_dir ../datasets/CBench \
      --rust_dir ../datasets/RBench \
      --output_dir "${GEPA_DIR}" \
      --endpoint "${ENDPOINT}" \
      --config "${CONFIG_PATH}" \
      --training_size 5 \
      --population_size 10 \
      --generations 5 \
      --batch_size 5 \
      --use_llm_feedback True
  ) 2>&1 | tee "$GEPA_LOG"

  if [[ ! -f "${GEPA_DIR}/best_prompt.txt" ]]; then
    echo "ERROR: best_prompt.txt NOT FOUND at ${GEPA_DIR}/best_prompt.txt" >&2
    exit 1
  fi
}

run_gepa() {
  GEPA_PROMPT="${GEPA_DIR}/best_prompt.txt"
  echo "==> Running model with GEPA prompt (${MODEL})"
  GEPA_RUN_LOG="${LOG_DIR}/run_gepa.log"
  activate_conda
  (
    cd "${REPO_ROOT}/src"
    python run.py \
      --mode "normal" \
      --benchmark_dir "../datasets/CBench" \
      --output_dir "../results/gepa_${MODEL}/${RUN_ID}/bullet_point_with_system_instructions" \
      --use_gepa_prompt \
      --gepa_prompt_path "../gepa_outputs/GEPA_${MODEL}/${RUN_ID}/best_prompt.txt" \
      --prompt_format bullet_point_with_system_instructions \
      --prompt_strategy all \
      --repairer_prompt ./prompts/repair_prompts/bullet_point/bullet_point.prompt \
      --repairer_format bullet_point_with_system_instructions \
      --repairer_strategy all \
      --iterations 3 \
      --config "${CONFIG_PATH}" \
      --endpoint "${ENDPOINT}" \
      --rust_dir "../datasets/RBench"
  ) 2>&1 | tee "$GEPA_RUN_LOG"
  append_result "gepa" "$GEPA_RUN_LOG" "${RESULTS_ROOT}/result_gepa.json"
}

echo "[$(date '+%Y-%m-%d %H:%M:%S')] Running GEPA-only workflow (SEQUENTIAL)..."
run_gepa_build
run_gepa

echo "==> Final results (saved to ${RESULTS_JSON})"
python - "$RESULTS_ROOT" "$RESULTS_JSON" <<'PY'
import json
import sys
from pathlib import Path

root = Path(sys.argv[1])
out_path = Path(sys.argv[2])
records = []
for path in sorted(root.glob("result_*.json")):
    try:
        records.append(json.loads(path.read_text()))
    except json.JSONDecodeError:
        pass
out_path.write_text(json.dumps(records, indent=2))
print(json.dumps(records, indent=2))
PY