From d0bb9e07f4c44dc5cde953a511ea928dca6a5b9c Mon Sep 17 00:00:00 2001 From: Aswin Prakash Thiyagarajan Date: Mon, 31 Aug 2026 20:55:34 -0400 Subject: [PATCH 1/2] [OPIK-8186] opik-instrument: add eval harness (verify coverage + no false success) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the opik-diagnose/evals pattern. Two functional fixtures: - clean: uninstrumented app; the skill must instrument, run, and verify a complete 3-span trace with reported span coverage. - missing_flush: already instrumented but no flush, so no complete trace lands; the skill must not claim success without a real trace (no_false_success) — the decisive test of OPIK-8185's coverage check. Plus triggering (selection_accuracy), a deterministic grader, and metrics. Self-tested: correct outputs pass 2/2; a false already_verified is caught. Co-Authored-By: Claude Opus 4.8 --- .../skills/opik-instrument/evals/.gitignore | 3 + .../skills/opik-instrument/evals/HARNESS.md | 96 ++++++++ .../skills/opik-instrument/evals/cases.yaml | 47 ++++ .../evals/fixtures/clean/app.py | 48 ++++ .../evals/fixtures/clean/expected.json | 8 + .../evals/fixtures/clean/pyproject.toml | 6 + .../evals/fixtures/missing_flush/app.py | 54 +++++ .../fixtures/missing_flush/expected.json | 8 + .../fixtures/missing_flush/pyproject.toml | 6 + .../skills/opik-instrument/evals/grader.py | 171 +++++++++++++++ .../skills/opik-instrument/evals/metrics.py | 56 +++++ .../skills/opik-instrument/evals/run_evals.py | 205 ++++++++++++++++++ 12 files changed, 708 insertions(+) create mode 100644 src/opik_mcp/skills/opik-instrument/evals/.gitignore create mode 100644 src/opik_mcp/skills/opik-instrument/evals/HARNESS.md create mode 100644 src/opik_mcp/skills/opik-instrument/evals/cases.yaml create mode 100644 src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/app.py create mode 100644 src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/expected.json create mode 100644 src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/pyproject.toml create mode 100644 src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/app.py create mode 100644 src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/expected.json create mode 100644 src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/pyproject.toml create mode 100644 src/opik_mcp/skills/opik-instrument/evals/grader.py create mode 100644 src/opik_mcp/skills/opik-instrument/evals/metrics.py create mode 100644 src/opik_mcp/skills/opik-instrument/evals/run_evals.py diff --git a/src/opik_mcp/skills/opik-instrument/evals/.gitignore b/src/opik_mcp/skills/opik-instrument/evals/.gitignore new file mode 100644 index 0000000..55b6b49 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/.gitignore @@ -0,0 +1,3 @@ +_work/ +__pycache__/ +*.pyc diff --git a/src/opik_mcp/skills/opik-instrument/evals/HARNESS.md b/src/opik_mcp/skills/opik-instrument/evals/HARNESS.md new file mode 100644 index 0000000..6825b9a --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/HARNESS.md @@ -0,0 +1,96 @@ +# `/opik-instrument` evals + +Test cases, automation, and success metrics for the `opik-instrument` skill — +mirrors the `opik-diagnose/evals` layout. Where diagnose seeds traces and grades +a shortlist, instrument stages an **app**, has the skill instrument + run + verify +it, and grades whether a **real, complete trace** was confirmed — not just that +code was edited or that "a trace arrived". + +## Layout + +``` +evals/ + cases.yaml # triggering + functional (clean, missing_flush) + fixtures/ + clean/ # uninstrumented app; correct instrumentation -> 3-span trace + missing_flush/ # already instrumented but no flush -> no complete trace (adversarial) + grader.py # deterministic: result.json vs expected.json (+ optional online integrity) + metrics.py # aggregate -> metrics + run_evals.py # orchestrator: prepare stages fixtures, grade scores result.json + _work/ # staged run dirs + reports (gitignored) +``` + +## Run it + +Deps: `pyyaml` (via `uv run --with pyyaml`). Running the skill on a fixture needs +an **LLM provider key** (e.g. `OPENAI_API_KEY`) and **Opik configured** +(`~/.opik.config` or `OPIK_API_KEY`), because the skill runs the app and confirms +a real trace. Grading is offline; the optional integrity re-read uses Opik only +when a `trace_id` is present and Opik is reachable. + +```bash +uv run --with pyyaml python run_evals.py prepare # stage fixture apps under _work/ +# ... run /opik-instrument in each _work/ dir (see PROMPT.txt). The skill +# writes result.json in that dir (contract below). +uv run --with pyyaml python run_evals.py grade # score result.json vs expected.json +``` + +**Triggering (`selection_accuracy`):** +```bash +uv run --with pyyaml python run_evals.py trigger-prepare +# ... a judge panel classifies each phrase (descriptions only) into verdicts.json ... +uv run --with pyyaml python run_evals.py trigger-grade +``` +The menu presents the real `opik-instrument` description alongside decoys +(`opik-diagnose`, `opik-explain`, `opik-evaluate`, `opik`, `code-review`), so +"add tracing" must select instrument and not the neighbours. + +## The `result.json` contract + +The skill writes this into the workdir after running: + +```json +{ + "status": "verified | blocked | already_verified | unsupported", + "trace_id": "0f1e...", "trace_url": "https://.../traces/...", + "changes": ["added opik to pyproject", "wrapped OpenAI client with track_openai", "..."], + "next_step": "add opik.flush_tracker() before exit, then re-run", + "coverage": { + "expected_sites": 3, + "spans_found": 3, + "spans": [ + {"name": "run", "type": "general"}, + {"name": "generate", "type": "llm"}, + {"name": "retrieve", "type": "tool"} + ] + } +} +``` + +`coverage` is what makes this an eval of the *verify-coverage* ability (OPIK-8185): +the skill must report which spans it actually confirmed, not just a boolean. + +## What each case proves + +- **clean** — correct instrumentation must land AND `verified` a complete trace, + reporting all three span types, every span well-formed, with code changed. +- **missing_flush** (adversarial) — already instrumented but no flush, so no + complete trace lands. The decisive check is **`no_false_success`**: the skill + must not claim `verified`/`already_verified` unless a real, complete trace backs + it. It passes whether it fixes the flush and verifies, or returns `blocked` with + a flush next-step. A skill that trusts the decorators and reports success **fails**. + +## Metrics + +`selection_accuracy`, `verify_correctness`, `no_false_success_rate` (target 1.0), +`coverage_reported_rate`, `type_coverage`, `well_formed_rate`, `instrumented_rate`, +`integrity_rate`, `schema_compliance`. See `metrics.py`. + +## TODO — a `partial_trace` fixture + +A stronger adversarial case is a trace that **arrives but is incomplete** at the +span level (the batching race: fast spans dropped or returned unnamed), where the +correct outcome is `blocked` with `spans_found < expected_sites`. Inducing that +deterministically needs a fixture that reliably drops a span at runtime (rather +than the always-empty missing-flush case). Tracked for a follow-up; the grader +already supports it via `coverage.spans_found` vs `expected_sites`. diff --git a/src/opik_mcp/skills/opik-instrument/evals/cases.yaml b/src/opik_mcp/skills/opik-instrument/evals/cases.yaml new file mode 100644 index 0000000..ace5b47 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/cases.yaml @@ -0,0 +1,47 @@ +# Test cases for the `/opik-instrument` skill. +# +# triggering -> does the skill load at the right times (and not the wrong ones)? +# functional -> given a fixture app, does it instrument + run + VERIFY correctly? +# +# Each functional fixture ships an `expected.json` (ground truth: expected span +# count/types + planted defect). The agent runs `/opik-instrument` on the copied +# workdir and writes `result.json`; the grader scores result.json against the +# case `assert` block and expected.json. No agent, no network at grade time +# (an optional online integrity re-read runs only if Opik is configured). + +triggering: + should_trigger: + - "add opik tracing to my app" + - "instrument my code with opik" + - "add observability to my agent" + - "trace my agent with opik" + - "wire up opik and show me a trace" + should_not_trigger: + - "which traces are worth attention" # opik-diagnose + - "explain why this trace failed" # opik-explain + - "build an evaluation for my agent" # opik-evaluate + - "what's the weather in San Francisco" # none + +functional: + # Positive path: correct instrumentation must land AND verify a complete trace, + # reporting span-level coverage (all expected types present, every span named). + - id: clean + fixture: clean + prompt: "Instrument the app in this directory with Opik and verify a real trace lands. Write your result as result.json in this directory." + assert: + status: verified + coverage_reported: true + cover_types: ["general", "llm", "tool"] + spans_well_formed: true + modified_code: true + + # Adversarial: already instrumented but exits without flush, so no complete + # trace lands. The decisive check is `no_false_success` — the skill must not + # claim verified/already_verified unless a real, complete trace backs it. + # Passes whether it fixes the flush and verifies, or blocks with a flush step. + - id: missing_flush + fixture: missing_flush + prompt: "This app is already instrumented with Opik. Audit it, run a safe path, and verify a real trace lands. Write your result as result.json in this directory." + assert: + no_false_success: true + spans_well_formed: true diff --git a/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/app.py b/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/app.py new file mode 100644 index 0000000..ad51007 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/app.py @@ -0,0 +1,48 @@ +"""A tiny, deliberately UNINSTRUMENTED agent. + +Golden path: run() -> retrieve() (a tool) -> generate() (an LLM call). +Correctly instrumented, one representative run should produce a 3-span trace: + general (run) -> tool (retrieve) + llm (generate) + +The skill under test must add Opik tracing, run this safely once, and verify +that a real, complete trace landed. Running needs an OpenAI-compatible key. +""" + +from __future__ import annotations + +import os + +from openai import OpenAI + +client = OpenAI() + +_CORPUS = {"opik": "Opik is an LLM observability tool for tracing and evaluating LLM apps."} + + +def retrieve(query: str) -> str: + """Tool: look up context for the query (deterministic, no network).""" + return _CORPUS.get(query.lower().split()[0], "no context found") + + +def generate(question: str, context: str) -> str: + """LLM call: answer the question using the retrieved context.""" + resp = client.chat.completions.create( + model=os.getenv("OPENAI_MODEL", "gpt-4o-mini"), + messages=[ + {"role": "system", "content": "Answer the question using only the context."}, + {"role": "user", "content": f"Context: {context}\nQuestion: {question}"}, + ], + temperature=0, + max_tokens=50, + ) + return resp.choices[0].message.content.strip() + + +def run(question: str) -> str: + """Entrypoint: retrieve context, then generate an answer.""" + context = retrieve(question) + return generate(question, context) + + +if __name__ == "__main__": + print(run("What is opik?")) diff --git a/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/expected.json b/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/expected.json new file mode 100644 index 0000000..8c79238 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/expected.json @@ -0,0 +1,8 @@ +{ + "case": "clean", + "framework": "openai", + "expected_sites": 3, + "expected_types": ["general", "llm", "tool"], + "defect": "none", + "note": "Correct instrumentation of the golden path yields a general root (run) with an llm (generate) and a tool (retrieve) child." +} diff --git a/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/pyproject.toml b/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/pyproject.toml new file mode 100644 index 0000000..dc7a6ca --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/fixtures/clean/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "instrument-eval-clean" +version = "0.0.0" +description = "Uninstrumented fixture app for the opik-instrument eval (clean case)." +requires-python = ">=3.10" +dependencies = ["openai>=1.0"] diff --git a/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/app.py b/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/app.py new file mode 100644 index 0000000..befc55b --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/app.py @@ -0,0 +1,54 @@ +"""Already instrumented with Opik — but with a PLANTED DEFECT. + +The tracing decorators are correct, yet the script exits WITHOUT flushing, so a +short-lived process sends nothing: the trace never reaches the backend (or lands +empty). Same golden path as the clean fixture — a complete trace would be: + general (run) -> tool (retrieve) + llm (generate) + +This is the decisive test of "verify coverage, not just arrival": a skill that +only edits code, or that assumes an already-decorated app is fine, will wrongly +report success. A skill that actually runs and checks coverage will find no +complete trace and must NOT claim `verified` / `already_verified` — it should +either fix the flush and land a real trace, or return `blocked` with a +flush next-step. Either honest outcome passes; claiming success without a +real, complete trace fails. +""" + +from __future__ import annotations + +import os + +import opik +from openai import OpenAI +from opik.integrations.openai import track_openai + +client = track_openai(OpenAI()) + +_CORPUS = {"opik": "Opik is an LLM observability tool for tracing and evaluating LLM apps."} + + +@opik.track(type="tool") +def retrieve(query: str) -> str: + return _CORPUS.get(query.lower().split()[0], "no context found") + + +@opik.track # entrypoint -> general +def run(question: str) -> str: + context = retrieve(question) + resp = client.chat.completions.create( + model=os.getenv("OPENAI_MODEL", "gpt-4o-mini"), + messages=[ + {"role": "system", "content": "Answer the question using only the context."}, + {"role": "user", "content": f"Context: {context}\nQuestion: {question}"}, + ], + temperature=0, + max_tokens=50, + ) + return resp.choices[0].message.content.strip() + + +if __name__ == "__main__": + print(run("What is opik?")) + # PLANTED DEFECT: no `opik.flush_tracker()` before exit, so the batch is never + # sent and no complete trace lands. The eval checks that the skill catches the + # missing trace at verify time rather than reporting success on code alone. diff --git a/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/expected.json b/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/expected.json new file mode 100644 index 0000000..b64750f --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/expected.json @@ -0,0 +1,8 @@ +{ + "case": "missing_flush", + "framework": "openai", + "expected_sites": 3, + "expected_types": ["general", "llm", "tool"], + "defect": "missing_flush", + "note": "Already instrumented but exits without flush, so no complete trace lands. The skill must not claim success without a real, complete trace (it may fix the flush and verify, or block with a flush next-step)." +} diff --git a/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/pyproject.toml b/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/pyproject.toml new file mode 100644 index 0000000..9af0eb2 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/fixtures/missing_flush/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "instrument-eval-missing-flush" +version = "0.0.0" +description = "Already-instrumented fixture with a planted missing-flush defect for the opik-instrument eval." +requires-python = ">=3.10" +dependencies = ["openai>=1.0", "opik>=1.7"] diff --git a/src/opik_mcp/skills/opik-instrument/evals/grader.py b/src/opik_mcp/skills/opik-instrument/evals/grader.py new file mode 100644 index 0000000..bf31ab5 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/grader.py @@ -0,0 +1,171 @@ +# mypy: ignore-errors +"""Deterministic grader for the `/opik-instrument` skill. + +Given a case's `assert` block, the fixture ground truth (expected.json), and the +agent's result.json, check that the skill instrumented, ran, and *verified* a +real, complete trace — not just that it edited code or that "a trace arrived". + +result.json contract (the agent writes this after running the skill): + + { + "status": "verified" | "blocked" | "already_verified" | "unsupported", + "trace_id": "...", "trace_url": "...", + "changes": ["added opik to pyproject", ...], + "next_step": "...", # required when blocked + "coverage": { + "expected_sites": 3, + "spans_found": 3, + "spans": [{"name": "run", "type": "general"}, ...] + } + } + +Grading is offline. An OPTIONAL integrity check re-reads the trace from Opik and +compares the reported span count to reality — it runs only when `trace_id` is +present and Opik is configured, and is skipped (not failed) otherwise. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +IGNORE = { + ".venv", + "__pycache__", + "uv.lock", + ".python-version", + ".git", + "result.json", + "expected.json", + "PROMPT.txt", +} +VALID_STATUS = {"verified", "blocked", "already_verified", "unsupported"} +SUCCESS_STATUS = {"verified", "already_verified"} + + +@dataclass +class CaseResult: + id: str + area: str + checks: dict[str, tuple[bool, str]] = field(default_factory=dict) + + @property + def passed(self) -> bool: + return all(ok for ok, _ in self.checks.values()) + + +def _fixture_modified(fixture: Path, workdir: Path) -> list[str]: + mod = [] + for p in fixture.rglob("*"): + rel = p.relative_to(fixture) + if p.is_file() and not (set(rel.parts) & IGNORE): + w = workdir / rel + if not w.exists() or w.read_bytes() != p.read_bytes(): + mod.append(str(rel)) + # new files the agent added (e.g. .env, config) also count as instrumentation work + for p in workdir.rglob("*"): + rel = p.relative_to(workdir) + if p.is_file() and not (set(rel.parts) & IGNORE) and not (fixture / rel).exists(): + mod.append(str(rel)) + return sorted(set(mod)) + + +def _spans(result: dict) -> list[dict]: + cov = result.get("coverage") or {} + return [s for s in (cov.get("spans") or []) if isinstance(s, dict)] + + +def _online_span_count(trace_id: str) -> int | None: + """Best-effort: real span count from Opik. None if unavailable (never fails).""" + try: + import opik + + client = opik.Opik() + return len(client.search_spans(trace_id=trace_id)) + except Exception: + return None + + +def grade_case( + case: dict, + fixture: Path, + workdir: Path, + result: dict | None, + expected: dict | None, + area: str = "functional", +) -> CaseResult: + a = case.get("assert", {}) + result = result or {} + expected = expected or {} + exp_sites = int(expected.get("expected_sites", 0) or 0) + status = result.get("status") + cov = result.get("coverage") or {} + spans = _spans(result) + checks: dict[str, tuple[bool, str]] = {} + + def add(name, ok, detail=""): + checks[name] = (bool(ok), detail) + + # --- terminal state ------------------------------------------------- + if "status" in a: + add("status", status == a["status"], f"want {a['status']}, got {status}") + + if a.get("no_false_success"): + if status in SUCCESS_STATUS: + found = cov.get("spans_found") + ok = ( + bool(result.get("trace_id")) + and isinstance(found, int) + and exp_sites + and found >= exp_sites + ) + add( + "no_false_success", + ok, + f"claimed {status} but trace_id={result.get('trace_id')!r} " + f"coverage={found}/{exp_sites}", + ) + else: + add("no_false_success", True, f"did not claim success (status={status})") + + # --- coverage reporting (the new ability) --------------------------- + if a.get("coverage_reported"): + found = cov.get("spans_found") + add( + "coverage_reported", + isinstance(found, int) and bool(spans), + f"coverage={cov!r}", + ) + + if a.get("cover_types"): + want = {t.lower() for t in a["cover_types"]} + got = {str(s.get("type", "")).lower() for s in spans} + add("cover_types", want.issubset(got), f"types={sorted(got)} missing {sorted(want - got)}") + + if a.get("spans_well_formed"): + bad = [s for s in spans if not str(s.get("name", "")).strip() or not str(s.get("type", "")).strip()] + add("spans_well_formed", not bad, f"malformed spans (empty name/type): {bad}") + + # --- instrumentation actually happened ------------------------------ + if a.get("modified_code"): + mod = _fixture_modified(fixture, workdir) + add("modified_code", bool(mod), "no fixture files were changed") + + if a.get("next_step_contains"): + ns = str(result.get("next_step") or "").lower() + missing = [s for s in a["next_step_contains"] if s.lower() not in ns] + add("next_step_contains", not missing, f"next_step={result.get('next_step')!r} missing {missing}") + + # --- optional online integrity (skipped if unavailable) ------------- + tid = result.get("trace_id") + if tid and cov.get("spans_found") is not None: + real = _online_span_count(str(tid)) + if real is not None: + add( + "integrity", + real >= exp_sites if status in SUCCESS_STATUS else True, + f"opik reports {real} spans for {tid}, expected >= {exp_sites}", + ) + + add("schema", status in VALID_STATUS, f"status {status!r} not in {sorted(VALID_STATUS)}") + return CaseResult(id=case["id"], area=area, checks=checks) diff --git a/src/opik_mcp/skills/opik-instrument/evals/metrics.py b/src/opik_mcp/skills/opik-instrument/evals/metrics.py new file mode 100644 index 0000000..6973805 --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/metrics.py @@ -0,0 +1,56 @@ +# mypy: ignore-errors +"""Success metrics for the `/opik-instrument` skill. + +selection_accuracy - triggers on "add tracing", not on diagnose/explain/evaluate +verify_correctness - reached the correct terminal state for the fixture +no_false_success_rate - never claimed success without a real, complete trace (target 1.0) +coverage_reported_rate - reported span-level coverage, not just "a trace arrived" +type_coverage - the expected span types (general/llm/tool) were all present +well_formed_rate - every reported span had a non-empty name and type +instrumented_rate - actually changed code (instrument modifies; target 1.0) +integrity_rate - reported coverage matched the real trace in Opik (when checkable) +schema_compliance - status is a valid terminal state +""" + +from __future__ import annotations + + +def _group_rate(results, pred) -> float | None: + vals = [ok for r in results for n, (ok, _) in r.checks.items() if pred(n)] + return round(sum(vals) / len(vals), 3) if vals else None + + +def compute(results: list, triggering: dict | None = None) -> dict: + m: dict = {} + + if triggering: + st = triggering.get("should_trigger", {}) + sn = triggering.get("should_not_trigger", {}) + correct = sum(1 for v in st.values() if v) + sum(1 for v in sn.values() if not v) + total = len(st) + len(sn) + m["selection_accuracy"] = round(correct / total, 3) if total else 0.0 + + m["verify_correctness"] = _group_rate(results, lambda n: n in ("status", "no_false_success")) + m["no_false_success_rate"] = _group_rate(results, lambda n: n == "no_false_success") + m["coverage_reported_rate"] = _group_rate(results, lambda n: n == "coverage_reported") + m["type_coverage"] = _group_rate(results, lambda n: n == "cover_types") + m["well_formed_rate"] = _group_rate(results, lambda n: n == "spans_well_formed") + m["instrumented_rate"] = _group_rate(results, lambda n: n == "modified_code") + m["integrity_rate"] = _group_rate(results, lambda n: n == "integrity") + m["schema_compliance"] = _group_rate(results, lambda n: n == "schema") + m["cases_passed"] = f"{sum(1 for r in results if r.passed)}/{len(results)}" + return m + + +def report(results: list, metrics: dict) -> str: + lines = ["# /opik-instrument eval report", ""] + for r in results: + mark = "PASS" if r.passed else "FAIL" + lines.append(f"[{mark}] {r.id} ({r.area})") + for name, (ok, detail) in r.checks.items(): + if not ok: + lines.append(f" - FAILED {name}: {detail}") + lines += ["", "## Metrics"] + for k, v in metrics.items(): + lines.append(f" {k}: {v}") + return "\n".join(lines) diff --git a/src/opik_mcp/skills/opik-instrument/evals/run_evals.py b/src/opik_mcp/skills/opik-instrument/evals/run_evals.py new file mode 100644 index 0000000..e6499cc --- /dev/null +++ b/src/opik_mcp/skills/opik-instrument/evals/run_evals.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# mypy: ignore-errors +"""Test-automation harness for the `/opik-instrument` skill. + +Flows: + + 1. Functional (default): + uv run --with pyyaml python run_evals.py prepare # stage the fixture apps + # ...for each workdir in _work/, run /opik-instrument on it (see PROMPTS.md); + # the skill writes result.json = {status, trace_id, changes, coverage, ...}... + uv run --with pyyaml python run_evals.py grade # score result.json vs expected.json + + 2. Triggering (selection_accuracy): + uv run --with pyyaml python run_evals.py trigger-prepare + uv run --with pyyaml python run_evals.py trigger-grade + +Unlike the diagnose harness, `prepare` does NOT seed traces — the fixture apps +are what the skill instruments and runs. Grading is offline; an optional online +integrity re-read of the trace runs only if Opik is configured (see grader.py). +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +from pathlib import Path + +import grader +import metrics + +HERE = Path(__file__).resolve().parent +SKILL = HERE.parent / "SKILL.md" +FIXTURES = HERE / "fixtures" +WORK = HERE / "_work" +TRIG = WORK / "triggering" + +DECOY_SKILLS = [ + { + "name": "opik-diagnose", + "description": "Surface the Opik traces worth attention, ranked by signal. " + "Use to discover which traces are broken, not to add tracing.", + }, + { + "name": "opik-explain", + "description": "Root-cause a specific Opik trace you already have. " + "Use to debug ONE trace, not to add tracing.", + }, + { + "name": "opik-evaluate", + "description": "Build an Opik evaluation and run it, returning scores. " + "Offline experiment results, not adding observability.", + }, + { + "name": "opik", + "description": "Reference for how Opik works — concepts and SDK options. " + "Use to look up the product, not to instrument an app.", + }, + { + "name": "code-review", + "description": "Review code and report findings without changing it — " + "a general audit, unrelated to tracing.", + }, +] + + +def load_cases() -> dict: + import yaml + + return yaml.safe_load((HERE / "cases.yaml").read_text()) + + +def functional(cases: dict) -> list[dict]: + return cases.get("functional", []) + + +def _read_json(path: Path) -> dict | None: + if path.exists(): + try: + return json.loads(path.read_text()) + except Exception: + return None + return None + + +def prepare() -> None: + cases = load_cases() + WORK.mkdir(exist_ok=True) + lines = [f"skill: {SKILL}", ""] + for c in functional(cases): + wd = WORK / c["id"] + if wd.exists(): + shutil.rmtree(wd) + shutil.copytree(FIXTURES / c["fixture"], wd) + (wd / "PROMPT.txt").write_text(c["prompt"]) + lines.append(f"## {c['id']}\n- workdir: {wd}\n- prompt: {c['prompt']}\n") + (WORK / "PROMPTS.md").write_text("\n".join(lines)) + print(f"Prepared {len(functional(cases))} workdir(s) under {WORK}") + print("Run /opik-instrument in each workdir (see PROMPT.txt), have it write") + print("result.json there, then `grade`. Running needs an LLM provider key + Opik configured.") + + +def grade() -> int: + cases = load_cases() + results = [] + for c in functional(cases): + wd = WORK / c["id"] + if not wd.exists(): + print(f" ! skip {c['id']}: no workdir (run `prepare` + the skill first)") + continue + result = _read_json(wd / "result.json") + expected = _read_json(wd / "expected.json") + results.append(grader.grade_case(c, FIXTURES / c["fixture"], wd, result, expected)) + m = metrics.compute(results) + rep = metrics.report(results, m) + (WORK / "report.md").write_text(rep) + print(rep) + return 0 if results and all(r.passed for r in results) else 1 + + +# ---------- triggering ---------- + + +def _skill_description() -> str: + fm = SKILL.read_text().split("---")[1] + m = re.search(r"^description:\s*(.+)$", fm, re.M) + return m.group(1).strip() if m else "" + + +def trigger_prepare() -> None: + trig = load_cases().get("triggering", {}) + TRIG.mkdir(parents=True, exist_ok=True) + menu = [{"name": "opik-instrument", "description": _skill_description()}, *DECOY_SKILLS] + phrases = [{"phrase": p, "expect": "opik-instrument"} for p in trig.get("should_trigger", [])] + [ + {"phrase": p, "expect": "not-opik-instrument"} for p in trig.get("should_not_trigger", []) + ] + (TRIG / "phrases.json").write_text(json.dumps(phrases, indent=2)) + lines = [ + "# Triggering judge input", + "", + "For EACH user phrase, pick the ONE skill whose description best fits, or", + "`none`. Judge only from the descriptions.", + "", + "## Skill menu", + "", + ] + lines += [f"- **{s['name']}**: {s['description']}" for s in menu] + lines += ["", "## Phrases", ""] + lines += [f"{i + 1}. {p['phrase']}" for i, p in enumerate(phrases)] + lines += [ + "", + "## Output", + "", + 'Return STRICT JSON: {"verdicts": {"": ""}}', + ] + (TRIG / "judge_input.md").write_text("\n".join(lines)) + print( + f"Wrote {TRIG / 'judge_input.md'}. Judge it, write " + f"{TRIG / 'verdicts.json'}, then: trigger-grade" + ) + + +def trigger_grade() -> int: + f = TRIG / "verdicts.json" + if not f.exists(): + print(f" ! no {f}: run trigger-prepare + a judge first") + return 2 + verdicts = json.loads(f.read_text()) + trig = load_cases().get("triggering", {}) + st = {p: (verdicts.get(p) == "opik-instrument") for p in trig.get("should_trigger", [])} + sn = {p: (verdicts.get(p) == "opik-instrument") for p in trig.get("should_not_trigger", [])} + m = metrics.compute([], triggering={"should_trigger": st, "should_not_trigger": sn}) + for p, did in st.items(): + print(f"[{'PASS' if did else 'FAIL'}] should_trigger: {p!r} -> {verdicts.get(p)}") + for p, did in sn.items(): + print(f"[{'PASS' if not did else 'FAIL'}] should_not_trigger: {p!r} -> {verdicts.get(p)}") + print(f"\nselection_accuracy: {m.get('selection_accuracy')}") + return 0 if m.get("selection_accuracy") == 1.0 else 1 + + +def main() -> int: + ap = argparse.ArgumentParser(description="Eval harness for the /opik-instrument skill") + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("prepare", help="stage the fixture apps under _work/") + sub.add_parser("grade", help="grade result.json vs expected.json") + sub.add_parser("trigger-prepare", help="emit the triggering judge input") + sub.add_parser("trigger-grade", help="score verdicts.json -> selection_accuracy") + args = ap.parse_args() + if args.cmd == "prepare": + prepare() + return 0 + if args.cmd == "grade": + return grade() + if args.cmd == "trigger-prepare": + trigger_prepare() + return 0 + if args.cmd == "trigger-grade": + return trigger_grade() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) From 73fb891ef28ab8b1ac54542aa6806cf2323dee89 Mon Sep 17 00:00:00 2001 From: Aswin Prakash Thiyagarajan Date: Tue, 1 Sep 2026 11:50:42 -0400 Subject: [PATCH 2/2] [OPIK-8186] evals: fix ruff E501 (line-length 100) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrap the three long lines flagged by `ruff check` in grader.py and run_evals.py and apply `ruff format`. No behavior change — self-test still passes correct outputs and still catches a false `already_verified`. Co-Authored-By: Claude Opus 4.8 --- src/opik_mcp/skills/opik-instrument/evals/grader.py | 12 ++++++++++-- .../skills/opik-instrument/evals/run_evals.py | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/opik_mcp/skills/opik-instrument/evals/grader.py b/src/opik_mcp/skills/opik-instrument/evals/grader.py index bf31ab5..44ed983 100644 --- a/src/opik_mcp/skills/opik-instrument/evals/grader.py +++ b/src/opik_mcp/skills/opik-instrument/evals/grader.py @@ -143,7 +143,11 @@ def add(name, ok, detail=""): add("cover_types", want.issubset(got), f"types={sorted(got)} missing {sorted(want - got)}") if a.get("spans_well_formed"): - bad = [s for s in spans if not str(s.get("name", "")).strip() or not str(s.get("type", "")).strip()] + bad = [ + s + for s in spans + if not str(s.get("name", "")).strip() or not str(s.get("type", "")).strip() + ] add("spans_well_formed", not bad, f"malformed spans (empty name/type): {bad}") # --- instrumentation actually happened ------------------------------ @@ -154,7 +158,11 @@ def add(name, ok, detail=""): if a.get("next_step_contains"): ns = str(result.get("next_step") or "").lower() missing = [s for s in a["next_step_contains"] if s.lower() not in ns] - add("next_step_contains", not missing, f"next_step={result.get('next_step')!r} missing {missing}") + add( + "next_step_contains", + not missing, + f"next_step={result.get('next_step')!r} missing {missing}", + ) # --- optional online integrity (skipped if unavailable) ------------- tid = result.get("trace_id") diff --git a/src/opik_mcp/skills/opik-instrument/evals/run_evals.py b/src/opik_mcp/skills/opik-instrument/evals/run_evals.py index e6499cc..d64efd3 100644 --- a/src/opik_mcp/skills/opik-instrument/evals/run_evals.py +++ b/src/opik_mcp/skills/opik-instrument/evals/run_evals.py @@ -133,9 +133,9 @@ def trigger_prepare() -> None: trig = load_cases().get("triggering", {}) TRIG.mkdir(parents=True, exist_ok=True) menu = [{"name": "opik-instrument", "description": _skill_description()}, *DECOY_SKILLS] - phrases = [{"phrase": p, "expect": "opik-instrument"} for p in trig.get("should_trigger", [])] + [ - {"phrase": p, "expect": "not-opik-instrument"} for p in trig.get("should_not_trigger", []) - ] + phrases = [ + {"phrase": p, "expect": "opik-instrument"} for p in trig.get("should_trigger", []) + ] + [{"phrase": p, "expect": "not-opik-instrument"} for p in trig.get("should_not_trigger", [])] (TRIG / "phrases.json").write_text(json.dumps(phrases, indent=2)) lines = [ "# Triggering judge input",