Skip to content

Commit 3a8ba2b

Browse files
feat: provider-truth cost metering with cost_source (#62)
Add CostSnapshot.cost_source (billed|estimated|console|unknown) and keep billed/console totals separate from token estimates. Normalize openrouter/ gateway prefixes for OpenRouter price lookup, consume extras.cost_usd and generation IDs on CLI turns even when tokens=0, and emit estimated_cost on provider_call. Analyze/export/leaderboard Pareto only include billed, estimated, or console — never unknown $0. Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Jason <jkbennitt@users.noreply.github.com>
1 parent 9a1dc75 commit 3a8ba2b

14 files changed

Lines changed: 774 additions & 47 deletions

‎scripts/analyze_spread.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@
1010
from collections import Counter, defaultdict
1111
from pathlib import Path
1212

13+
from rle.tracking.cost_tracker import (
14+
authoritative_cost_usd,
15+
infer_cost_source,
16+
is_pareto_cost_source,
17+
)
18+
1319
# Failure substrings attributable to the harness / known RIMAPI quirks, not the model.
1420
HARNESS_FAILURE_MARKERS = [
1521
"Invalid plant definition", # Plant_Rice / Plant_Potato def-name quirk
@@ -122,10 +128,12 @@ def main() -> None:
122128
delta = [round(a - b, 4) for a, b in zip(traj, base_slice)]
123129
ticks_above = sum(1 for x in delta if x > 0)
124130
c = m["summary"]["cost_snapshot"]
125-
# Billed ground truth (run_scenario reconciles via OpenRouter's
126-
# generation API) — same schema we hand-patched into the v0.3.0
127-
# leaderboard from the dashboard. Estimates stay for comparison.
131+
# Billed / console / estimated stay distinct. cost_source is the
132+
# enum (billed|estimated|console|unknown); unknown $0 is not a
133+
# Pareto point.
128134
billed = m["summary"].get("billed_cost")
135+
source = infer_cost_source(c, billed if isinstance(billed, dict) else None)
136+
display = authoritative_cost_usd(c, billed if isinstance(billed, dict) else None)
129137
rows.append({
130138
"model": m["summary"]["model"],
131139
"name": name,
@@ -143,11 +151,22 @@ def main() -> None:
143151
"avg_latency_s": round(m["deliberation"]["avg_latency_ms"] / 1000, 1),
144152
"avg_confidence": m["deliberation"]["avg_confidence"],
145153
"wall_min": round(c["wall_time_s"] / 60, 1),
146-
"est_cost_usd": round(c["estimated_cost_usd"], 2),
154+
"est_cost_usd": round(c.get("estimated_cost_usd") or 0, 2),
147155
"real_cost_usd": (
148-
round(billed["billed_cost_usd"], 3) if billed else None
156+
round(float(billed["billed_cost_usd"]), 3)
157+
if isinstance(billed, dict) and billed.get("billed_cost_usd") is not None
158+
else (
159+
round(float(c["billed_cost_usd"]), 3)
160+
if c.get("billed_cost_usd") is not None else None
161+
)
162+
),
163+
"console_cost_usd": (
164+
round(float(c["console_cost_usd"]), 3)
165+
if c.get("console_cost_usd") is not None else None
149166
),
150-
"cost_source": billed["source"] if billed else None,
167+
"display_cost_usd": round(display, 3) if display is not None else None,
168+
"cost_source": source.value,
169+
"pareto_eligible": is_pareto_cost_source(source),
151170
"trajectory": traj,
152171
"days": days,
153172
"baseline_slice": [round(x, 4) for x in base_slice],

‎scripts/export_site_data.py‎

Lines changed: 73 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
from decimal import ROUND_HALF_UP, Decimal
2323
from pathlib import Path
2424

25+
from rle.tracking.cost_tracker import is_pareto_cost_source
26+
2527
# Display name / family / family swatch per model slug, mirroring the palette
2628
# in AppSprout-Site client/src/rle/data.ts. Unknown slugs fall back to a
2729
# derived family + neutral color with a warning, so a new model never blocks
@@ -43,6 +45,32 @@
4345
}
4446
FALLBACK_COLOR = "#B7AC8B"
4547

48+
# Optional published-pack backfill (Site may apply later). Not live-metered
49+
# here and not fetched from any API — values from the n1-final pack review:
50+
# CLI coding-agent harness estimated $1.594858 from 796917 tokens; Felix keep
51+
# billed $0.857296; ACP console ~$7; raw = unknown / omit from cost Pareto.
52+
PACK_COST_BACKFILL: dict[str, dict[str, object]] = {
53+
"cli-agent": {
54+
"cost_usd": 1.594858,
55+
"cost_source": "estimated",
56+
"note": "796917 tokens; pricing slug was openrouter/x-ai/grok-4.6",
57+
},
58+
"felix": {
59+
"cost_usd": 0.857296,
60+
"cost_source": "billed",
61+
},
62+
"acp": {
63+
"cost_usd": 7.0,
64+
"cost_source": "console",
65+
"note": "n1-final console ~$7",
66+
},
67+
"raw-grok": {
68+
"cost_usd": None,
69+
"cost_source": "unknown",
70+
"note": "omit from cost Pareto",
71+
},
72+
}
73+
4674
GITHUB_URL = "https://github.com/AppSprout-dev/RLE"
4775
GITHUB_REPO = "AppSprout-dev/RLE"
4876
DISCORD_URL = "https://discord.gg/vKuyjqNEea"
@@ -69,11 +97,30 @@ def registry_entry(slug: str) -> tuple[str, str, str]:
6997
return slug, family, FALLBACK_COLOR
7098

7199

100+
def _row_cost_source(row: dict) -> str:
101+
raw = row.get("cost_source")
102+
return str(raw) if raw else "unknown"
103+
104+
72105
def build_model(row: dict) -> dict:
73106
slug = row["model"]
74107
display, family, color = registry_entry(slug)
75-
real = row.get("real_cost_usd")
76-
cost = real if real is not None else row["est_cost_usd"]
108+
source = _row_cost_source(row)
109+
eligible = bool(row.get("pareto_eligible", is_pareto_cost_source(source)))
110+
# Prefer the analyze-spread authoritative figure; never fall back to
111+
# unknown $0 (that used to land CLI/ACP/raw rows on the Site Pareto).
112+
display_cost = row.get("display_cost_usd")
113+
if display_cost is None and eligible:
114+
real = row.get("real_cost_usd")
115+
console = row.get("console_cost_usd")
116+
if source == "billed" and real is not None:
117+
display_cost = real
118+
elif source == "console" and console is not None:
119+
display_cost = console
120+
elif source == "estimated":
121+
display_cost = row.get("est_cost_usd")
122+
if not eligible:
123+
display_cost = None
77124
return {
78125
"slug": slug,
79126
"key": row["name"],
@@ -89,13 +136,24 @@ def build_model(row: dict) -> dict:
89136
"exArtifactSuccess": rnd(row["ex_artifact_success"], 2),
90137
"avgLatencyS": rnd(row["avg_latency_s"], 1),
91138
"wallMin": rnd(row["wall_min"], 1),
92-
"costUsd": rnd(cost, 2),
93-
"costEstimated": real is None,
139+
"costUsd": rnd(display_cost, 2) if display_cost is not None else None,
140+
"costEstimated": source == "estimated",
141+
"costSource": source,
142+
"paretoEligible": eligible and display_cost is not None,
94143
}
95144

96145

97146
def build_meta(board: dict, rows: list[dict], args: argparse.Namespace) -> dict:
98-
metered = [r["real_cost_usd"] for r in board["rows"] if r.get("real_cost_usd") is not None]
147+
metered: list[float] = []
148+
for r in board["rows"]:
149+
source = _row_cost_source(r)
150+
if source not in ("billed", "console"):
151+
continue
152+
amount = r.get("real_cost_usd") if source == "billed" else r.get("console_cost_usd")
153+
if amount is None:
154+
amount = r.get("display_cost_usd")
155+
if amount is not None:
156+
metered.append(float(amount))
99157
ticks = max((len(r.get("trajectory", [])) for r in board["rows"]), default=0)
100158
return {
101159
"scenario": args.scenario,
@@ -127,8 +185,8 @@ def emit_ts(meta: dict, models: list[dict], spread_dir: Path) -> str:
127185
f" ticks: {meta['ticks']},",
128186
f" nRuns: {meta['nRuns']},",
129187
f' date: "{meta["date"]}",',
130-
" // OpenRouter spend for the metered models; subscription-billed models",
131-
" // (costEstimated: true) are not included.",
188+
" // Billed + console spend; estimated/unknown rows are not included.",
189+
" // Plot Pareto only when costSource is billed|estimated|console.",
132190
f" totalSpendUsd: {fmt(meta['totalSpendUsd'], 2)},",
133191
f" baselineMeanTteDays: {meta['baselineMeanTteDays']},",
134192
f' githubUrl: "{meta["githubUrl"]}",',
@@ -151,8 +209,10 @@ def emit_ts(meta: dict, models: list[dict], spread_dir: Path) -> str:
151209
f" endDay: {m['endDay']}, rawActionSuccess: {fmt(m['rawActionSuccess'], 2)},"
152210
f" exArtifactSuccess: {fmt(m['exArtifactSuccess'], 2)},"
153211
f" avgLatencyS: {fmt(m['avgLatencyS'], 1)}, wallMin: {fmt(m['wallMin'], 1)},",
154-
f" costUsd: {fmt(m['costUsd'], 2)},"
155-
f" costEstimated: {'true' if m['costEstimated'] else 'false'},",
212+
f" costUsd: {fmt(m['costUsd'], 2) if m['costUsd'] is not None else 'null'},"
213+
f" costEstimated: {'true' if m['costEstimated'] else 'false'},"
214+
f" costSource: \"{m['costSource']}\","
215+
f" paretoEligible: {'true' if m['paretoEligible'] else 'false'},",
156216
" },",
157217
]
158218
lines += ["];", ""]
@@ -185,7 +245,10 @@ def main() -> None:
185245

186246
json_path = out / "site_data.json"
187247
json_path.write_text(
188-
json.dumps({"meta": meta, "models": models}, indent=2) + "\n",
248+
json.dumps(
249+
{"meta": meta, "models": models, "costBackfill": PACK_COST_BACKFILL},
250+
indent=2,
251+
) + "\n",
189252
encoding="utf-8")
190253

191254
print(f"Wrote {ts_path}")

‎scripts/run_benchmark.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -535,15 +535,18 @@ async def main(args: argparse.Namespace) -> None: # noqa: PLR0912, PLR0915
535535
# Reconcile estimates against OpenRouter's billed ground truth
536536
# (token-count estimates diverged up to 4x on the v0.3.0 spread).
537537
billed_report = None
538-
effective_base_url = config.provider_base_url or ""
539538
openai_key = os.environ.get("OPENAI_API_KEY", "")
540539
generation_ids = cost_tracker.generation_ids
541-
if "openrouter.ai" in effective_base_url and openai_key and generation_ids:
540+
# Provider-truth: generation IDs + key are enough. CLI/ACP harnesses
541+
# often report localhost as base_url while still billing through OpenRouter.
542+
if openai_key and generation_ids:
542543
print(
543544
f"\nReconciling billed cost for {len(generation_ids)} "
544545
"generations against OpenRouter...",
545546
)
546547
billed_report = await fetch_billed_costs(generation_ids, openai_key)
548+
if billed_report is not None:
549+
cost_tracker.apply_billed(billed_report.billed_cost_usd)
547550

548551
clean_results = [r for r in results if not r.get("harness_failed")]
549552
metadata = collect_metadata(random_seed=args.seed)

‎scripts/run_scenario.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,15 +298,18 @@ async def main(args: argparse.Namespace) -> None:
298298
# token-count estimator diverged up to 4x in both directions on the
299299
# v0.3.0 spread (thinking-model usage shapes, caching discounts).
300300
billed_report = None
301-
effective_base_url = config.provider_base_url or ""
302301
openai_key = os.environ.get("OPENAI_API_KEY", "")
303302
generation_ids = cost_tracker.generation_ids
304-
if "openrouter.ai" in effective_base_url and openai_key and generation_ids:
303+
# Provider-truth: generation IDs + key are enough. CLI/ACP harnesses
304+
# often report localhost as base_url while still billing through OpenRouter.
305+
if openai_key and generation_ids:
305306
print(
306307
f"\nReconciling billed cost for {len(generation_ids)} generations "
307308
"against OpenRouter...",
308309
)
309310
billed_report = await fetch_billed_costs(generation_ids, openai_key)
311+
if billed_report is not None:
312+
cost_tracker.apply_billed(billed_report.billed_cost_usd)
310313

311314
if args.output:
312315
output_dir = Path(args.output)

‎src/rle/harness/cli_base.py‎

Lines changed: 89 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
from rle.orchestration.action_executor import ActionExecutor
3333
from rle.rimapi.schemas import GameState
3434
from rle.rimapi.sse_client import RimAPIEvent
35+
from rle.tracking.cost_tracker import CostSource, CostTracker
3536
from rle.tracking.event_log import EventType
3637

3738
logger = logging.getLogger(__name__)
@@ -121,7 +122,12 @@ def mcp_listen(self, config: RLEConfig) -> McpListenSettings:
121122

122123
@dataclass
123124
class TurnResult:
124-
"""What the agent produced for one prompt."""
125+
"""What the agent produced for one prompt.
126+
127+
``extras`` may carry provider-truth metering that is independent of
128+
token counts: ``cost_usd``, ``cost_source`` (``billed`` / ``console`` /
129+
``estimated``), and ``generation_id`` / ``generation_ids``.
130+
"""
125131

126132
text: str = ""
127133
prompt_tokens: int = 0
@@ -130,6 +136,85 @@ class TurnResult:
130136
extras: dict[str, Any] = field(default_factory=dict)
131137

132138

139+
def _extras_generation_ids(extras: dict[str, Any]) -> list[str]:
140+
"""Collect generation IDs from extras (singular, plural, or nested)."""
141+
ids: list[str] = []
142+
raw = extras.get("generation_ids", extras.get("generation_id"))
143+
if raw is None:
144+
raw = extras.get("generationId")
145+
if isinstance(raw, str):
146+
if raw:
147+
ids.append(raw)
148+
elif isinstance(raw, (list, tuple)):
149+
ids.extend(str(item) for item in raw if item)
150+
return ids
151+
152+
153+
def _extras_cost_usd(extras: dict[str, Any]) -> float | None:
154+
raw = extras.get("cost_usd", extras.get("costUsd"))
155+
if raw is None:
156+
return None
157+
try:
158+
return float(raw)
159+
except (TypeError, ValueError):
160+
return None
161+
162+
163+
def _extras_cost_source(extras: dict[str, Any]) -> CostSource:
164+
raw = extras.get("cost_source", extras.get("costSource", "console"))
165+
try:
166+
return CostSource(str(raw))
167+
except ValueError:
168+
return CostSource.CONSOLE
169+
170+
171+
def apply_turn_metering(tracker: CostTracker | None, turn: TurnResult) -> dict[str, Any]:
172+
"""Record tokens, extras.cost_usd, and generation IDs on ``tracker``.
173+
174+
Cost-only turns (tokens=0 but ``cost_usd`` / generation IDs present) are
175+
first-class — nothing here is gated on a non-zero token count. Returns
176+
the kwargs for a ``provider_call`` event, or ``{}`` when there is
177+
nothing to emit.
178+
"""
179+
extras = turn.extras
180+
cost_usd = _extras_cost_usd(extras)
181+
gen_ids = _extras_generation_ids(extras)
182+
has_tokens = bool(turn.prompt_tokens or turn.completion_tokens or turn.reasoning_tokens)
183+
if tracker is not None:
184+
for gen_id in gen_ids:
185+
tracker.record_generation_id(gen_id)
186+
if cost_usd is not None:
187+
tracker.record_provider_cost(
188+
cost_usd,
189+
source=_extras_cost_source(extras),
190+
count_call=not has_tokens,
191+
)
192+
if has_tokens:
193+
tracker.record_raw(
194+
turn.prompt_tokens, turn.completion_tokens, turn.reasoning_tokens,
195+
)
196+
estimated = 0.0
197+
if tracker is not None:
198+
estimated = tracker.estimate_call_cost(
199+
turn.prompt_tokens, turn.completion_tokens, turn.reasoning_tokens,
200+
)
201+
if not (has_tokens or cost_usd is not None or gen_ids):
202+
return {}
203+
payload: dict[str, Any] = {
204+
"prompt_tokens": turn.prompt_tokens,
205+
"completion_tokens": turn.completion_tokens,
206+
"reasoning_tokens": turn.reasoning_tokens,
207+
"estimated_cost": estimated,
208+
"raw_output": turn.text[:_RAW_OUTPUT_CHARS],
209+
"raw_output_truncated": len(turn.text) > _RAW_OUTPUT_CHARS,
210+
}
211+
if cost_usd is not None:
212+
payload["cost_usd"] = cost_usd
213+
if gen_ids:
214+
payload["generation_ids"] = gen_ids
215+
return payload
216+
217+
133218
class HeadlessCliHarness(BaseHarness, ABC):
134219
name: ClassVar[str] = "headless-cli"
135220

@@ -280,18 +365,9 @@ async def step(
280365
agent=self.name, latency_ms=latency_ms, num_actions=n_actions,
281366
summary=step.plan.summary, error_type=None if status == "success" else status,
282367
)
283-
if turn.prompt_tokens or turn.completion_tokens:
284-
if self.ctx.cost_tracker:
285-
self.ctx.cost_tracker.record_raw(
286-
turn.prompt_tokens, turn.completion_tokens, turn.reasoning_tokens,
287-
)
288-
self.ctx.emit(
289-
EventType.PROVIDER_CALL, tick, agent=self.name,
290-
prompt_tokens=turn.prompt_tokens, completion_tokens=turn.completion_tokens,
291-
reasoning_tokens=turn.reasoning_tokens,
292-
raw_output=turn.text[:_RAW_OUTPUT_CHARS],
293-
raw_output_truncated=len(turn.text) > _RAW_OUTPUT_CHARS,
294-
)
368+
metering = apply_turn_metering(self.ctx.cost_tracker, turn)
369+
if metering:
370+
self.ctx.emit(EventType.PROVIDER_CALL, tick, agent=self.name, **metering)
295371

296372
extras = {**step.extras, "status": status, "latency_ms": latency_ms, **turn.extras}
297373
return StepResult(

‎src/rle/harness/felix/harness.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,8 +429,10 @@ async def _deliberate_agent(
429429
if not isinstance(rt, int):
430430
rt = 0
431431
if isinstance(pt, int) and isinstance(ct, int):
432+
estimated_cost = 0.0
432433
if self.ctx.cost_tracker:
433434
self.ctx.cost_tracker.record_raw(pt, ct, rt)
435+
estimated_cost = self.ctx.cost_tracker.estimate_call_cost(pt, ct, rt)
434436
raw_output = agent._last_raw_output
435437
raw_output_truncated = (
436438
raw_output[:_RAW_OUTPUT_CHARS] if raw_output else None
@@ -442,6 +444,7 @@ async def _deliberate_agent(
442444
EventType.PROVIDER_CALL, tick, agent=plan.role,
443445
prompt_tokens=pt, completion_tokens=ct,
444446
reasoning_tokens=rt,
447+
estimated_cost=estimated_cost,
445448
raw_output=raw_output_truncated,
446449
raw_output_truncated=was_truncated,
447450
)

0 commit comments

Comments
 (0)