diff --git a/README.md b/README.md index 1213f29..cf89f14 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Razorpay's MCP server lets an agent *pay*. Bazaar's MCP lets an agent *find whom Everyone assumes Shopify-scale catalogs, GTINs and cards. Bazaar compiles a WhatsApp seller's Google Sheet and answers "deliver to 560034 by Friday?" — the 80% of the market the others cannot reach. -**Economics (from the results):** quote maths, ranking and the policy gate never call a model, so cost does not scale with order value. The negotiator — the only model call on the money path — routes to a small model and caches, landing under ₹1 of model cost per completed order against ₹9–14 of take-rate revenue on a typical basket. Revenue lines: a take-rate on agent-originated GMV, a **Razorpay Agentic Plan** for brands with no Razorpay-hosted checkout (Shopify's Agentic plan, for India), and an agent-order protection bundle (RTO Shield + chargeback) priced per agent order. +**Economics (measured, not claimed).** Quote maths, ranking and the policy gate never call a model, so cost does not scale with order value — only the seller's *propose* step does. A cold-cache probe (every call live and metered by the token counters in the LLM client) put the model cost at **₹3.37 per completed order on gpt-4o** — 40 tasks, 27 orders, ₹90.87 total ([`results/gpt4o_costprobe/RESULTS.md`](results/gpt4o_costprobe/RESULTS.md)). That is against ₹9–14 of take-rate revenue on a typical basket, and it drops roughly 10–15× when propose is routed to gpt-4o-mini (or to zero on the free gpt-oss backend), because the deterministic paths carry the rest. Revenue lines: a take-rate on agent-originated GMV, a **Razorpay Agentic Plan** for brands with no Razorpay-hosted checkout (Shopify's Agentic plan, for India), and an agent-order protection bundle (RTO Shield + chargeback) priced per agent order. ## What breaks, and what happens @@ -218,6 +218,8 @@ Real incidents from this build, kept because the fixes became the architecture: | latency p50 / p95 | 47 / 62 ms (deterministic) | a cache hit ≈ the offline figures; a real gpt-4o proposal adds ~1.5–4 s (the p95 tracks how many calls in a run are live vs cached) | | model failovers during the run | — | 0 | +> **Why report a 0.094?** The held-out `name` score is the *offline dictionary* backend's, and a dictionary literally cannot know a brand it was never given ("Aashirvaad", "Daawat") — that is exactly why a model backend exists, and gpt-4o lifts the same column to 0.469. It is reported anyway because it is honest and because it is **not a safety number**: a name the compiler is unsure of is sent to the merchant's review queue, never guessed, so a low name score costs review clicks, not a wrong price or a wrong order (those fields score 1.000 on both backends). Cost per completed order is a measured number, not a claim — see the economics section above. + ### Real agents, not scripts The 200-task table above is produced by a deterministic scripted buyer — reproducible, and the honest baseline. But "an agent *could*" is weaker than "an agent *did*", so three separate pieces of evidence show real models on the wire: diff --git a/bazaar/compiler/heldout.py b/bazaar/compiler/heldout.py index 2c9bf6b..04c2a01 100644 --- a/bazaar/compiler/heldout.py +++ b/bazaar/compiler/heldout.py @@ -32,7 +32,10 @@ def run_heldout(llm: LLM, heldout_dir: Path, merchants_by_vertical: dict[str, An rows_total = 0 for csv_path in sorted(heldout_dir.glob("*.csv")): truth = json.loads(csv_path.with_suffix("").with_suffix(".truth.json").read_text(encoding="utf-8")) - template = merchants_by_vertical[truth["vertical"]].model_copy(update={"products": []}) + base = merchants_by_vertical.get(truth["vertical"]) or next(iter(merchants_by_vertical.values()), None) + if base is None: + continue # no merchant loaded to use as a template (e.g. a trimmed corpus) + template = base.model_copy(update={"products": []}) compiled = compile_rows(read_csv(csv_path), template, llm, workers=workers) rows = truth["rows"] rows_total += len(rows) diff --git a/bazaar/llm/cache.py b/bazaar/llm/cache.py index bebe593..51f4037 100644 --- a/bazaar/llm/cache.py +++ b/bazaar/llm/cache.py @@ -67,3 +67,8 @@ def stats(self) -> dict[str, int]: with self._lock: n = self._db.execute("SELECT COUNT(*) FROM calls").fetchone()[0] return {"hits": self.hits, "misses": self.misses, "stored": n} + + def usage(self) -> dict[str, float]: + # token/cost accounting lives on the wrapped backend; cache hits never reach it, so + # this reports the cost of the real API calls this run (the misses) + return self.inner.usage() if hasattr(self.inner, "usage") else {} diff --git a/bazaar/llm/openai_client.py b/bazaar/llm/openai_client.py index 0d6ca79..cdbe03f 100644 --- a/bazaar/llm/openai_client.py +++ b/bazaar/llm/openai_client.py @@ -10,6 +10,15 @@ from bazaar.llm.base import LLM, LLMError +# USD per 1M tokens (input, output). Only the models we actually route to; extend as needed. +_PRICE_PER_MTOK: dict[str, tuple[float, float]] = { + "gpt-4o": (2.50, 10.00), + "gpt-4o-mini": (0.15, 0.60), + "openai/gpt-oss-120b": (0.0, 0.0), # Groq free tier + "qwen/qwen3.8-27b": (0.0, 0.0), +} +USD_TO_INR = 88.0 + class OpenAILLM(LLM): name = "openai" @@ -22,6 +31,24 @@ def __init__(self, api_key: str, model: str = "gpt-4o", base_url: str = "", task self._client = OpenAI(api_key=api_key, base_url=base_url or None) self._model = model self._task_models = dict(task_models or {}) # e.g. {"normalize_product": "gpt-4o-mini"} + # token/cost accounting — only real API calls reach here (cache hits never do), so this + # is a true measurement of what the run spent, not an estimate + self.prompt_tokens = 0 + self.completion_tokens = 0 + self.usd = 0.0 + + def usage(self) -> dict[str, float]: + return {"prompt_tokens": self.prompt_tokens, "completion_tokens": self.completion_tokens, "usd": round(self.usd, 4), "inr": round(self.usd * USD_TO_INR, 2)} + + def _account(self, model: str, resp) -> None: + u = getattr(resp, "usage", None) + if u is None: + return + pt, ct = int(getattr(u, "prompt_tokens", 0) or 0), int(getattr(u, "completion_tokens", 0) or 0) + self.prompt_tokens += pt + self.completion_tokens += ct + pin, pout = _PRICE_PER_MTOK.get(model, (0.0, 0.0)) + self.usd += pt / 1e6 * pin + ct / 1e6 * pout def complete_json(self, task: str, system: str, user: str, schema: dict[str, Any]) -> dict[str, Any]: return self._call(task, system, user, schema) @@ -32,10 +59,11 @@ def complete_json_image(self, task: str, system: str, user: str, image_b64: str, def _call(self, task: str, system: str, user: Any, schema: dict[str, Any]) -> dict[str, Any]: fn_name = f"answer_{task}" + model = self._task_models.get(task, self._model) tool = {"type": "function", "function": {"name": fn_name, "description": f"Return the structured answer for task '{task}'.", "parameters": schema}} try: resp = self._client.chat.completions.create( - model=self._task_models.get(task, self._model), + model=model, temperature=0, messages=[{"role": "system", "content": system}, {"role": "user", "content": user}], tools=[tool], @@ -43,6 +71,7 @@ def _call(self, task: str, system: str, user: Any, schema: dict[str, Any]) -> di ) except Exception as e: # noqa: BLE001 raise LLMError(str(e)) from e + self._account(model, resp) choice = resp.choices[0] if resp.choices else None calls = getattr(choice.message, "tool_calls", None) if choice else None if not calls: diff --git a/bazaar/simulator/run.py b/bazaar/simulator/run.py index d93b424..76b961b 100644 --- a/bazaar/simulator/run.py +++ b/bazaar/simulator/run.py @@ -220,6 +220,13 @@ def log(msg: str) -> None: if hasattr(llm, "stats"): report["backend"]["llm_cache"] = llm.stats() + if hasattr(llm, "usage"): + # cost per completed order, measured from real token usage (cache hits cost nothing). + # On a warm-cache replay this is the marginal cost of the calls that ran live this time. + u = llm.usage() + orders = report["transactions"]["orders"] + u["inr_per_order"] = round(u["inr"] / orders, 4) if orders else 0.0 + report["backend"]["llm_usage"] = u if hasattr(llm, "status"): # provenance: total_failovers=0 proves the real model answered (not the deterministic # fallback); on a cache replay misses=0 is expected and does not mean the model was skipped @@ -335,6 +342,16 @@ def render_markdown(r: dict[str, Any]) -> str: misses = cache.get("misses", 0) prov += f" LLM cache: {cache.get('hits', 0)} hits / {misses} misses" prov += (" — every call served from a prior run's cache; a warm-cache replay costs nothing and re-bills nothing, and the failover count above proves the model, not the fallback, produced the cached answers." if misses == 0 else f" ({misses} real model calls this run; the rest replayed from cache).") + usage = b.get("llm_usage") + if usage and usage.get("prompt_tokens"): + per = usage["inr_per_order"] + prov += ( + f"\n\n**Cost.** The live calls this run used {usage['prompt_tokens'] + usage['completion_tokens']:,} tokens " + f"(₹{usage['inr']:.2f} at listed {b.get('model', 'gpt-4o')} rates), which is **₹{per:.2f} per completed order** — " + f"quote maths, ranking and the policy gate never call a model, so only the seller's propose step costs anything, " + f"and it drops ~10–15× on gpt-4o-mini (or to zero on the free gpt-oss backend). " + f"(A warm-cache re-generation bills only the calls that changed, so its per-order figure is lower than this cold-cache measurement.)" + ) lines += ["", prov] lines += ["", f"_Elapsed {r['elapsed_s']} s._", ""] return "\n".join(lines) diff --git a/results/gpt4o_costprobe/RESULTS.md b/results/gpt4o_costprobe/RESULTS.md new file mode 100644 index 0000000..cb7be65 --- /dev/null +++ b/results/gpt4o_costprobe/RESULTS.md @@ -0,0 +1,63 @@ +# Bazaar — measured results + +Generated 2026-09-05T11:57:45Z by `python -m bazaar.simulator.run` (v0.1.0, llm=`openai`, payments=`fake`). Nothing here is hand-edited. + +## Catalog compiler (20 merchants, messy CSV → agent-readable catalog) + +| field | accuracy | +|---|---| +| name | 0.811 | +| price | 1.000 | +| unit | 0.966 | +| pack_size | 0.948 | +| category | 0.917 | +| gst | 0.925 | +| stock | 0.891 | + +Review rate 0.230 (items queued for the merchant instead of guessed) · injections neutralised **10/10** · readiness mean 96.3 (min 91). + +**Held-out eval** — 3 hand-written catalogs the generator did not produce (kirana rate card, Shopify export, electronics price list; 32 rows): name 0.500 · price 1.000 · unit 0.969 · pack_size 0.938 · stock 1.000 · gst 1.000 · review rate 0.719. Cells the source doesn't state (e.g. GST on a Shopify export) are review-queued, never guessed. + +## Transactions (40 buyer tasks, 25 possible / 15 impossible by construction) + +| metric | Bazaar | ablation: same catalog & index, negotiation off, same-city filter, no serviceability answers | +|---|---|---| +| orders | **27** | 23 | +| task → order | 67.5% | 57.5% | +| completion on possible tasks | 100.0% | 84.0% | +| GMV | **₹62,802** | ₹54,230 | +| discounts given (all rule-bounded) | ₹802 | — | +| negotiation rounds | 9 | 0 | + +Lift: **+4 orders, +₹8,572 GMV (1.16×)**. + +**4 orders (₹8,895) could not have happened at all without Bazaar** — 3 needed a bounded offer, 1 needed a other; 50% of them arrived in Hindi or Hinglish. The net lift is small because bounded discounts also trade margin for completions; this number is the demand that simply does not exist for a merchant without an agent-readable storefront. + +Declines on impossible tasks — precision 1.000, recall 1.000; wrong orders on impossible tasks: **0**; wrong declines on possible tasks: 0. Overall task accuracy 100.0%. Errors: 0. + +By language: hi-Latn 100.0%, en 100.0%, hi 100.0%. Latency p50 1689.5 ms · p95 4103.8 ms (in-process, llm=`openai`). + +## Trust + +- Audit entries 258, hash chain intact: **True**, Merkle root `b843402f86485d8e…` +- Explanations present on 100.0% of agent turns +- Grants issued 27, used 27; fairness-ledger entries 9, inconsistencies **0** + +## False-positive cost — policy strictness sweep + +Same tasks, tighter merchant per-order cap. Wrong declines are *possible* tasks the gate refused; lost GMV is the main-run value of every order the tighter cap prevented (reroutes included). The first row is the default cap and must match the table above. + +| per-order cap | orders | wrong declines on possible tasks | lost GMV | wrong orders on impossible tasks | +|---|---|---|---|---| +| ₹50,000 (default) | 27 | **0** | ₹0 | 0 | +| ₹10,000 (Reserve Pay block) | 25 | **2** | ₹36,120 | 0 | +| ₹5,000 | 24 | **3** | ₹42,524 | 0 | +| ₹2,000 | 23 | **4** | ₹46,540 | 0 | + +## Provenance + +Backend `openai` (model `gpt-4o`). Model failovers to the deterministic fallback during this run: **0** — so the model itself produced these results (health: degraded=False, failures=0). + +**Cost.** The live calls this run used 561,032 tokens (₹90.87 at listed gpt-4o rates), which is **₹3.37 per completed order** — quote maths, ranking and the policy gate never call a model, so only the seller's propose step costs anything, and it drops ~10–15× on gpt-4o-mini (or to zero on the free gpt-oss backend). (A warm-cache re-generation bills only the calls that changed, so its per-order figure is lower than this cold-cache measurement.) + +_Elapsed 573.3 s._ diff --git a/results/gpt4o_costprobe/results.json b/results/gpt4o_costprobe/results.json new file mode 100644 index 0000000..47d3718 --- /dev/null +++ b/results/gpt4o_costprobe/results.json @@ -0,0 +1,261 @@ +{ + "version": "0.1.0", + "generated_at": "2026-09-05T11:57:45Z", + "backend": { + "llm": "openai", + "payments": "fake", + "model": "gpt-4o", + "compile_model": "gpt-4o-mini", + "cache": false, + "llm_usage": { + "prompt_tokens": 506706, + "completion_tokens": 54326, + "usd": 1.0327, + "inr": 90.87, + "inr_per_order": 3.3656 + }, + "llm_health": { + "backend": "openai", + "degraded": false, + "forced_down": false, + "total_failures": 0, + "total_failovers": 0, + "last_error": "" + } + }, + "compiler": { + "merchants": 20, + "products": 387, + "accuracy": { + "name": 0.8113695090439277, + "price": 1.0, + "unit": 0.9664082687338501, + "pack_size": 0.9483204134366925, + "category": 0.917312661498708, + "gst": 0.9250645994832042, + "stock": 0.8914728682170543 + }, + "review_rate": 0.22997416020671835, + "injections_present": 10, + "injections_stripped": 10, + "readiness_mean": 96.3, + "readiness_min": 91, + "readiness_truth_mean": 100 + }, + "compiler_heldout": { + "catalogs": [ + { + "catalog": "boutique_shopify", + "vertical": "apparel", + "rows": 10, + "compiled": 10, + "field_hits": { + "name": 9, + "price": 10, + "unit": 10, + "pack_size": 10, + "stock": 10, + "gst": 0 + } + }, + { + "catalog": "electronics_pricelist", + "vertical": "electronics_accessories", + "rows": 10, + "compiled": 10, + "field_hits": { + "name": 4, + "price": 10, + "unit": 10, + "pack_size": 9, + "stock": 10, + "gst": 0 + } + }, + { + "catalog": "kirana_rate_card", + "vertical": "grocery", + "rows": 12, + "compiled": 12, + "field_hits": { + "name": 3, + "price": 12, + "unit": 11, + "pack_size": 11, + "stock": 12, + "gst": 12 + } + } + ], + "rows": 32, + "accuracy": { + "name": 0.5, + "price": 1.0, + "unit": 0.969, + "pack_size": 0.938, + "stock": 1.0, + "gst": 1.0 + }, + "scored_cells": { + "name": 32, + "price": 32, + "unit": 32, + "pack_size": 32, + "stock": 32, + "gst": 12 + }, + "review_rate": 0.719 + }, + "transactions": { + "tasks": 40, + "orders": 27, + "rerouted_orders": 2, + "task_to_order_rate": 0.675, + "possible_tasks": 25, + "possible_completion_rate": 1.0, + "gmv_paise": 6280212, + "discount_paise": 80230, + "negotiation_rounds": 9, + "declines": { + "precision": 1.0, + "recall": 1.0, + "impossible_tasks": 15, + "wrong_orders_on_impossible": 0, + "wrong_declines_on_possible": 0 + }, + "policy_declines": 0, + "errors": 0, + "accuracy": 1.0, + "p50_latency_ms": 1689.5, + "p95_latency_ms": 4103.8, + "by_language": { + "hi-Latn": 1.0, + "en": 1.0, + "hi": 1.0 + }, + "outcomes": { + "buyer_walked_budget": 5, + "stock": 5, + "order": 27, + "no_merchant": 3 + }, + "expected": { + "decline_budget": 5, + "decline_stock": 5, + "order": 25, + "decline_unserviceable": 4, + "decline_unknown_item": 1 + }, + "declined_checks": {}, + "errors_detail": [] + }, + "trust": { + "audit_entries": 258, + "chain_ok": true, + "merkle_root": "b843402f86485d8e529d080c12cf3907d78d3c4117ccd4a8e556d2e5ba3a799a", + "ledger": { + "entries": 9, + "distinct_rules": 8, + "inconsistencies": 0 + }, + "grants_issued": 27, + "grants_used": 27, + "explanations_present": 1.0 + }, + "baseline_no_bazaar": { + "orders": 23, + "task_to_order_rate": 0.575, + "possible_completion_rate": 0.84, + "gmv_paise": 5423008, + "definition": "same compiled catalog and index, but with negotiation OFF and a same-city-only filter, and no serviceability answers — an honest ablation of Bazaar's agent features, not a no-Bazaar world. The orders-unlocked metric below counts what this ablation cannot complete at all." + }, + "lift": { + "orders": 4, + "gmv_paise": 857204, + "gmv_multiple": 1.16, + "orders_unlocked": { + "count": 4, + "by_reason": { + "bounded_offer": 3, + "other": 1 + }, + "gmv_paise": 889487, + "vernacular_share": 0.5, + "detail": [ + { + "task_id": "t009", + "language": "en", + "baseline_outcome": "buyer_walked_budget", + "reason": "bounded_offer", + "gmv_paise": 56700 + }, + { + "task_id": "t026", + "language": "hi", + "baseline_outcome": "buyer_walked_budget", + "reason": "bounded_offer", + "gmv_paise": 42893 + }, + { + "task_id": "t035", + "language": "hi", + "baseline_outcome": "buyer_walked_budget", + "reason": "bounded_offer", + "gmv_paise": 640395 + }, + { + "task_id": "t037", + "language": "en", + "baseline_outcome": "stock", + "reason": "other", + "gmv_paise": 149499 + } + ] + } + }, + "false_positive_sweep": [ + { + "max_order_paise": 5000000, + "orders": 27, + "gmv_paise": 6280212, + "wrong_declines_on_possible": 0, + "wrong_orders_on_impossible": 0, + "lost_gmv_paise": 0, + "declined_checks": {} + }, + { + "max_order_paise": 1000000, + "orders": 25, + "gmv_paise": 2668212, + "wrong_declines_on_possible": 2, + "wrong_orders_on_impossible": 0, + "lost_gmv_paise": 3612000, + "declined_checks": { + "under_order_cap": 6 + } + }, + { + "max_order_paise": 500000, + "orders": 24, + "gmv_paise": 2027817, + "wrong_declines_on_possible": 3, + "wrong_orders_on_impossible": 0, + "lost_gmv_paise": 4252395, + "declined_checks": { + "under_order_cap": 8 + } + }, + { + "max_order_paise": 200000, + "orders": 23, + "gmv_paise": 1626167, + "wrong_declines_on_possible": 4, + "wrong_orders_on_impossible": 0, + "lost_gmv_paise": 4654045, + "declined_checks": { + "under_order_cap": 11 + } + } + ], + "elapsed_s": 573.3 +} \ No newline at end of file