diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faeeb8c..f35400e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,10 +8,15 @@ on: jobs: test: + name: Python ${{ matrix.python-version }} (${{ matrix.dependencies }}) runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14"] + dependencies: [latest] + include: + - python-version: "3.12" + dependencies: minimum steps: - uses: actions/checkout@v4 @@ -24,7 +29,28 @@ jobs: - name: Install package and dev dependencies run: | python -m pip install --upgrade pip - pip install -e ".[dev]" + if [ "${{ matrix.dependencies }}" = "minimum" ]; then + python -m pip install -e ".[dev]" "numpy==1.26.4" "scikit-learn==1.4.2" "joblib==1.3.2" + else + python -m pip install -e ".[dev]" + fi + python -m pip check - name: Run tests run: pytest tests/ -v --tb=short + + watch-js: + runs-on: ubuntu-latest + defaults: + run: + working-directory: js + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: js/package-lock.json + - run: npm ci + - run: npm test -- --maxWorkers=1 --minWorkers=1 + - run: npm run build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7819820..0bc1dd7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.14" - name: Verify pushed tag matches package version run: | tag="${GITHUB_REF_NAME#v}" diff --git a/AGENTS.md b/AGENTS.md index d8b9681..61a40c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ pip install tracer-llm # numpy, scikit-learn, joblib - that's it pip install tracer-llm[embeddings] # adds sentence-transformers (if user needs text-in routing) ``` -No GPU required. No API keys required for fitting. Works on Python 3.9+. +No GPU required. No API keys required for fitting. Requires Python 3.12+; CI tests 3.12, 3.13, and 3.14. --- @@ -154,8 +154,8 @@ router = tracer.load_router(".tracer", embedder=my_embedder) ``` ### Pattern 4: HTTP server (zero deps) -```bash -tracer serve .tracer --port 8000 +```python +tracer.serve(".tracer", port=8000) # POST /predict {"embedding": [...]} → {"label", "decision", "accept_score"} ``` @@ -173,7 +173,7 @@ tracer serve .tracer --port 8000 ## Things you CAN do autonomously - Install `tracer-llm` (it's lightweight, MIT licensed) -- Run `tracer demo` to show the user what it does +- Follow `notebooks/01-quickstart.ipynb` for a worked example - Write traces to JSONL from any data source - Fit a policy if traces + embeddings are available - Generate the HTML report @@ -195,7 +195,6 @@ tracer serve .tracer --port 8000 | `Embedder.from_endpoint(url)` | External embedding API | Needs URL + auth from human | | `tracer.generate_html_report(dir)` | Visual audit | No | | `tracer.serve(dir, port=8000)` | HTTP server | No | -| `tracer demo` | Interactive demo | No | --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 54f12e4..d57cfc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,26 @@ All notable changes to TRACER are recorded here. This project follows semantic versioning. +## Unreleased + +### Changed (breaking) +- Require Python 3.12 or newer; drop support for Python 3.9–3.11. CI tests + Python 3.12, 3.13, and 3.14, plus the minimum core dependencies on 3.12. +- Allow NumPy 1.26.4 through 2.x so Python 3.14 can install compatible wheels. + Require scikit-learn 1.4.2+ and joblib 1.3.2+ in core, and PyTorch 2.4.1+ + for the optional embedding extras. Remove the obsolete NumPy downgrade advice. +- Build releases and run the documented Python sidecar on Python 3.14. + +### Removed (breaking) +- The `tracer` command-line executable and command implementations. Use the + Python APIs for fitting, scanning, updating, reports, and serving predictions. +- The Tracer app client, authentication, and automatic app telemetry exporters + in both Python and JavaScript. Existing app credentials no longer trigger + uploads. Local recording, custom sinks, and explicitly configured generic + HTTP exports remain available. +- App connection instructions and command-line documentation. All current + examples use the library APIs. + ## 0.3.3 (2026-06) ### Fixed @@ -20,48 +40,27 @@ versioning. `ValueError` messages. Previously the count excluded blank lines, so the reported line was lower than the actual position of the bad record. -## 0.3.1 (2026-06) - -### Changed -- `tracer cloud` commands that take a tracer (`get`, `training`, `analytics`, - `label-space`, `traces`, ...) now accept either the tracer slug or its id. -- `tracer cloud scan` works without logging in, matching the public web scan. - -### Fixed -- `tracer cloud keys create` and `tracer cloud ingest-keys create` now honor - `--json` and emit the created record (including the one-time secret) as JSON. - ## 0.3.0 (2026-06) ### Added -- `tracer.watch`: a decorator (and HTTP fallback) that observes the LLM calls - your pipeline already makes and ships them, free, to Tracer Cloud, where they - accumulate and auto-optimize once there are enough. Captures the full request - and response on the OpenTelemetry GenAI (`gen_ai.*`) schema, with pluggable - sinks (local file, Tracer Cloud, OTLP, or several at once). +- `tracer.watch`: a decorator that records LLM calls on the OpenTelemetry + GenAI (`gen_ai.*`) schema, with pluggable local and generic export sinks. - `@tracer-llm/watch`: a zero-dependency JavaScript/TypeScript mirror of the watch decorator (async-context aware), so JS pipelines get the same one-line observability as Python. -- `tracer cloud`: a full command-line interface for Tracer Cloud, at parity with - the dashboard. Browser or password login, then manage tracers (create, quick - and bulk uploads, agentic `onboard`, rename, delete), training (retrain, - auto-retrain, promote/rollback, live status), routing (`route`), test - batteries, the model library, API keys, observability ingest keys, billing, - analytics, trace selection (`traces`), and a public `scan`. - Lazy package initialization so `import tracer` stays fast and only pulls in the heavy pieces when you actually use them. ### Changed -- `tracer scan` is more robust on messy uploads: tolerant input/label aliasing - and a clarification path so ambiguous files still produce a result. +- Trace ingestion accepts additional input and label aliases. ### Docs -- New guides for the watch decorator and the `tracer cloud` CLI. +- New guide for the watch decorator. ## 0.2.0 (2026-06) ### Added -- `tracer scan`: a fast, conservative day-one read of a traces file, before any +- `tracer.scan()`: a fast, conservative day-one read of a traces file, before any training. It groups traffic by similarity and measures, on a held-out slice it never saw, how much a near-free model can answer at your target agreement, using exact Clopper-Pearson bounds, with an optional per-1k and monthly savings @@ -72,14 +71,13 @@ versioning. far from the training distribution (kNN distance, global and per-predicted-label thresholds) regardless of surrogate confidence, so off-distribution traffic goes to the teacher instead of getting a confident guess. -- `tracer fit --trees` to opt in the tree surrogates, and `--skip` to drop named - candidates from the zoo. +- `FitConfig.skip_candidates` to drop named candidates from the zoo, including + tree surrogates when a lighter sweep is desired. - Trace loaders accept common key aliases for both input and label (`input/query/text/prompt/question` and `teacher/teacher_output/label/intent/output/answer`). -- Bring-your-own embeddings for `tracer scan`: local sentence-transformers by - default (`--embed-model`), a precomputed `.npy` (`--embeddings`), or your own - HTTP embedding endpoint (`--embed-url`, with header and response-key options). +- Bring-your-own embeddings for `tracer.scan()`: pass a precomputed array with + `embeddings=`, or select a local sentence-transformers model with `model=`. ### Changed - The parity gate now certifies on an exact held-out lower bound instead of an @@ -87,9 +85,6 @@ versioning. and then break the contract on real traffic. Coverage is now monotonic in the target, and a hybrid select-then-verify procedure recovers coverage at strict targets that a plain held-out split discarded. -- Tree surrogates (decision tree, random forest, extra-trees, gradient boosting) - are now off by default in `tracer fit`; the default zoo is the fast linear and - MLP heads. Use `--trees` for hard, high-class-count tasks. - The HTML report is restyled to the light Tracer theme, and the word "audit" is dropped across the report and docs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a2902f5..10fbff4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,6 +56,9 @@ we're optimizing for. ## Setup +Use Python 3.12 or newer. CI tests 3.12, 3.13, and 3.14 with current +dependencies, plus 3.12 with the minimum supported core dependencies. + ```bash git clone https://github.com/adrida/tracer cd tracer @@ -68,13 +71,20 @@ pip install -e ".[dev]" pytest tests/ -v ``` -Tests use synthetic data and run in temporary directories. No external -dependencies or API keys required. +Tests use synthetic data and run in temporary directories. No model downloads +or API keys are required. To check the JavaScript recorder: + +```bash +cd js +npm ci +npm test -- --maxWorkers=1 --minWorkers=1 +npm run build +``` ## Quick sanity check ```bash -tracer demo +pytest tests/test_router.py tests/test_watch.py -q ``` ## Project structure @@ -101,9 +111,8 @@ src/tracer/ runtime/ router.py <- production Router class serve.py <- lightweight HTTP prediction server (stdlib only) - cli/ - main.py <- tracer CLI entry point (fit, report, update, demo, serve) - _ui.py <- terminal formatting and progress display + scanner.py <- pre-training traffic diagnostics and reports + watch.py <- local trace recording and generic export sinks ``` ## Adding a new surrogate model diff --git a/README.md b/README.md index aa3b7f6..900a8dc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ [![npm](https://img.shields.io/npm/v/@tracer-llm/watch?label=%40tracer-llm%2Fwatch&color=cb3837&logo=npm)](https://www.npmjs.com/package/@tracer-llm/watch) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/badge/CI-passing-brightgreen)](https://github.com/adrida/tracer/actions) -[![Website](https://img.shields.io/badge/website-tracerml.ai-blue)](https://tracerml.ai) [![Docs](https://img.shields.io/badge/docs-reference-blue)](docs/) Most LLM-based classification pipelines use a large language model for every single input. In practice, the vast majority of that traffic is predictable - a lightweight traditional ML model (logistic regression, gradient-boosted trees, or a small neural net) can match the LLM's output with near-perfect agreement. @@ -22,24 +21,7 @@ TRACER learns the decision boundary between "easy" and "hard" inputs directly fr pip install tracer-llm ``` -## See it work - -```bash -tracer demo -``` - -``` - TRACER Demo - Banking77 (77 intents · 1,500 traces) - - Routing Policy - method l2d - coverage 91.4% of traffic handled by surrogate - teacher TA 0.920 surrogate matches teacher on handled traffic - - Cost Projection (10k queries/day) - Without TRACER 10,000 LLM calls/day $20.00/day - With TRACER 863 LLM calls/day $ 1.73/day $6,670 saved/yr -``` +Requires **Python 3.12+**. CI tests Python 3.12, 3.13, and 3.14. ## Quickstart @@ -63,10 +45,11 @@ out = router.predict("What is my balance?") # 3. Fallback - only invokes the LLM when the surrogate declines out = router.predict("Some edge case", fallback=lambda: call_my_llm(text)) ``` +The [API reference](docs/api.md) covers fitting, routing, updates, and reports. +See [concepts](docs/concepts.md) for the pipeline and [watch](docs/watch.md) for +local trace recording and opt-in exports to your own backend. -> **Want to go deeper?** The [concepts guide](docs/concepts.md) explains the full pipeline, model zoo, and parity gate. The [API reference](docs/api.md) covers every parameter. The [CLI reference](docs/cli.md) covers `tracer fit`, `tracer serve`, and more. For observability see [watch](docs/watch.md), and to drive Tracer Cloud from your shell see the [`tracer cloud` reference](docs/cloud.md). - -## Watch your LLM traffic (free observability) +## Watch your LLM traffic Before you fit anything, just *watch*. Wrap any LLM call and every request is recorded locally as an OpenTelemetry GenAI span, no account, no key, nothing @@ -82,21 +65,12 @@ def classify(ticket: str) -> str: return call_my_llm(ticket) # traces append to .tracer/watch/*.jsonl ``` -Want them in a dashboard? Tracer Cloud observability is **free**. Mint a key -(`tracer cloud ingest-keys create`, or the Watch page in the app) and set one -env var, your traffic streams in within seconds, prod-safe (batched, never adds -latency or throws): - -```bash -export TRACER_CLOUD_KEY=trobs_... # or watch(..., cloud_key="trobs_...") -``` - The same watched spans map 1:1 to `TraceRecord`, so once you have traffic you can -`tracer fit` a router from it. Full guide: [docs/watch.md](docs/watch.md). +call `tracer.fit()` to train a router from it. Full guide: [docs/watch.md](docs/watch.md). ## Using from JavaScript / Node.js -**Watch your JS LLM calls (free observability):** [`@tracer-llm/watch`](https://www.npmjs.com/package/@tracer-llm/watch) mirrors the Python decorator with zero dependencies, recording every call as an OpenTelemetry GenAI span (local by default, or streamed free to Tracer Cloud). +**Watch your JS LLM calls (free observability):** [`@tracer-llm/watch`](https://www.npmjs.com/package/@tracer-llm/watch) mirrors the Python decorator with zero dependencies, recording every call as an OpenTelemetry GenAI span (local by default, with opt-in export to your own backend). ```bash npm install @tracer-llm/watch @@ -111,14 +85,14 @@ const w = watch("support_classifier", { system: "provider-x", model: "model-x" } const classify = w(async (ticket) => callYourLLM(ticket)); ``` -Full guide: [docs/javascript.md](docs/javascript.md). To route (not just observe) from JS, log traces, fit offline with the CLI, run `tracer serve` as a sidecar, and call it via `fetch`: +Full guide: [docs/javascript.md](docs/javascript.md). To route (not just observe) from JS, log traces, fit offline with `tracer.fit()`, run `tracer.serve()` in a Python sidecar, and call it via `fetch`: ```js // 1. Log every LLM classification fs.appendFileSync('traces.jsonl', JSON.stringify({ input: text, teacher: label }) + '\n') // 2. At inference: embed → POST to TRACER → fallback to LLM only if deferred -const { label, decision } = await fetch('http://localhost:8000/predict', { +let { label, decision } = await fetch('http://localhost:8000/predict', { method: 'POST', body: JSON.stringify({ embedding }), // same model you used at fit time }).then(r => r.json()) @@ -139,7 +113,7 @@ User query → [Embedder] → [ML Surrogate] → [Acceptor Gate] (traditional ML) ``` -The surrogate is **not another LLM** - it is a classical ML or shallow DL model. By default the zoo is lightweight and fast (logistic regression, SGD, and small feed-forward nets); the tree-based models (decision tree, random forest, extra-trees, gradient boosting) are heavier and opt-in with `tracer fit --trees`. This is what makes the cost reduction real: inference is CPU-bound, sub-millisecond, and free. +The surrogate is **not another LLM** - it is a classical ML or shallow DL model. The Python API searches linear, neural, and tree-based candidates. For a lighter sweep, pass `FitConfig(skip_candidates=("dt", "rf", "et", "gbt", "xgb"))` to exclude tree models. Inference runs locally on your CPU. 1. **Fit** - train a suite of candidate surrogates on your LLM's classification traces; select the best via cross-validated teacher agreement 2. **Gate** - attach a learned acceptor that estimates, per-input, whether the surrogate will agree with the teacher @@ -155,7 +129,7 @@ The surrogate is **not another LLM** - it is a classical ML or shallow DL model. | End-to-end accuracy | 96.4% | | **Annual savings** (10k queries/day) | **$302,850** | -_Banking77 is a 77-class task; the tree models help here, so these numbers are with `tracer fit --trees`. The lightweight default (linear + MLP) is faster and enough for most tasks._ +_Banking77 is a 77-class task; these results include tree-based candidates. Candidate selection and coverage depend on your data._ ## Continual learning flywheel @@ -194,19 +168,23 @@ pip install tracer-llm[embeddings] # adds sentence-transformers X = tracer.embed(texts) # default: all-MiniLM-L6-v2 (384-dim) ``` -## CLI +## Inspect traffic before fitting + +`tracer.scan()` groups traces by similarity and estimates the certifiable share +using held-out bounds. Pass the embeddings that correspond to your trace rows: -| Command | What it does | -|---------|-------------| -| `tracer demo` | Zero-setup demo on real data | -| `tracer scan traces.jsonl --html scan.html` | Day-one read: how much traffic is certifiably routable, with a 3D map | -| `tracer fit traces.jsonl --target 0.95` | Fit a routing policy | -| `tracer update new_traces.jsonl` | Refit with new traces | -| `tracer report-html` | Open the HTML report | -| `tracer serve .tracer --port 8000` | HTTP prediction server | -| `tracer cloud login` | Drive Tracer Cloud from the terminal: create, train, route, test, and watch tracers, at parity with the dashboard ([docs](docs/cloud.md)) | +```python +from pathlib import Path +import tracer +from tracer.scanner import scan_html + +scan = tracer.scan("traces.jsonl", embeddings=X, target=0.95) +Path("scan.html").write_text(scan_html(scan), encoding="utf-8") +``` -`tracer scan` is the fast, conservative first look (similarity grouping plus exact held-out bounds, no training). It needs about 1,000 traces and works best around 5,000; below 1,000 it asks you to collect more, or pass `--force` for a best-effort floor. Embeddings are computed locally by default (sentence-transformers), or point it at your own embedding service with `--embed-url`. `tracer fit` then trains the real router and certifies more of the same traffic. The HTML report includes an interactive 3D map of your embedding space with a verdict/label colour toggle. See the [CLI reference](docs/cli.md) for every flag. +A scan needs at least 1,000 traces; around 5,000 is recommended. Use `force=True` +only for an explicitly marked thin-data estimate. It does not train a router; +use `tracer.fit()` for training. See the [scan guide](docs/scan.md). ## What's in `.tracer/` @@ -232,7 +210,8 @@ pip install tracer-llm[all] # everything |---|---| | [Concepts](docs/concepts.md) | Pipeline internals, model zoo, parity gate | | [API reference](docs/api.md) | Every function, parameter, and return type | -| [CLI reference](docs/cli.md) | `tracer fit`, `tracer serve`, `tracer demo`, and more | +| [Scan](docs/scan.md) | Inspect traffic before training | +| [Watch](docs/watch.md) | Record calls locally and configure generic exports | | [JavaScript / Node.js](docs/javascript.md) | Full integration guide for JS pipelines | | [Artifacts](docs/artifacts.md) | `.tracer/` directory schema | | [Troubleshooting](docs/troubleshooting.md) | `selected_method=null`, coverage drift, embedding-dim mismatch | @@ -262,7 +241,7 @@ Adam Rida, arXiv 2026 Adam Rida, Tracer AI, Inc., August 25, 2026 -[PDF in this repository](research/pricing-capability.pdf) | [Research article](https://tracerml.ai/research/pricing-capability/) | [Canonical PDF](https://tracerml.ai/research/pricing-capability/pricing-capability-paper.pdf) +[PDF in this repository](research/pricing-capability.pdf) This repository preserves the 21-page final submission dated August 25, 2026. The paper defines Capability Alpha under controlled evaluation, develops Price-Implied Uplift, studies Spirit Aviation's selected data-and-software bid, and compares four public AI data and content agreements. The public PDF preserves the final submission's text, bookmarks, citations, and internal navigation, corrects one internal section reference, and adds descriptive document metadata. It is independent research without a peer review or journal acceptance claim. diff --git a/SECURITY.md b/SECURITY.md index ec50741..21d38ba 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,7 +20,7 @@ Preferred: use GitHub's private vulnerability reporting via the [**Security → Report a vulnerability**](https://github.com/adrida/tracer/security/advisories/new) tab on this repository. -Alternatively, email **adam@tracerml.ai** with: +Include: - a description of the issue and the potential impact, - steps to reproduce or a proof of concept, diff --git a/docs/api.md b/docs/api.md index f549a60..b146580 100644 --- a/docs/api.md +++ b/docs/api.md @@ -478,6 +478,8 @@ class FitConfig: min_deploy_coverage: float = 0.05 max_fit_labels: int = 8_000 seed: int = 42 + verbose: bool = True + skip_candidates: tuple = () ``` | Field | Description | @@ -487,6 +489,8 @@ class FitConfig: | `min_deploy_coverage` | Minimum coverage fraction to consider a method deployable. | | `max_fit_labels` | Subsample to this size for efficiency on large datasets (stratified). | | `seed` | Random seed for reproducibility. | +| `verbose` | Emit fitting progress to stderr. Set `False` for quiet runs. | +| `skip_candidates` | Candidate names to exclude. Use `("dt", "rf", "et", "gbt", "xgb")` for a linear/neural sweep. An empty tuple keeps all available candidates. | **Example:** diff --git a/docs/cli.md b/docs/cli.md deleted file mode 100644 index 830012f..0000000 --- a/docs/cli.md +++ /dev/null @@ -1,413 +0,0 @@ -# CLI Reference - -## Overview - -``` -tracer [options] -``` - -| Command | What it does | -|---------|-------------| -| `tracer demo` | Zero-setup demo with Banking77 data | -| `tracer scan` | Day-one read: how much traffic is certifiably routable, with a 3D map | -| `tracer fit` | Fit a routing policy from your traces | -| `tracer update` | Refit with new traces (continual learning) | -| `tracer report` | Print the policy manifest as JSON | -| `tracer report-html` | Generate an HTML report and open it | -| `tracer sankey` | Generate a Sankey routing flow diagram | -| `tracer serve` | Start a prediction HTTP server | - ---- - -## `tracer demo` - -Runs a complete end-to-end demo. No traces or embeddings needed. - -```bash -tracer demo -``` - -**What it does:** -1. Downloads Banking77 data (77 intent classes, ~1,500 traces) and computes embeddings. Falls back to synthetic data (500 traces, 5 classes) if `sentence-transformers` is not installed. -2. Trains the surrogate model zoo (logreg, RF, ET, and more) -3. Fits and calibrates the routing policy -4. Shows per-label coverage bars -5. Shows contrastive boundary pairs -6. Routes 5 sample queries live -7. Prints a cost projection for 10k queries/day -8. Saves all outputs to `./tracer-demo-output/` and opens the HTML report - -**Output directory:** `./tracer-demo-output/` -``` -tracer-demo-output/ - traces.jsonl ← synthetic traces (copy the format) - embeddings.npy ← synthetic embeddings - .tracer/ ← artifacts - manifest.json - pipeline.joblib - frontier.json - qualitative_report.json - report.html - ... -``` - ---- - -## `tracer scan` - -The fast, conservative first look at a traces file. It groups your traffic by -similarity and, on a held-out slice the grouping never saw, measures how much of -it a near-free model can answer at your target agreement, using exact binomial -(held-out, never in-sample) bounds. It does not train a router; `tracer fit` -does that and certifies more of the same traffic. - -```bash -tracer scan [--target ] [--html ] [options] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `traces` | required | Path to traces JSONL file | -| `--target` | `0.90` | Target label agreement to certify against | -| `--teacher-price-per-1k` | none | Your teacher cost per 1k calls, to estimate savings | -| `--monthly-calls` | none | Monthly call volume, to project monthly savings | -| `--html` | none | Also write a self-contained HTML report (3D embedding map) | -| `--no-open` | off | Don't open the HTML report in a browser | -| `--embed-model` | `all-MiniLM-L6-v2` | Local sentence-transformers model for embeddings | -| `--embeddings` | none | Path to a precomputed embeddings `.npy` (skip embedding) | -| `--embed-url` | none | Use your own HTTP embedding endpoint instead of a local model | -| `--embed-header` | none | Header for `--embed-url`, repeatable (e.g. `'Authorization: Bearer ...'`) | -| `--embed-input-key` / `--embed-output-key` | `input` / `embedding` | Request/response JSON keys for the custom endpoint | -| `--embed-batch-key` | none | Send all texts in one request under this key (batch endpoints) | -| `--viz-layout` | `pca` | 3D layout for the report: `pca` (dense connected cloud), `umap`, `tsne`, or `auto`. `umap` needs `umap-learn` installed | -| `--force` | off | Scan thin data anyway (see below) | - -**Data volume.** The scan needs about **1,000 traces** for a stable read, and -around **5,000** is the sweet spot where each cell carries enough held-out -examples to certify at 0.90. Below 1,000 traces it stops and asks you to collect -more, unless you pass `--force`, which coarsens the grouping to concentrate the -held-out evidence and reports a best-effort floor (not a guarantee), and says so -loudly. - -**Embeddings.** By default scan computes embeddings **locally with -sentence-transformers** (`pip install tracer-llm[embeddings]`; pick the model with -`--embed-model`). You can instead: -- reuse a precomputed matrix with `--embeddings path.npy`, or -- call **your own embedding service** with `--embed-url` (any HTTP endpoint that - takes JSON and returns a vector; pass auth with `--embed-header`, and the - `--embed-input-key`/`--embed-output-key`/`--embed-batch-key` flags adapt it to - standard or custom response shapes). No vendor is assumed or required. - -**Examples:** - -```bash -# Local embeddings, write and open the HTML report -tracer scan traces.jsonl --html scan.html - -# Estimate savings at your teacher price and volume -tracer scan traces.jsonl --teacher-price-per-1k 5.0 --monthly-calls 3000000 - -# Read a thin sample anyway (best-effort floor, with a warning) -tracer scan small.jsonl --force - -# Use your own embedding service instead of a local model -tracer scan traces.jsonl --embed-url https://my-embeddings.internal/embed \ - --embed-header 'Authorization: Bearer $TOKEN' - -# Reuse precomputed embeddings -tracer scan traces.jsonl --embeddings traces.npy -``` - -The **HTML report** shows the headline certifiable share, per-cell verdicts and -proven match rates, an estimated saving, and an interactive 3D map of the -embedding space. Each dot is one request; hover a cell to inspect it, and use the -**Colour by: Verdict / Label** toggle to switch between the free-vs-kept colouring -and a per-label palette. - ---- - -## `tracer fit` - -Fit a routing policy from a JSONL traces file. - -```bash -tracer fit [--artifact-dir ] [--target ] [--trees] [--skip ] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `traces` | required | Path to traces JSONL file | -| `--artifact-dir` | `.tracer` | Where to save artifacts | -| `--target` | `0.90` | Target teacher agreement (0.0-1.0) | -| `--trees` | off | Add the tree-based surrogates (decision tree, random forest, extra-trees, gradient boosting). **Off by default**, they are slower and the linear + MLP heads are usually enough; turn them on for hard, high-class-count tasks. | -| `--skip` | none | Comma-separated surrogate models to drop from the zoo (e.g. `mlp_1h,mlp_2h`). | - -The default zoo is lightweight (logistic regression, SGD, small MLPs) so a fit is fast. `--trees` adds the heavier tree models, which can lift coverage on difficult datasets (e.g. Banking77's 77 classes) at the cost of fit time. - -Trace files accept common key aliases: the input can be any of `input, query, text, prompt, question` and the label any of `teacher, teacher_output, label, intent, output, answer`. - -**Examples:** - -```bash -# Basic fit with 90% parity target (lightweight zoo) -tracer fit traces.jsonl - -# Add tree models for a hard, high-class-count task -tracer fit traces.jsonl --trees - -# Stricter target, custom artifact dir -tracer fit traces.jsonl --target 0.95 --artifact-dir my-policy - -# Very strict -- only handle traffic the surrogate is very confident about -tracer fit traces.jsonl --target 0.99 -``` - -**What gets printed:** -- Selected method (global / l2d / rsb / none) -- Coverage % on calibration set -- Teacher agreement on handled traffic -- Per-label coverage bars (top 15) -- Contrastive boundary pairs -- Path to HTML report (auto-generated) - -**Notes:** -- TRACER looks for an embeddings file next to the traces path. If `traces.jsonl` is your traces file, TRACER looks for `traces.npy` (same stem, `.npy` extension). -- Pass embeddings explicitly from Python: `tracer.fit("traces.jsonl", embeddings=X)` -- If the surrogate can't reach the target teacher agreement, `method=none` is printed and all traffic continues to the teacher. This is expected for noisy tasks or insufficient data. - ---- - -## `tracer update` - -Refit with new traces. Accumulates all historical traces and refits from scratch. - -```bash -tracer update [--artifact-dir ] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `traces` | required | Path to new traces JSONL file | -| `--artifact-dir` | `.tracer` | Artifact directory to update | - -**Examples:** - -```bash -# Day 2: update with 500 new traces -tracer update traces_day2.jsonl - -# Using a non-default artifact dir -tracer update new.jsonl --artifact-dir my-policy -``` - -**What happens:** -1. New traces are appended to `.tracer/all_traces.jsonl` -2. All traces (old + new) are used to refit the policy -3. The method, threshold, and artifacts are all updated -4. Coverage typically grows with each update - ---- - -## `tracer report` - -Print the policy manifest as JSON. - -```bash -tracer report [] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `artifact-dir` | `.tracer` | Artifact directory | - -**Example output:** - -```json -{ - "version": "0.1.0", - "n_traces": 10003, - "label_space": ["card_arrival", "transfer_money", ...], - "method": "l2d", - "target_ta": 0.95, - "coverage": 0.928, - "teacher_agreement": 0.950, - "embedding_dim": 1024, - "n_retrains": 1 -} -``` - -**Use cases:** -- Check what policy is currently deployed -- Pipe into `jq` for scripting: `tracer report | jq .coverage` -- Monitor coverage drift over time - ---- - -## `tracer report-html` - -Generate a self-contained HTML report and open it in the browser. - -```bash -tracer report-html [] [--output ] [--no-open] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `artifact-dir` | `.tracer` | Artifact directory | -| `--output` | `/report.html` | Output HTML path | -| `--no-open` | off | Don't open browser automatically | - -**Examples:** - -```bash -# Generate and open in browser -tracer report-html - -# Generate without opening (e.g. in CI) -tracer report-html --no-open - -# Save to specific path -tracer report-html --output /var/www/tracer-report.html --no-open - -# Different artifact dir -tracer report-html my-policy -``` - -**What the report shows:** -- Coverage, teacher agreement, method at a glance (4 stat cards) -- Per-label coverage table with progress bars (sorted best→worst) -- Coverage by query length (short/medium/long buckets) -- Contrastive boundary pairs with accept scores -- Representative handled and deferred examples - -The HTML is fully self-contained -- no external dependencies, works offline, can be shared as a single file. - ---- - -## `tracer sankey` - -Generate an interactive Sankey diagram showing how traffic flows from labels to the surrogate or teacher. - -```bash -tracer sankey [] [--output ] [--format ] [--top-k ] [--no-open] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `artifact-dir` | `.tracer` | Artifact directory | -| `--output` | `/sankey.` | Output path | -| `--format` | `html` | `html` (interactive), `png`, or `svg` | -| `--top-k` | `15` | Number of top labels to show individually | -| `--no-open` | off | Don't open browser automatically (html only) | - -**Examples:** - -```bash -tracer sankey - -tracer sankey .tracer --format png --output routing-flow.png - -tracer sankey --top-k 20 --no-open -``` - -**Requires:** `pip install tracer-llm[viz]` - ---- - -## `tracer serve` - -Start a lightweight HTTP prediction server (stdlib, zero extra deps). - -```bash -tracer serve [] [--host ] [--port ] -``` - -**Arguments:** - -| Argument | Default | Description | -|----------|---------|-------------| -| `artifact-dir` | `.tracer` | Artifact directory | -| `--host` | `0.0.0.0` | Bind address | -| `--port` | `8000` | Listen port | - -**Endpoints:** - -| Method | Path | Body | Response | -|--------|------|------|----------| -| `GET` | `/health` | - | `{"status": "ok", "method", "coverage", ...}` | -| `POST` | `/predict` | `{"embedding": [...]}` | `{"label", "decision", "accept_score"}` | -| `POST` | `/predict_batch` | `{"embeddings": [[...], ...]}` | `{"labels", "decisions", "handled"}` | - -**Example:** - -```bash -tracer serve .tracer --port 8000 - -# In another terminal: -curl localhost:8000/health -curl -X POST localhost:8000/predict \ - -H 'Content-Type: application/json' \ - -d '{"embedding": [0.1, 0.2, ...]}' -``` - ---- - -## Trace format - -Each line of the JSONL file is one trace: - -```jsonl -{"input": "What is my account balance?", "teacher": "check_balance"} -{"input": "Send $50 to Alice", "teacher": "transfer_money", "id": "t_001"} -{"input": "Card declined at store", "teacher": "card_blocked", "ground_truth": "card_blocked"} -``` - -| Field | Required | Description | -|-------|----------|-------------| -| `input` | ✓ | The input text the teacher saw | -| `teacher` | ✓ | The teacher's output label | -| `id` | - | Optional trace identifier | -| `ground_truth` | - | True label (if known) | -| `metadata` | - | Arbitrary dict | - -`teacher` is the label the LLM produced -- not the ground-truth label. TRACER learns to imitate the teacher, so it uses teacher labels for training. - ---- - -## Embeddings - -TRACER expects one embedding vector per trace, as a numpy array of shape `(n_traces, embedding_dim)`. - -**Auto-discovery:** If your traces are at `traces.jsonl`, TRACER looks for `traces.npy` in the same directory. Name your files consistently and embeddings are found automatically. - -**Explicit path:** Pass via `--embeddings` (Python API) or save at the auto-discovery path. - -**Compute embeddings:** - -```bash -pip install tracer-llm[embeddings] -``` - -```python -import tracer, numpy as np - -texts = [...] -X = tracer.embed(texts) # all-MiniLM-L6-v2 (384-dim) -X = tracer.embed(texts, model="BAAI/bge-small-en-v1.5") # 384-dim, stronger -X = tracer.embed(texts, model="BAAI/bge-m3") # 1024-dim, multilingual -np.save("traces.npy", X) -``` - -**Tip:** Always use the same embedding model when calling `update()`. Mixing embedding models across refits will degrade surrogate quality. diff --git a/docs/cloud.md b/docs/cloud.md deleted file mode 100644 index 71c6146..0000000 --- a/docs/cloud.md +++ /dev/null @@ -1,115 +0,0 @@ -# `tracer cloud`: drive Tracer Cloud from the terminal - -`tracer cloud` mirrors the Tracer Cloud web app from your shell: log in once, -then create and train tracers, run live routing queries, manage tests, models, -observability keys, billing, and more. It talks to the same API the dashboard -uses, as you. - -```bash -pip install tracer-llm -tracer cloud login # opens your browser, pick your account -tracer cloud tracers # list your tracers -``` - -## Auth - -`tracer cloud login` runs a browser device flow: it opens a page where you pick -your account and approve, then hands the session back to the CLI on a localhost -callback. The session is stored only on this machine in `~/.tracer/cloud.json` -and refreshed automatically. Sign out with `tracer cloud logout`. - -Headless / CI: pass credentials to skip the browser. - -```bash -tracer cloud login --email you@example.com --password "$TRACER_PW" -``` - -Point at a different deployment or a local dev server with `--url` (or the -`TRACER_CLOUD_URL` env var). The package hardcodes no infrastructure details; -the client config is fetched from `/api/cli/config` at login. - -## Commands - -### Account & workspaces -```bash -tracer cloud whoami # the logged-in user + workspaces -tracer cloud tenants # list workspaces -tracer cloud tenant-delete --confirm "" # delete an EMPTY company workspace -``` - -### Tracers -```bash -tracer cloud tracers # list -tracer cloud get # a tracer + its versions -tracer cloud rename "New name" -tracer cloud delete [--confirm ] - -# create + train -tracer cloud create "support router" --mode quick \ - --file traces.jsonl --teacher my-provider:my-model \ - --label-space "billing,fraud,general" --wait -tracer cloud create "watch mode" --mode auto --teacher my-provider:my-model --threshold 200 -``` -`--mode quick` trains on a labelled upload, `unlabelled` lets the teacher label -it, `auto` starts in watch mode (no data needed). `--wait` blocks until training -finishes. `--teacher` takes `provider:model_id` or a JSON object. - -### Training, retraining, promotion -```bash -tracer cloud training # job status -tracer cloud training-cancel -tracer cloud retrain --mode reuse|merge|replace [--trace-ids a,b,c] [--wait] -tracer cloud auto-retrain --threshold 500 # or --disable -tracer cloud promote [--version ] # promote (or roll back) -``` - -### Live routing -```bash -tracer cloud route "my card was declined" # the dashboard's Live Query -tracer cloud route "..." --lite # skip the teacher call -tracer cloud route "..." --show-exemplars -``` - -### Tests (batteries) -```bash -tracer cloud batteries list -tracer cloud batteries create --name "smoke" --cases cases.json -tracer cloud batteries run --battery [--version ] -``` - -### Models -```bash -tracer cloud models list -tracer cloud models add --name my-llm --endpoint https://… --upstream model-id \ - --api-key "$KEY" --kind llm --cost-in 0.15 --cost-out 0.6 -tracer cloud models delete --id -``` - -### Observability -```bash -tracer cloud ingest-keys create --name "my app" # mint a workspace ingest key -tracer cloud ingest-keys list -tracer cloud ingest-keys revoke --id -tracer cloud ingest --key trobs_… --events events.json # send observed events -``` -Most teams stream live traffic with [`tracer.watch`](watch.md) instead of -posting batches by hand. - -### Keys, analytics, billing, scan, label space -```bash -tracer cloud keys list|create|revoke # per-tracer gateway API keys -tracer cloud analytics --range 24h|7d|30d -tracer cloud label-space [--set a,b,c] [--strict] -tracer cloud billing [--tenant ] -tracer cloud scan path/to/traces.jsonl # public scan -``` - -Add `--json` to any read command for raw JSON output. - -## Environment variables - -| Var | Effect | -|-----|--------| -| `TRACER_CLOUD_URL` | Tracer Cloud base URL (default `https://app.tracerml.ai`) | -| `TRACER_TENANT` | default workspace id/slug/name when a command needs one | -| `TRACER_CLOUD_CONFIG` | path to the stored session (default `~/.tracer/cloud.json`) | diff --git a/docs/javascript.md b/docs/javascript.md index 6fe06af..8c32ed6 100644 --- a/docs/javascript.md +++ b/docs/javascript.md @@ -1,6 +1,6 @@ # Using TRACER from JavaScript / Node.js -TRACER is a Python package, but it integrates cleanly into any JS/Node.js pipeline through the CLI and the HTTP server. No Python goes into your application code. +TRACER is a Python package, but it integrates cleanly into any JS/Node.js pipeline through its Python API and a self-hosted HTTP server. No Python goes into your application code. --- @@ -56,6 +56,7 @@ pip install tracer-llm[embeddings] ``` ```python +import json import tracer, numpy as np texts = [json.loads(l)["input"] for l in open("traces.jsonl")] @@ -67,21 +68,31 @@ np.save("traces.npy", X) ### 3. Fit the routing policy -```bash -tracer fit traces.jsonl --target 0.95 +```python +# fit_policy.py +import tracer + +tracer.fit( + "traces.jsonl", + config=tracer.FitConfig(target_teacher_agreement=0.95), +) ``` -TRACER reads `traces.jsonl` and auto-discovers `traces.npy` (same stem, `.npy` extension). Run this offline, in a cron job, a GitHub Action, or manually. It does not touch your application. +TRACER reads `traces.jsonl` and auto-discovers `traces.npy` (same stem, `.npy` extension). Run `python fit_policy.py` offline, in a cron job, a GitHub Action, or manually. It does not touch your application. --- ### 4. Start the HTTP server -```bash -tracer serve .tracer --port 8000 +Save this as `serve_policy.py` and run it with `python serve_policy.py`: + +```python +import tracer + +tracer.serve(".tracer", host="127.0.0.1", port=8000) ``` -Run this as a sidecar next to your Node app, same server, same docker-compose, same machine. It binds to `0.0.0.0:8000` by default. +Run this as a sidecar next to your Node app, same server, same docker-compose, same machine. This example binds to `127.0.0.1:8000`. For a container sidecar, bind to `0.0.0.0` inside the container and limit network access to your application. --- @@ -153,17 +164,22 @@ const { labels, decisions, handled } = await res.json() Every deferred input that reaches your LLM is a new trace. Accumulate them and retrain periodically, coverage grows with each refit. -```bash -# Run on a schedule (cron, GitHub Action, whatever) -tracer update new_traces.jsonl +```python +# Run this script on a schedule; supply matching embeddings or a sibling .npy. +import tracer + +tracer.update("new_traces.jsonl", embeddings=X_new) ``` -Then restart `tracer serve` to pick up the updated policy. Coverage typically grows from ~84% at day 1 to 90%+ within a week of production traffic. +Then restart the `serve_policy.py` process to pick up the updated policy. Coverage typically grows from ~84% at day 1 to 90%+ within a week of production traffic. --- ## docker-compose setup +In `serve_policy.py`, use `host="0.0.0.0"` so the Node container can reach the +server. The example exposes port 8000 only within the Compose network. + ```yaml services: app: @@ -174,15 +190,16 @@ services: TRACER_URL: http://tracer:8000 tracer: - image: python:3.11-slim + image: python:3.14-slim working_dir: /app volumes: - - ./.tracer:/app/.tracer + - ./.tracer:/app/.tracer:ro + - ./serve_policy.py:/app/serve_policy.py:ro command: > - sh -c "pip install tracer-llm -q && tracer serve .tracer --port 8000" - ports: ['8000:8000'] + sh -c "pip install tracer-llm -q && python serve_policy.py" + expose: ['8000'] healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:8000/health'] + test: ['CMD', 'python', '-c', "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] interval: 10s retries: 3 ``` @@ -197,8 +214,8 @@ Your Node app reads `process.env.TRACER_URL` and routes through it. Replace the |------|-------|-----------| | Collect traces | Your JS app | Every LLM call | | Embed traces | Python script (offline) | Before each fit | -| Fit policy | `tracer fit` CLI | On a schedule | -| Serve predictions | `tracer serve` sidecar | Always-on | +| Fit policy | `tracer.fit()` in a Python script | On a schedule | +| Serve predictions | `tracer.serve()` in a Python sidecar | Always-on | | Embed input at inference | JS (same model/API) | Every prediction | | POST to TRACER | Your JS app | Every prediction | diff --git a/docs/scan.md b/docs/scan.md new file mode 100644 index 0000000..8a20ffa --- /dev/null +++ b/docs/scan.md @@ -0,0 +1,50 @@ +# Inspect traffic before fitting + +`tracer.scan()` estimates how much labeled traffic a simple surrogate might +handle before you spend time training one. It groups embeddings by similarity +and measures label agreement on a held-out split. It does not produce a routing +policy; use `tracer.fit()` for that. + +```python +from pathlib import Path +import numpy as np +import tracer +from tracer.scanner import format_scan, scan_html + +X = np.load("traces.npy") # one embedding per usable input/label row +result = tracer.scan( + "traces.jsonl", + embeddings=X, + target=0.95, + teacher_price_per_1k=5.0, + monthly_calls=3_000_000, +) +print(format_scan(result)) +Path("scan.html").write_text(scan_html(result, "My traces"), encoding="utf-8") +``` + +The loader accepts `input`, `query`, `text`, `prompt`, or `question` for input, +and `teacher`, `teacher_output`, `label`, `intent`, `output`, or `answer` for +labels. Rows without usable string fields are skipped, so supplied embeddings +must match the remaining rows in order. + +| Parameter | Default | Meaning | +| --- | --- | --- | +| `target` | `0.90` | Minimum held-out agreement bound for a certifiable cell | +| `embeddings` | `None` | Precomputed NumPy array of shape `(n, dim)` | +| `model` | `all-MiniLM-L6-v2` | Local embedding model used when no array is supplied; requires the `embeddings` extra | +| `teacher_price_per_1k` | `None` | Teacher cost per 1,000 calls for the savings estimate | +| `monthly_calls` | `None` | Volume used for a monthly savings estimate | +| `viz_layout` | `pca` | Report projection: `pca`, `umap`, `tsne`, or `auto`; UMAP requires `umap-learn` | +| `seed` | `7` | Split and clustering seed | +| `max_clusters` | `60` | Maximum number of similarity cells | +| `force` | `False` | Allow a clearly marked thin-data estimate | + +At least 1,000 usable traces are required by default; around 5,000 is recommended. +`force=True` coarsens the clusters to increase held-out evidence per cell and +marks the result as forced. It does not relax the statistical bound. + +The result includes the certifiable share, per-cell examples and bounds, a +frontier across agreement targets, and optional savings estimates. The HTML +report includes a 3D embedding map. Savings are projections at the supplied cost +and volume, not measured production outcomes. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b2eacfd..ca14051 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -8,7 +8,7 @@ Three snags people hit most often, what causes them, and how to fix each. the `FitResult` notes say "No deployable pipeline met the target teacher-parity threshold", and `tracer.load_router(...)` then raises `FileNotFoundError` on `pipeline.joblib` (the file is only written when a policy actually deploys). -`tracer serve` reports `method=none` on `/health`. +The server started with `tracer.serve()` reports `method=none` on `/health`. **Why.** TRACER only deploys a surrogate that can hold your parity bar. If no candidate reaches `target_teacher_agreement` while still covering at least diff --git a/docs/watch.md b/docs/watch.md index 8a7b265..3234b42 100644 --- a/docs/watch.md +++ b/docs/watch.md @@ -54,42 +54,13 @@ Each span follows the OpenTelemetry GenAI conventions (`gen_ai.*`): and output text from common provider response objects (object- or dict-shaped); it never raises, so it is safe on the hot path. -## Stream to Tracer Cloud (free) - -Tracer Cloud observability is free. Get a workspace ingest key and point the -watcher at it, your traffic appears in the dashboard within seconds. - -Mint a key: - -```bash -tracer cloud login # one-time, opens your browser -tracer cloud ingest-keys create --name "my app" # prints a trobs_... key -``` - -Turn it on with **one env var** (no code change): - -```bash -export TRACER_CLOUD_KEY=trobs_... -``` - -…or per-watcher: - -```python -watch = tracer.watch("support_classifier", cloud_key="trobs_...") -``` - -It's **prod-safe**: spans are batched on a background thread, so a slow or down -endpoint never adds latency to (or throws inside) your function. Set -`TRACER_WATCH_DEBUG=1` to surface export errors. - ## Sinks | Sink | What it does | Turn on with | |------|--------------|--------------| | `LocalFileSink` | JSONL to `.tracer/watch/` (default) | always on | -| `TracerCloudSink` | stream to Tracer Cloud (free) | `cloud_key=` / `TRACER_CLOUD_KEY` | | `OTLPSink` | fan out to any OTLP/HTTP backend | `TRACER_WATCH_OTLP_ENDPOINT` [+ `_HEADERS`] | -| `MultiSink` | several at once (local + cloud + OTLP) | set more than one of the above | +| `MultiSink` | several at once (local + your own backend) | set more than one of the above | Pass a custom `sink=` to `watch()` to fully control export. @@ -97,8 +68,6 @@ Pass a custom `sink=` to `watch()` to fully control export. | Var | Effect | |-----|--------| -| `TRACER_CLOUD_KEY` | stream watched spans to Tracer Cloud (free) | -| `TRACER_CLOUD_URL` | override the Cloud base URL (default `https://app.tracerml.ai`) | | `TRACER_WATCH_DIR` | local JSONL directory (default `.tracer/watch`) | | `TRACER_WATCH_OTLP_ENDPOINT` | also POST spans to this OTLP/HTTP endpoint | | `TRACER_WATCH_OTLP_HEADERS` | comma-separated `k=v` headers for the OTLP endpoint | @@ -107,12 +76,12 @@ Pass a custom `sink=` to `watch()` to fully control export. ## JavaScript / TypeScript The same watcher ships for JS/TS as `@tracer-llm/watch` (zero dependencies, same -capture, same free Tracer Cloud streaming): +capture, local files and opt-in generic export): ```ts import { watch } from "@tracer-llm/watch"; -const w = watch("support-router"); // local by default; cloud with one key +const w = watch("support-router"); // local by default // wrap a function const classify = w(async (ticket: string) => @@ -132,11 +101,28 @@ await w.span({ input, userId: "u_42" }, async (s) => { }); ``` -Set `TRACER_CLOUD_KEY` (or `watch(name, { cloudKey })`) to stream to Tracer -Cloud. See the `@tracer-llm/watch` package README for details. +See the [`@tracer-llm/watch` package README](../js/README.md) for details. ## From watched traffic to a router -Watched spans map 1:1 to `tracer.types.TraceRecord`, so once you've collected -real traffic you can fit a routing policy straight from it, or let Tracer Cloud -auto-optimize one for you. See the [concepts guide](concepts.md). +Convert saved spans to the fitting schema before passing them to `tracer.fit()`: + +```python +import json +from pathlib import Path +from tracer.watch import GenAISpan + +source = Path(".tracer/watch/support_classifier.jsonl") +with Path("traces.jsonl").open("w", encoding="utf-8") as output: + for line in source.read_text(encoding="utf-8").splitlines(): + span = GenAISpan(**json.loads(line)) + record = span.to_trace_record() + output.write(json.dumps({ + "input": record.input_text, + "teacher": record.teacher_label, + }) + "\n") +``` + +Use calls whose outputs are classification labels, then compute embeddings for +the same rows and call `tracer.fit("traces.jsonl", embeddings=X)`. See the +[concepts guide](concepts.md). diff --git a/js/README.md b/js/README.md index cff567a..14432c1 100644 --- a/js/README.md +++ b/js/README.md @@ -11,8 +11,7 @@ are appended to `./.tracer/watch/.jsonl`. portable to any OTel-aware backend. No proprietary schema. - Zero runtime dependencies. Node stdlib + the global `fetch` only. - Prod-safe: telemetry never throws into, or adds latency to, the host call. - Cloud sends are batched on a timer; a slow or down endpoint can never block - your function, and a thrown request is swallowed. + Generic HTTP exports are sent asynchronously; export failures are swallowed. ## Install @@ -79,37 +78,17 @@ await w.span( Spans opened inside another span inherit the parent's `traceId` and point at it via `parentSpanId`, forming a trace tree (tracked with `AsyncLocalStorage`). -## Free Tracer Cloud streaming (one line) - -Set one environment variable and your watched traffic streams to the dashboard -within seconds. No code change: - -```sh -export TRACER_CLOUD_KEY=trobs_... -``` - -or pass it inline: - -```ts -const w = watch("classifier", { cloudKey: "trobs_..." }); -``` - ## Sinks | Sink | What it does | Turned on by | | ----------------- | ----------------------------------------------------------------------- | -------------------------------------------------------- | | `LocalFileSink` | Append spans as JSONL to `/.jsonl`. No network, no key. | Default, always on. | -| `TracerCloudSink` | Batch + stream spans to the free Tracer Cloud observability. | `cloudKey` option or `TRACER_CLOUD_KEY`. | | `OTLPSink` | POST an OTel GenAI-shaped payload to any OTLP/HTTP backend. | `TRACER_WATCH_OTLP_ENDPOINT` (+ `TRACER_WATCH_OTLP_HEADERS`). | | `MultiSink` | Fan-out to several sinks at once. | Composed automatically when more than one is configured. | -`sinkFromEnv(name, cloudKey?)` composes the chain: local file + cloud (if a key -is set) + OTLP (if an endpoint is set). - -The cloud sink routes by key prefix, matching the two product paths: - -- `trobs_*` (workspace ingest key) -> `POST /v1/observe`, body `{ events }` -- otherwise (per-tracer gateway) -> `POST /v1/ingest`, body `{ source, events }` +`sinkFromEnv(name)` composes local files with the generic HTTP sink when +`TRACER_WATCH_OTLP_ENDPOINT` is explicitly configured. Pass `sink` to `watch()` +for a custom exporter. There is no default remote destination. ## Provider response auto-extraction @@ -126,8 +105,6 @@ Both objects and plain dicts are handled. | Variable | Purpose | Default | | ----------------------------- | --------------------------------------------------------------- | ---------------- | | `TRACER_WATCH_DIR` | Directory for local JSONL files. | `.tracer/watch` | -| `TRACER_CLOUD_KEY` | Key that enables free Tracer Cloud streaming. | (unset) | -| `TRACER_CLOUD_URL` | Base URL for the cloud sink. | provider default | | `TRACER_WATCH_OTLP_ENDPOINT` | Endpoint for the OTLP/HTTP sink. | (unset) | | `TRACER_WATCH_OTLP_HEADERS` | Comma-separated `k=v` headers for the OTLP sink. | (unset) | | `TRACER_WATCH_DEBUG` | Print why a telemetry send was dropped (never affects the host).| (unset) | diff --git a/js/src/index.ts b/js/src/index.ts index 0889b38..4127a70 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -13,8 +13,7 @@ * - The same span maps 1:1 to a TRACER trace record (input + the model's * answer), so watched traffic feeds the optimizer directly. * - Exporters (sinks) are pluggable. `LocalFileSink` (default, no key) writes - * JSONL. `TracerCloudSink` streams the same spans to the free Tracer Cloud - * observability with one key. `OTLPSink` fans the SAME spans out over + * JSONL. `OTLPSink` exports spans to an explicitly configured endpoint over * OTLP/HTTP to any OTLP/HTTP backend, zero code change. * * Zero runtime dependencies (Node stdlib only). Prod-safe: telemetry never @@ -437,7 +436,7 @@ export class OTLPSink implements Sink { } } -/** Fan-out to several sinks (e.g. local + cloud + your own OTLP backend). */ +/** Fan-out to several sinks (e.g. local + your own OTLP backend). */ export class MultiSink implements Sink { readonly sinks: Sink[]; constructor(sinks: Sink[]) { @@ -465,166 +464,16 @@ export class MultiSink implements Sink { } } -// Default Tracer Cloud endpoint. Override with TRACER_CLOUD_URL. -const DEFAULT_CLOUD_URL = "https://app.tracerml.ai"; -const USER_AGENT = `tracer-watch-js/${VERSION}`; - -export interface TracerCloudSinkOptions { - baseUrl?: string; - source?: string; - batchSize?: number; - flushIntervalMs?: number; - timeoutMs?: number; - maxQueue?: number; -} - -/** - * Stream observed spans to Tracer Cloud (free observability). - * - * Point it at a Tracer Cloud key and your watched traffic shows up in the - * dashboard within seconds. No login, no SDK -- just a key (`cloudKey=...` or - * `TRACER_CLOUD_KEY`). - * - * Routes by key type, matching the two product paths: - * - `trobs_*` (workspace ingest key) -> `/v1/observe` (tenant-wide) - * - otherwise (per-tracer gateway) -> `/v1/ingest` (bound to a tracer) - * - * Prod-safe: sends are batched on a timer, so a slow or down endpoint never - * adds latency to (or crashes) the host function. Drops silently on overflow / - * error; set TRACER_WATCH_DEBUG=1 to see why. - */ -export class TracerCloudSink implements Sink { - readonly key: string; - readonly baseUrl: string; - readonly path: string; - readonly source: string; - readonly batchSize: number; - readonly flushIntervalMs: number; - readonly timeoutMs: number; - readonly maxQueue: number; - /** trobs_ = workspace ingest key -> /v1/observe; otherwise per-tracer -> /v1/ingest */ - readonly observe: boolean; - - private queue: Array> = []; - private timer: ReturnType | undefined; - private closed = false; - - constructor(key: string, opts: TracerCloudSinkOptions = {}) { - this.key = key; - this.baseUrl = (opts.baseUrl || getEnv("TRACER_CLOUD_URL") || DEFAULT_CLOUD_URL).replace( - /\/+$/, - "", - ); - this.observe = key.startsWith("trobs_"); - this.path = this.observe ? "/v1/observe" : "/v1/ingest"; - this.source = opts.source ?? "watch"; - this.batchSize = opts.batchSize ?? 25; - this.flushIntervalMs = opts.flushIntervalMs ?? 2000; - this.timeoutMs = opts.timeoutMs ?? 10000; - this.maxQueue = opts.maxQueue ?? 10000; - - this.timer = setInterval(() => { - void this.flush(); - }, this.flushIntervalMs); - // Do not keep the event loop alive just for telemetry. - if (this.timer && typeof (this.timer as { unref?: () => void }).unref === "function") { - (this.timer as { unref?: () => void }).unref!(); - } - } - - /** Internal event mapping (exposed for tests). */ - event(span: GenAISpan): Record { - if (this.observe) { - return { - ts: span.startTime || null, - system: span.system ?? null, - model: span.responseModel || span.requestModel || null, - input: span.inputText, - output: span.outputText, - prompt_tokens: span.inputTokens ?? null, - completion_tokens: span.outputTokens ?? null, - latency_ms: span.latencyMs ?? null, - cost_usd: span.costUsd ?? null, - status: span.status, - trace_id: span.traceId, - tags: span.tags, - }; - } - // per-tracer /v1/ingest shape - return { - input: span.inputText, - teacher: span.outputText || null, - output: span.outputText || null, - model: span.responseModel || span.requestModel || "observed", - cost_usd: span.costUsd ?? null, - ts: span.startTime || null, - }; - } - - emit(span: GenAISpan): void { - try { - if (this.queue.length >= this.maxQueue) { - debugLog("cloud queue full, dropping span"); - return; - } - this.queue.push(this.event(span)); - } catch (e) { - debugLog(`cloud enqueue failed: ${String(e)}`); - } - } - - /** POST one batch. Exposed for tests. Never throws. */ - async post(events: Array>): Promise { - const body = this.observe ? { events } : { source: this.source, events }; - const headers = { - "Content-Type": "application/json", - Authorization: `Bearer ${this.key}`, - // A real UA: a default UA can trip bot protection (403/1010) at the edge. - "User-Agent": USER_AGENT, - }; - try { - await postJson(`${this.baseUrl}${this.path}`, body, headers, this.timeoutMs); - } catch (e) { - // never let telemetry crash or block the host - debugLog(`cloud export failed: ${String(e)}`); - } - } - - /** Drain and POST everything currently queued, in batches. */ - async flush(): Promise { - while (this.queue.length) { - const batch = this.queue.splice(0, this.batchSize); - await this.post(batch); - } - } - - async close(): Promise { - if (this.closed) return; - this.closed = true; - if (this.timer) { - clearInterval(this.timer); - this.timer = undefined; - } - await this.flush(); - } -} - /** * Compose the default sink chain from the environment: * LocalFileSink (always) - * + TracerCloudSink (if cloudKey or TRACER_CLOUD_KEY) * + OTLPSink (if TRACER_WATCH_OTLP_ENDPOINT [+ TRACER_WATCH_OTLP_HEADERS]) */ -export function sinkFromEnv(name: string, cloudKey?: string): Sink { +export function sinkFromEnv(name: string): Sink { const dir = getEnv("TRACER_WATCH_DIR") || ".tracer/watch"; const local = new LocalFileSink(name, dir); const sinks: Sink[] = [local]; - const key = cloudKey || getEnv("TRACER_CLOUD_KEY"); - if (key) { - sinks.push(new TracerCloudSink(key, { source: name })); - } - const endpoint = getEnv("TRACER_WATCH_OTLP_ENDPOINT"); if (endpoint) { const headers: Record = {}; @@ -679,7 +528,7 @@ async function postJson( } // --------------------------------------------------------------------------- // -// Watcher: the wrapper / decorator / async span. The one object the cloud reuses. +// Watcher: the wrapper / decorator / async span. Records calls to local or explicitly configured sinks. // --------------------------------------------------------------------------- // /** Tracks the currently-open span so nested calls form a trace tree. */ @@ -735,10 +584,8 @@ export interface WatchOptions { system?: string; model?: string; operation?: string; - cloudKey?: string; sink?: Sink; defaultTags?: string[]; - baseUrl?: string; extractInput?: (args: unknown[]) => string; extractOutput?: (result: unknown) => string; } @@ -842,25 +689,13 @@ class WatcherClass { this.system = opts.system; this.model = opts.model; this.operation = opts.operation ?? "chat"; - // Default: local-only. Free Tracer Cloud streaming turns on with a single - // key (cloudKey or TRACER_CLOUD_KEY). OTLP fan-out via TRACER_WATCH_OTLP_*. - this.sink = opts.sink ?? this.sinkFromEnvWithBase(name, opts.cloudKey, opts.baseUrl); + // Local-only unless a custom sink or generic export endpoint is configured. + this.sink = opts.sink ?? sinkFromEnv(name); this.defaultTags = [...(opts.defaultTags || [])]; this.extractInput = opts.extractInput ?? defaultExtractInput; this.extractOutput = opts.extractOutput ?? defaultExtractOutput; } - private sinkFromEnvWithBase(name: string, cloudKey?: string, baseUrl?: string): Sink { - // sinkFromEnv handles the local + OTLP + cloud composition. When a baseUrl - // is supplied we rebuild the cloud sink with it so opts.baseUrl is honored. - const base = sinkFromEnv(name, cloudKey); - if (!baseUrl) return base; - const rebind = (s: Sink): Sink => - s instanceof TracerCloudSink ? new TracerCloudSink(s.key, { source: s.source, baseUrl }) : s; - if (base instanceof MultiSink) return new MultiSink(base.sinks.map(rebind)); - return rebind(base); - } - /** Wrap a (sync or async) function so each call records a span. */ wrap(fn: F): F { const self = this; diff --git a/js/test/watch.test.ts b/js/test/watch.test.ts index 644c006..9458d7a 100644 --- a/js/test/watch.test.ts +++ b/js/test/watch.test.ts @@ -1,6 +1,6 @@ /** * Tests for @tracer-llm/watch: spans, OTel mapping, response extraction (both - * shapes), key-prefix routing, cloud-sink prod-safety + batching/flush, nested + * shapes), opt-in generic export, local-only defaults, nested * spans, the wrapper/decorator/span forms. All hermetic: no network, temp dirs, * fetch is mocked. */ @@ -12,9 +12,6 @@ import * as path from "node:path"; import { GenAISpan, LocalFileSink, - MultiSink, - OTLPSink, - TracerCloudSink, Watcher, extractResponse, sinkFromEnv, @@ -297,7 +294,7 @@ describe("extractResponse", () => { // ----- sinkFromEnv composition ----------------------------------------------- // describe("sinkFromEnv", () => { const saved: Record = {}; - const keys = ["TRACER_WATCH_DIR", "TRACER_CLOUD_KEY", "TRACER_WATCH_OTLP_ENDPOINT"]; + const keys = ["TRACER_WATCH_DIR", "TRACER_CLOUD_KEY", "TRACER_CLOUD_URL", "TRACER_WATCH_OTLP_ENDPOINT"]; beforeEach(() => { for (const k of keys) saved[k] = process.env[k]; }); @@ -316,153 +313,53 @@ describe("sinkFromEnv", () => { expect(sink).toBeInstanceOf(LocalFileSink); }); - it("adds the cloud sink when a key is present", async () => { - process.env.TRACER_WATCH_DIR = tmpDir(); - process.env.TRACER_CLOUD_KEY = "trobs_abc"; - delete process.env.TRACER_WATCH_OTLP_ENDPOINT; - const sink = sinkFromEnv("x") as MultiSink; - expect(sink).toBeInstanceOf(MultiSink); - const kinds = new Set(sink.sinks.map((s) => s.constructor.name)); - expect(kinds.has("LocalFileSink")).toBe(true); - expect(kinds.has("TracerCloudSink")).toBe(true); - await sink.close(); - }); - - it("cloudKey arg routes per-tracer for non-trobs keys", async () => { - process.env.TRACER_WATCH_DIR = tmpDir(); - delete process.env.TRACER_CLOUD_KEY; - const sink = sinkFromEnv("x", "trc_xyz") as MultiSink; - expect(sink).toBeInstanceOf(MultiSink); - const cloud = sink.sinks.find((s) => s instanceof TracerCloudSink) as TracerCloudSink; - expect(cloud).toBeTruthy(); - expect(cloud.observe).toBe(false); - await sink.close(); - }); + it.each(["trobs_obsolete", "trc_obsolete"])( + "old app credentials (%s) never enable network export", + async (key) => { + const dir = tmpDir(); + process.env.TRACER_WATCH_DIR = dir; + process.env.TRACER_CLOUD_KEY = key; + process.env.TRACER_CLOUD_URL = "https://unused.invalid"; + delete process.env.TRACER_WATCH_OTLP_ENDPOINT; + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + try { + const recorder = watch("local"); + await recorder.span({ input: "ticket" }, async (span) => { + span.setOutput("billing"); + }); + await recorder.close(); + expect(fetchMock).not.toHaveBeenCalled(); + const saved = JSON.parse(fs.readFileSync(path.join(dir, "local.jsonl"), "utf8")); + expect(saved.input_text).toBe("ticket"); + expect(saved.output_text).toBe("billing"); + } finally { + vi.unstubAllGlobals(); + } + }, + ); - it("adds the OTLP sink when an endpoint is present", async () => { - process.env.TRACER_WATCH_DIR = tmpDir(); - delete process.env.TRACER_CLOUD_KEY; + it("exports only to the explicitly configured generic endpoint", async () => { + const dir = tmpDir(); + process.env.TRACER_WATCH_DIR = dir; + process.env.TRACER_CLOUD_KEY = "trobs_obsolete"; process.env.TRACER_WATCH_OTLP_ENDPOINT = "http://collector/v1/traces"; - const sink = sinkFromEnv("x") as MultiSink; - expect(sink.sinks.some((s) => s instanceof OTLPSink)).toBe(true); - }); -}); - -// ----- TracerCloudSink: routing, mapping, prod-safety, batching -------------- // -describe("TracerCloudSink", () => { - it("routes by key prefix", async () => { - const obs = new TracerCloudSink("trobs_k", { baseUrl: "http://x" }); - const ing = new TracerCloudSink("trc_k", { baseUrl: "http://x" }); - expect(obs.path).toBe("/v1/observe"); - expect(obs.observe).toBe(true); - expect(ing.path).toBe("/v1/ingest"); - expect(ing.observe).toBe(false); - await obs.close(); - await ing.close(); - }); - - it("observe event shape", async () => { - const s = new TracerCloudSink("trobs_k", { baseUrl: "http://x" }); - const span = new GenAISpan({ - system: "provider-x", - requestModel: "model-x", - responseModel: "model-x", - inputText: "in", - outputText: "out", - inputTokens: 5, - outputTokens: 2, - costUsd: 0.001, - status: "ok", - traceId: "t1", - }); - const e = s.event(span); - expect(e.input).toBe("in"); - expect(e.output).toBe("out"); - expect(e.model).toBe("model-x"); - expect(e.prompt_tokens).toBe(5); - expect(e.completion_tokens).toBe(2); - expect(e.cost_usd).toBe(0.001); - await s.close(); - }); - - it("ingest event shape", async () => { - const s = new TracerCloudSink("trc_k", { baseUrl: "http://x" }); - const span = new GenAISpan({ requestModel: "model-x", inputText: "in", outputText: "out", costUsd: 0.002 }); - const e = s.event(span); - expect(e.input).toBe("in"); - expect(e.teacher).toBe("out"); - expect(e.output).toBe("out"); - expect(e.model).toBe("model-x"); - await s.close(); - }); - - it("post builds the request (url, auth, UA, body) via global fetch", async () => { - let seen: { url?: string; headers?: Record; body?: string } = {}; - const fetchMock = vi.fn(async (url: string, init: RequestInit) => { - seen = { - url, - headers: init.headers as Record, - body: init.body as string, - }; - return { text: async () => "{}" } as any; - }); + const fetchMock = vi.fn(async () => ({ text: async () => "{}" })); vi.stubGlobal("fetch", fetchMock); - const s = new TracerCloudSink("trobs_k", { baseUrl: "http://host", source: "app" }); - await s.post([{ input: "x" }]); - expect(seen.url).toBe("http://host/v1/observe"); - expect(seen.headers!.Authorization).toBe("Bearer trobs_k"); - expect(String(seen.headers!["User-Agent"]).toLowerCase()).toContain("tracer-watch"); - expect(JSON.parse(seen.body!)).toEqual({ events: [{ input: "x" }] }); - await s.close(); - vi.unstubAllGlobals(); - }); - - it("ingest post wraps events with source", async () => { - let body: any; - vi.stubGlobal( - "fetch", - vi.fn(async (_url: string, init: RequestInit) => { - body = JSON.parse(init.body as string); - return { text: async () => "{}" } as any; - }), - ); - const s = new TracerCloudSink("trc_k", { baseUrl: "http://host", source: "app" }); - await s.post([{ input: "x" }]); - expect(body).toEqual({ source: "app", events: [{ input: "x" }] }); - await s.close(); - vi.unstubAllGlobals(); - }); - - it("post swallows errors (prod-safe): a throwing fetch must not reject", async () => { - vi.stubGlobal( - "fetch", - vi.fn(async () => { - throw new Error("network down"); - }), - ); - const s = new TracerCloudSink("trobs_k", { baseUrl: "http://host" }); - await expect(s.post([{ input: "x" }])).resolves.toBeUndefined(); - await s.close(); - vi.unstubAllGlobals(); - }); - - it("emit + flush on close (batched) via mocked fetch", async () => { - const posted: any[] = []; - vi.stubGlobal( - "fetch", - vi.fn(async (_url: string, init: RequestInit) => { - posted.push(JSON.parse(init.body as string)); - return { text: async () => "{}" } as any; - }), - ); - const s = new TracerCloudSink("trobs_k", { baseUrl: "http://host", flushIntervalMs: 50 }); - s.emit(new GenAISpan({ inputText: "a", outputText: "b" })); - s.emit(new GenAISpan({ inputText: "c", outputText: "d" })); - await s.close(); // flushes the queue - const flat = posted.flatMap((b) => b.events); - expect(flat).toHaveLength(2); - expect(new Set(flat.map((e: any) => e.input))).toEqual(new Set(["a", "c"])); - vi.unstubAllGlobals(); + try { + const recorder = watch("export"); + await recorder.span({ input: "ticket" }, async (span) => { + span.setOutput("billing"); + }); + await recorder.close(); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, options] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe("http://collector/v1/traces"); + expect(JSON.parse(options.body as string).attributes["gen_ai.operation.name"]).toBe("chat"); + expect(fs.existsSync(path.join(dir, "export.jsonl"))).toBe(true); + } finally { + vi.unstubAllGlobals(); + } }); }); diff --git a/llms.txt b/llms.txt index 67986c0..3515c84 100644 --- a/llms.txt +++ b/llms.txt @@ -4,6 +4,7 @@ > Turn LLM classification traces into routing policies. Surrogate handles easy traffic; teacher LLM handles hard. Formal quality guarantees. ## Install +Requires Python 3.12+. CI tests Python 3.12, 3.13, and 3.14. pip install tracer-llm ## Core API (4 functions) @@ -24,7 +25,8 @@ Embedder.from_callable(fn) - AGENTS.md - full integration guide for AI agents - README.md - user-facing docs - docs/api.md - Python API reference -- docs/cli.md - CLI reference +- docs/scan.md - pre-training diagnostics +- docs/watch.md - local trace recording and generic exports - docs/concepts.md - how it works - docs/artifacts.md - .tracer/ directory schema - docs/troubleshooting.md - common errors and fixes @@ -35,7 +37,6 @@ https://arxiv.org/abs/2604.14531 Pricing Capability research paper: https://github.com/adrida/tracer/blob/main/research/pricing-capability.pdf -https://tracerml.ai/research/pricing-capability/ Current final submission: 21 pages, dated August 25, 2026. Defines Capability Alpha under controlled evaluation, develops Price-Implied Uplift, and places the Spirit case beside a public transaction panel. diff --git a/pyproject.toml b/pyproject.toml index 849fcca..6344e68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "TRACER: Trace-Based Adaptive Cost-Efficient Routing. Turn LLM traces into parity-gated routing policies - cut 90%+ of LLM calls with formal guarantees." readme = "README.md" license = {file = "LICENSE"} -requires-python = ">=3.9" +requires-python = ">=3.12" keywords = [ "llm", "routing", "surrogate", "cost-reduction", "learn-to-defer", "conformal-prediction", "machine-learning", "explainability", "xai", @@ -20,31 +20,27 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "numpy>=1.21,<2.1", - "scikit-learn>=1.0", - "joblib>=1.1", + "numpy>=1.26.4,<3", + "scikit-learn>=1.4.2", + "joblib>=1.3.2", ] [project.optional-dependencies] embeddings = [ "sentence-transformers>=2.2,<5", - "torch>=2.0", - "numpy>=1.21,<2.1", + "torch>=2.4.1", ] faiss = ["faiss-cpu>=1.7"] notebooks = [ "faiss-cpu>=1.7", "sentence-transformers>=2.2,<5", - "torch>=2.0", - "numpy>=1.21,<2.1", + "torch>=2.4.1", "pandas>=1.4", "matplotlib>=3.5", "datasets>=2.0", @@ -55,8 +51,7 @@ xgboost = ["xgboost>=1.7"] dev = ["pytest>=7.0", "pytest-cov>=4.0"] all = [ "sentence-transformers>=2.2,<5", - "torch>=2.0", - "numpy>=1.21,<2.1", + "torch>=2.4.1", "faiss-cpu>=1.7", "xgboost>=1.7", "plotly>=5.0", @@ -71,9 +66,6 @@ Repository = "https://github.com/adrida/tracer" Documentation = "https://github.com/adrida/tracer#readme" "Bug Tracker" = "https://github.com/adrida/tracer/issues" -[project.scripts] -tracer = "tracer.cli.main:main" - [tool.hatch.version] path = "src/tracer/__init__.py" diff --git a/src/tracer/__init__.py b/src/tracer/__init__.py index 9ce8548..d069060 100644 --- a/src/tracer/__init__.py +++ b/src/tracer/__init__.py @@ -15,8 +15,8 @@ # Lazy public API (PEP 562). The heavy ML stack (numpy / scikit-learn / # hdbscan / sentence-transformers) is only imported on first attribute access, -# so `import tracer` -- and lightweight entry points like `tracer cloud` -- -# start instantly instead of paying multi-second import cost up front. +# so `import tracer` and local trace recording start instantly instead of +# paying multi-second import cost up front. # # `fit` also collides with the tracer.fit subpackage, so it cannot be healed by # __getattr__ alone (the submodule attribute shadows it). tracer.api re-asserts diff --git a/src/tracer/cli/__init__.py b/src/tracer/cli/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/src/tracer/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/tracer/cli/_ui.py b/src/tracer/cli/_ui.py deleted file mode 100644 index 7ae3a9c..0000000 --- a/src/tracer/cli/_ui.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Terminal UI utilities -- colors, spinner, formatted output. - -Zero external dependencies. Falls back to plain text when not a TTY. -""" - -from __future__ import annotations - -import itertools -import sys -import threading -import time -from contextlib import contextmanager -from typing import Optional - - -# ── ANSI ────────────────────────────────────────────────────────────────────── - -def _is_tty() -> bool: - return hasattr(sys.stdout, "isatty") and sys.stdout.isatty() - - -class _Ansi: - RESET = "\033[0m" - BOLD = "\033[1m" - DIM = "\033[2m" - - BLACK = "\033[30m" - RED = "\033[31m" - GREEN = "\033[32m" - YELLOW = "\033[33m" - BLUE = "\033[34m" - MAGENTA = "\033[35m" - CYAN = "\033[36m" - WHITE = "\033[37m" - - BG_BLACK = "\033[40m" - - def __getattr__(self, name): - return "" # fallback: return empty string when color disabled - - -_NO_COLOR = _Ansi() -_NO_COLOR.RESET = _NO_COLOR.BOLD = _NO_COLOR.DIM = "" -for _k in ("BLACK","RED","GREEN","YELLOW","BLUE","MAGENTA","CYAN","WHITE","BG_BLACK"): - setattr(_NO_COLOR, _k, "") - -def _C() -> _Ansi: - """Return color codes if TTY, empty strings otherwise.""" - return _Ansi() if _is_tty() else _NO_COLOR - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - -def hr(char="─", width=56) -> str: - c = _C() - return f"{c.DIM}{char * width}{c.RESET}" - - -def header(title: str, subtitle: str = "") -> None: - c = _C() - print() - print(f" {c.BOLD}{c.CYAN}▸ TRACER{c.RESET} {c.WHITE}{title}{c.RESET}") - if subtitle: - print(f" {c.DIM}{subtitle}{c.RESET}") - print(f" {hr()}") - - -def section(title: str) -> None: - c = _C() - print() - print(f" {c.BOLD}{c.YELLOW}{title}{c.RESET}") - print(f" {hr('·')}") - - -def stat(label: str, value: str, note: str = "", color: str = "") -> None: - c = _C() - val_color = getattr(c, color.upper(), "") if color else c.WHITE - pad = 18 - print(f" {c.DIM}{label:<{pad}}{c.RESET}{c.BOLD}{val_color}{value}{c.RESET}" - + (f" {c.DIM}{note}{c.RESET}" if note else "")) - - -def success(msg: str) -> None: - c = _C() - print(f" {c.GREEN}✔{c.RESET} {msg}") - - -def warn(msg: str) -> None: - c = _C() - print(f" {c.YELLOW}⚠{c.RESET} {msg}") - - -def info(msg: str) -> None: - c = _C() - print(f" {c.DIM}→{c.RESET} {msg}") - - -def bar_line(label: str, rate: float, count: int, width: int = 22) -> None: - c = _C() - filled = int(rate * width) - empty = width - filled - if rate >= 0.85: - fill_c = c.GREEN - elif rate >= 0.60: - fill_c = c.YELLOW - else: - fill_c = c.RED - bar = f"{fill_c}{'█' * filled}{c.DIM}{'░' * empty}{c.RESET}" - pct = f"{rate:>5.1%}" - print(f" {c.DIM}{label:<28}{c.RESET}{bar} {c.BOLD}{pct}{c.RESET} {c.DIM}n={count}{c.RESET}") - - -def pair_block(teacher_label: str, handled: str, deferred: str, - h_score: Optional[float] = None, d_score: Optional[float] = None) -> None: - c = _C() - hs = f" {c.DIM}score={h_score:.2f}{c.RESET}" if h_score is not None else "" - ds = f" {c.DIM}score={d_score:.2f}{c.RESET}" if d_score is not None else "" - print(f" {c.DIM}label:{c.RESET} {c.BOLD}{teacher_label}{c.RESET}") - print(f" {c.GREEN}SURROGATE{c.RESET} {handled[:72]}{hs}") - print(f" {c.RED}→ LLM {c.RESET} {deferred[:72]}{ds}") - print() - - -def route_line(text: str, decision: str, label: str, score: Optional[float]) -> None: - c = _C() - if decision == "handled": - tag = f"{c.BG_BLACK}{c.GREEN} SURR. {c.RESET}" - label_str = f" {c.DIM}→{c.RESET} {c.CYAN}{label}{c.RESET}" - else: - tag = f"{c.BG_BLACK}{c.YELLOW} → LLM {c.RESET}" - label_str = "" - score_str = f" {c.DIM}({score:.2f}){c.RESET}" if score is not None else "" - print(f" {tag}{score_str} {text[:60]}{label_str}") - - -def cost_table(coverage: float, daily_queries: int = 10_000, cost_per_call: float = 0.002) -> None: - c = _C() - llm_full = daily_queries - llm_tracer = int(daily_queries * (1 - coverage)) - day_full = llm_full * cost_per_call - day_tracer = llm_tracer * cost_per_call - annual_save = (day_full - day_tracer) * 365 - - print(f" {c.DIM}{'':4}{'LLM calls/day':>15} {'cost/day':>10} {'annual':>10}{c.RESET}") - print(f" {c.DIM}Without TRACER{c.RESET} {llm_full:>13,} ${day_full:>9,.2f} -") - print(f" {c.GREEN}{c.BOLD}With TRACER {c.RESET} {llm_tracer:>13,} ${day_tracer:>9,.2f} {c.BOLD}{c.GREEN}${annual_save:>9,.0f} saved/yr{c.RESET}") - reduction = coverage - print(f" {c.DIM}({reduction:.0%} fewer LLM calls){c.RESET}") - - -# ── Spinner ─────────────────────────────────────────────────────────────────── - -_SPIN_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] -_SPIN_FRAMES_PLAIN = ["|", "/", "-", "\\"] - - -class Spinner: - """Context manager that displays an animated spinner while work runs.""" - - def __init__(self, msg: str, done_msg: str = "done"): - self.msg = msg - self.done_msg = done_msg - self._stop = threading.Event() - self._thread: Optional[threading.Thread] = None - - def _spin(self) -> None: - if not _is_tty(): - sys.stdout.write(f" {self.msg}...\n") - sys.stdout.flush() - return - frames = _SPIN_FRAMES if _is_tty() else _SPIN_FRAMES_PLAIN - c = _C() - for frame in itertools.cycle(frames): - if self._stop.is_set(): - break - sys.stdout.write(f"\r {c.CYAN}{frame}{c.RESET} {self.msg}...") - sys.stdout.flush() - time.sleep(0.08) - - def __enter__(self): - self._stop.clear() - self._thread = threading.Thread(target=self._spin, daemon=True) - self._thread.start() - return self - - def __exit__(self, *_): - self._stop.set() - if self._thread: - self._thread.join(timeout=0.5) - c = _C() - if _is_tty(): - sys.stdout.write(f"\r {c.GREEN}✔{c.RESET} {self.msg} {c.DIM}{self.done_msg}{c.RESET}\n") - sys.stdout.flush() - - -@contextmanager -def step(msg: str, done_msg: str = "done"): - with Spinner(msg, done_msg): - yield diff --git a/src/tracer/cli/main.py b/src/tracer/cli/main.py deleted file mode 100644 index ab0d798..0000000 --- a/src/tracer/cli/main.py +++ /dev/null @@ -1,736 +0,0 @@ -"""TRACER CLI. - - tracer demo - tracer fit traces.jsonl [--artifact-dir .tracer] [--target 0.90] - tracer report [.tracer] - tracer report-html [.tracer] - tracer update new_traces.jsonl [--artifact-dir .tracer] -""" - -from __future__ import annotations - -import argparse -import json -import sys -import textwrap - - -# ── helpers ─────────────────────────────────────────────────────────────────── - -def _cmd_scan(args): - from tracer.scanner import ( - scan, format_scan, scan_html, load_scan_traces, - MIN_SCAN_TRACES, SUGGESTED_SCAN_TRACES, - ) - from tracer.cli._ui import header, step, warn, info - - header("scan", f"traces={args.traces} target={args.target:.0%}") - - # Data-volume gate (checked before the embedding step). - n_traces = len(load_scan_traces(args.traces)[0]) - if n_traces < MIN_SCAN_TRACES and not args.force: - warn(f"Only {n_traces:,} traces. tracer scan needs at least " - f"{MIN_SCAN_TRACES:,} for a stable read, and ~{SUGGESTED_SCAN_TRACES:,} " - f"is the sweet spot.") - info("Collect more traces, or re-run with --force to scan anyway on thin " - "data (results will be a best-effort floor, not a guarantee).") - sys.exit(1) - if args.force: - warn("--force: bypassing the thin-data guard. Clustering will be coarsened to " - "concentrate held-out evidence; reported bounds are a best-effort floor, " - "not a guarantee.") - if n_traces < MIN_SCAN_TRACES: - info(f"{n_traces:,} traces is below the {MIN_SCAN_TRACES:,} floor; " - f"~{SUGGESTED_SCAN_TRACES:,} is the sweet spot for a real verdict.") - - # Embeddings. Default: sentence-transformers locally (the model below). - # --embeddings reuses a precomputed .npy; --embed-url calls your own HTTP - # embedding service (any HTTP endpoint), no vendor lock-in. - embeddings = None - if args.embeddings: - import numpy as _np - embeddings = _np.load(args.embeddings) - elif args.embed_url: - from tracer.embeddings.embedder import Embedder - headers = {} - for h in (args.embed_header or []): - k, _, v = h.partition(":") - headers[k.strip()] = v.strip() - texts, _ = load_scan_traces(args.traces) - try: - with step(f"Embedding {len(texts)} traces via {args.embed_url}"): - embeddings = Embedder.from_endpoint( - args.embed_url, headers=headers or None, - input_key=args.embed_input_key, output_key=args.embed_output_key, - batch_key=args.embed_batch_key, - ).embed(texts) - except Exception as e: - warn(f"Embedding endpoint failed ({args.embed_url}): {e}") - info("Check the URL, --embed-header auth, and the " - "--embed-input-key/--embed-output-key for your endpoint's shape.") - sys.exit(1) - - with step("Clustering + exact bounds"): - result = scan( - args.traces, - target=args.target, - embeddings=embeddings, - model=args.embed_model, - teacher_price_per_1k=args.teacher_price_per_1k, - monthly_calls=args.monthly_calls, - viz_layout=args.viz_layout, - force=args.force, - ) - print(format_scan(result)) - if args.html: - import pathlib - out = pathlib.Path(args.html) - out.write_text(scan_html(result, source_name=pathlib.Path(args.traces).name), - encoding="utf-8") - print(f"\nHTML report written to {out}") - if not args.no_open: - import webbrowser - webbrowser.open(out.resolve().as_uri()) - - -def _cmd_fit(args): - from tracer.api import fit - from tracer.config import FitConfig - from tracer.cli._ui import header, section, step, success, warn, stat, bar_line, hr, _C - - c = _C() - header("fit", f"traces={args.traces} target-TA={args.target:.0%}") - - # Tree surrogates (decision tree, random forest, extra-trees, gradient - # boosting) are heavy and off by default; --trees opts them back in, and - # --skip drops named candidates explicitly. - TREES = ("dt", "rf", "et", "gbt") - skip = {s.strip() for s in (args.skip or "").split(",") if s.strip()} - if not getattr(args, "trees", False): - skip |= set(TREES) - config = FitConfig(target_teacher_agreement=args.target, - skip_candidates=tuple(sorted(skip))) - - with step("Fitting routing policy (linear + MLP zoo + calibration)"): - result = fit(args.traces, artifact_dir=args.artifact_dir, config=config) - - _print_fit_result(result) - - -def _cmd_report(args): - from tracer.api import report - from tracer.cli._ui import _C, header, stat, section - - c = _C() - m = report(args.artifact_dir) - header("report", args.artifact_dir) - print(json.dumps({ - "version": m.version, - "n_traces": m.n_traces, - "label_space": m.label_space, - "method": m.selected_method, - "target_ta": m.target_teacher_agreement, - "coverage": m.coverage_cal, - "teacher_agreement": m.teacher_agreement_cal, - "embedding_dim": m.embedding_dim, - "n_retrains": m.n_retrains, - }, indent=2)) - - -def _cmd_report_html(args): - from tracer.analysis.html_report import generate_html_report - from tracer.cli._ui import success, info, step, _C - import webbrowser - - c = _C() - with step("Generating HTML report"): - out = generate_html_report(args.artifact_dir, output_path=args.output) - - success(f"HTML report saved → {c.CYAN}{out}{c.RESET}") - if not args.no_open: - info("Opening in browser...") - webbrowser.open(f"file://{out}") - - -def _cmd_sankey(args): - from tracer.analysis.sankey import generate_sankey - from tracer.cli._ui import success, info, step, _C - import webbrowser - - c = _C() - with step("Generating Sankey diagram"): - out = generate_sankey( - args.artifact_dir, - output_path=args.output, - fmt=args.format, - top_k=args.top_k, - ) - - success(f"Sankey diagram saved -> {c.CYAN}{out}{c.RESET}") - if args.format == "html" and not args.no_open: - info("Opening in browser...") - webbrowser.open(f"file://{out}") - - -def _cmd_serve(args): - from tracer.runtime.serve import serve - serve(artifact_dir=args.artifact_dir, host=args.host, port=args.port) - - -def _cmd_update(args): - from tracer.api import update - from tracer.cli._ui import header, step - - header("update", f"new traces={args.traces}") - - with step("Refitting with new traces (continual learning)"): - result = update(args.traces, artifact_dir=args.artifact_dir) - - _print_fit_result(result) - - -def _cmd_demo(args): - """Run a zero-setup demo. Uses Banking77 data if available, synthetic otherwise.""" - import shutil - from pathlib import Path - - import numpy as np - - from tracer.api import fit, load_router - from tracer.config import FitConfig - from tracer.analysis.html_report import generate_html_report - from tracer.cli._ui import ( - _C, header, section, stat, bar_line, pair_block, - route_line, cost_table, hr, success, info, warn, step, - ) - - c = _C() - - # ── Detect Banking77 data (already on disk in the repo) ─────────────────── - # Check relative to this file (tracer/src/tracer/cli/) → tracer/notebooks/data/ - _pkg_root = Path(__file__).resolve().parent.parent.parent.parent - _b77_traces = _pkg_root / "notebooks" / "data" / "banking77_traces.jsonl" - _b77_emb = _pkg_root / "notebooks" / "data" / "banking77_traces.npy" - # Also check from cwd (running from repo root) - if not _b77_traces.exists(): - _b77_traces = Path.cwd() / "tracer" / "notebooks" / "data" / "banking77_traces.jsonl" - _b77_emb = Path.cwd() / "tracer" / "notebooks" / "data" / "banking77_traces.npy" - - use_banking77 = _b77_traces.exists() and _b77_emb.exists() - - # ── Build dataset ───────────────────────────────────────────────────────── - demo_out_dir = Path.cwd() / "tracer-demo-output" - shutil.rmtree(demo_out_dir, ignore_errors=True) - demo_out_dir.mkdir(parents=True) - traces_path = demo_out_dir / "traces.jsonl" - emb_path = demo_out_dir / "embeddings.npy" - artifact_dir = demo_out_dir / ".tracer" - - if use_banking77: - # ── Banking77: real 77-class bank intent data (already downloaded) ──── - # Subsample to 1500 for demo speed; still representative - DEMO_N = 1_500 - header( - "Demo - Banking77 Intent Classifier (real data)", - f"77 intents · {DEMO_N:,} traces · 1024-dim BGE-M3 embeddings", - ) - info(f"Found Banking77 data at {_b77_traces.parent}") - print() - - # Load and subsample - import random as _rnd - _rnd.seed(42) - all_lines = _b77_traces.read_text(encoding="utf-8").splitlines() - sampled_lines = _rnd.sample(all_lines, min(DEMO_N, len(all_lines))) - - # Write sampled traces - with traces_path.open("w") as f: - for line in sampled_lines: - f.write(line + "\n") - - # Load full embeddings, extract matching rows by position in sampled lines - _all_indices = {line: i for i, line in enumerate(all_lines)} - row_indices = [_all_indices[line] for line in sampled_lines] - X_full = np.load(_b77_emb, mmap_mode="r") - X = X_full[row_indices].astype(np.float32) - np.save(emb_path, X) - - # Build label-to-embedding index for live routing - import json as _json - records = [_json.loads(l) for l in sampled_lines] - teacher_list = [r["teacher"] for r in records] - label_to_idx: dict[str, list[int]] = {} - for i, lbl in enumerate(teacher_list): - label_to_idx.setdefault(lbl, []).append(i) - - # Pick 5 real Banking77 queries from the sampled data (diverse intents) - _target_labels = [ - "card_arrival", "exchange_rate", "transfer_not_received_by_bank", - "card_payment_not_recognised", "change_email", - ] - _all_labels_present = set(teacher_list) - # Fall back to first 5 available labels if any aren't in this subsample - _fallback_labels = [l for l in sorted(_all_labels_present) - if l not in _target_labels] - _demo_label_list = [] - for lbl in _target_labels: - if lbl in _all_labels_present: - _demo_label_list.append(lbl) - elif _fallback_labels: - _demo_label_list.append(_fallback_labels.pop(0)) - # Build text lookup from raw records - _label_text = {} - for rec in records: - lbl = rec["teacher"] - if lbl not in _label_text: - _label_text[lbl] = rec.get("input", rec.get("text", "")) - demo_queries = [( - _label_text.get(lbl, lbl), # actual text from the dataset - lbl, - ) for lbl in _demo_label_list[:5]] - - def _pick_emb(preferred_label): - idxs = label_to_idx.get(preferred_label) - if idxs: - return X[idxs[0]], preferred_label - # fallback: first available - first = next(iter(label_to_idx)) - return X[label_to_idx[first][0]], first - - config = FitConfig(target_teacher_agreement=0.92, - frontier_targets=(0.88, 0.92)) - model_zoo_desc = "logreg · rf · et" - use_fast_zoo = True - - else: - # ── Synthetic fallback: clean 5-class data ───────────────────────────── - N, DIM, N_CLS = 2_000, 128, 5 - header( - "Demo - Synthetic Banking Intent Classifier", - f"5 intents · {N:,} traces · {DIM}-dim embeddings " - f"(tip: put Banking77 data in tracer/notebooks/data/ for a real demo)", - ) - rng = np.random.RandomState(42) - intent_names = ["check_balance", "transfer_money", "card_blocked", - "loan_inquiry", "account_settings"] - centers = rng.randn(N_CLS, DIM) * 2.5 - labels_int = rng.randint(0, N_CLS, size=N) - X = centers[labels_int] + rng.randn(N, DIM) * 1.2 - teacher_list = [intent_names[i] for i in labels_int] - ground_truth = list(teacher_list) - for i in range(N): - if rng.random() < 0.12: - teacher_list[i] = intent_names[rng.randint(0, N_CLS)] - - sample_texts = { - "check_balance": ["What is my current account balance?", - "How much money do I have?", - "Can you check my account balance?"], - "transfer_money": ["I want to send money to my friend", - "Transfer $500 to savings", - "How do I wire money abroad?"], - "card_blocked": ["My card is not working", - "Why was my card blocked?", - "My card got declined at the store"], - "loan_inquiry": ["What are your loan rates?", - "I'd like to apply for a personal loan", - "How much can I borrow?"], - "account_settings": ["Change my email address", - "How do I reset my password?", - "Update my mailing address"], - } - with traces_path.open("w") as f: - for i in range(N): - intent = ground_truth[i] - texts = sample_texts[intent] - text = texts[i % len(texts)] - if i >= len(texts): - text = f"{text} (query #{i})" - f.write(json.dumps({"input": text, "teacher": teacher_list[i], - "ground_truth": ground_truth[i], - "id": str(i)}) + "\n") - np.save(emb_path, X.astype(np.float32)) - - # Build label index for live routing - label_to_idx = {} - for i, lbl in enumerate(teacher_list): - label_to_idx.setdefault(lbl, []).append(i) - - demo_queries = [ - ("What is my current account balance?", "check_balance"), - ("I want to send money to my friend", "transfer_money"), - ("My card is not working", "card_blocked"), - ("What are your loan rates?", "loan_inquiry"), - ("Change my email address", "account_settings"), - ] - - def _pick_emb(preferred_label): - idxs = label_to_idx.get(preferred_label, [0]) - return X[idxs[0]].astype(np.float32), preferred_label - - config = FitConfig(target_teacher_agreement=0.95, - frontier_targets=(0.90, 0.95)) - model_zoo_desc = "logreg · mlp · rf · et" - use_fast_zoo = False - - # ── Fit ─────────────────────────────────────────────────────────────────── - surrogate_log: list[tuple[str, float]] = [] - - from tracer.fit import surrogate as _s_mod - from tracer.fit import pipeline as _p_mod - _orig_candidates = _s_mod._candidates - - if use_fast_zoo: - # Banking77: logreg + trees (fast on high-dim), skip MLP (slow) - def _demo_candidates(n_samples: int, skip=()) -> dict: - from sklearn.linear_model import LogisticRegression - from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier - from sklearn.pipeline import Pipeline - from sklearn.preprocessing import StandardScaler - SEED = 42 - return { - "logreg_c1": lambda: Pipeline([("scale", StandardScaler()), - ("clf", LogisticRegression(C=1.0, max_iter=1000, - random_state=SEED))]), - "logreg_c10": lambda: Pipeline([("scale", StandardScaler()), - ("clf", LogisticRegression(C=10.0, max_iter=1000, - random_state=SEED))]), - "rf": lambda: RandomForestClassifier(n_estimators=150, n_jobs=-1, - random_state=SEED), - "et": lambda: ExtraTreesClassifier(n_estimators=150, n_jobs=-1, - random_state=SEED), - } - else: - # Synthetic: add tree models - def _demo_candidates(n_samples: int, skip=()) -> dict: - from sklearn.linear_model import LogisticRegression - from sklearn.neural_network import MLPClassifier - from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier - from sklearn.pipeline import Pipeline - from sklearn.preprocessing import StandardScaler - SEED = 42 - return { - "logreg_c1": lambda: Pipeline([("scale", StandardScaler()), - ("clf", LogisticRegression(C=1.0, max_iter=500, - random_state=SEED))]), - "logreg_c10": lambda: Pipeline([("scale", StandardScaler()), - ("clf", LogisticRegression(C=10.0, max_iter=500, - random_state=SEED))]), - "mlp_1h": lambda: Pipeline([("scale", StandardScaler()), - ("clf", MLPClassifier(hidden_layer_sizes=(256,), max_iter=100, - early_stopping=True, random_state=SEED))]), - "rf": lambda: RandomForestClassifier(n_estimators=100, - n_jobs=-1, random_state=SEED), - "et": lambda: ExtraTreesClassifier(n_estimators=100, - n_jobs=-1, random_state=SEED), - } - - _orig_search = _p_mod.search_best_surrogate - - def _patched_search(X_tr, y_tr, X_val, y_val, on_candidate=None, **kwargs): - # Accept **kwargs so future args (e.g. `skip=`) flow through without - # needing a matching signature update here. - return _orig_search(X_tr, y_tr, X_val, y_val, - on_candidate=on_candidate or ( - lambda n, f: surrogate_log.append((n, f))), - **kwargs) - - _s_mod._candidates = _demo_candidates - _p_mod.search_best_surrogate = _patched_search - - try: - with step(f"Training model zoo ({model_zoo_desc})"): - result = fit(traces_path, artifact_dir=artifact_dir, - embeddings=X.astype(np.float32), config=config) - finally: - _s_mod._candidates = _orig_candidates - _p_mod.search_best_surrogate = _orig_search - - # ── Results ─────────────────────────────────────────────────────────────── - m = result.manifest - qr = result.qualitative_report - - section("Model Selection") - if surrogate_log: - # Deduplicate: each model may appear multiple times (once per pipeline family) - seen: dict[str, float] = {} - for name, f1 in surrogate_log: - if name not in seen or f1 > seen[name]: - seen[name] = f1 - best_name = max(seen, key=lambda k: seen[k]) - for name, f1 in seen.items(): - marker = f" {c.GREEN}← selected{c.RESET}" if name == best_name else "" - print(f" {c.DIM}{name:<18}{c.RESET} val F1 = {c.BOLD}{f1:.4f}{c.RESET}{marker}") - - section("Routing Policy") - method_color = {"global": "GREEN", "l2d": "CYAN", "rsb": "MAGENTA"}.get( - m.selected_method or "", "WHITE") - stat("method", m.selected_method or "none (parity gate blocked)", color=method_color.lower()) - stat("coverage", f"{m.coverage_cal:.1%}" if m.coverage_cal else "-", - note="of traffic handled by surrogate", color="green") - stat("teacher TA", f"{m.teacher_agreement_cal:.3f}" if m.teacher_agreement_cal else "-", - note="surrogate matches teacher on handled traffic", color="cyan") - stat("traces", f"{m.n_traces:,}") - stat("labels", str(len(m.label_space))) - - for note in result.notes: - if note.startswith("Deployed") or note.startswith("Loaded"): - info(note) - else: - warn(note) - - if qr: - section("Per-Label Coverage (top 15)") - label_slices = [s for s in qr.slices if s.slice_name.startswith("label:")] - for s in sorted(label_slices, key=lambda x: -x.handled_rate)[:15]: - bar_line(s.slice_name.replace("label:", ""), s.handled_rate, s.count) - - if qr.boundary_pairs: - section(f"Boundary Pairs ({len(qr.boundary_pairs)} examples - same label, different routing)") - print() - for bp in qr.boundary_pairs[:4]: - pair_block(bp.teacher_label, bp.handled_preview, bp.deferred_preview, - bp.handled_score, bp.deferred_score) - - # ── Live routing ────────────────────────────────────────────────────────── - # Pull real examples from the qualitative report: 3 handled + 2 deferred. - # This guarantees a realistic mix instead of cherry-picking embeddings. - if m.selected_method and qr: - section("Live Routing (sample queries from the training set)") - handled_ex = [e for e in qr.handled_examples if e.local_label][:3] - deferred_ex = qr.deferred_examples[:2] - for ex in handled_ex + deferred_ex: - route_line(ex.input_preview[:70], ex.decision, - ex.local_label or "", ex.accept_score) - - # ── Cost ────────────────────────────────────────────────────────────────── - if m.coverage_cal: - section("Cost Projection (10 k queries / day)") - cost_table(m.coverage_cal) - - # ── HTML ────────────────────────────────────────────────────────────────── - html_path = generate_html_report(artifact_dir) - - # ── Output paths ────────────────────────────────────────────────────────── - section("Demo Outputs Saved") - success(f"Traces → {c.CYAN}{traces_path}{c.RESET}") - success(f"Embeddings → {c.CYAN}{emb_path}{c.RESET}") - success(f"Artifacts → {c.CYAN}{artifact_dir}{c.RESET}") - success(f"HTML report → {c.CYAN}{html_path}{c.RESET}") - - print() - print(f" {c.DIM}{hr()}{c.RESET}") - print(f" {c.BOLD}Next steps{c.RESET}") - print(f" {c.DIM}1.{c.RESET} Open the HTML report:") - print(f" {c.CYAN}open {html_path}{c.RESET}") - print(f" {c.DIM}2.{c.RESET} Fit on your own traces:") - print(f" {c.CYAN}tracer fit traces.jsonl --target 0.90{c.RESET}") - print(f" {c.DIM}3.{c.RESET} Route with text directly (no manual embedding):") - print(f" {c.CYAN}from tracer import Embedder{c.RESET}") - print(f" {c.CYAN}embedder = Embedder.from_sentence_transformers('BAAI/bge-small-en-v1.5'){c.RESET}") - print(f" {c.CYAN}router = tracer.load_router('.tracer', embedder=embedder){c.RESET}") - print(f" {c.CYAN}router.predict('What is my balance?'){c.RESET}") - print(f" {c.DIM}4.{c.RESET} Docs: {c.CYAN}https://github.com/adrida/tracer{c.RESET}") - print() - - -def _print_fit_result(result): - """Colored summary of a FitResult -- used by fit and update commands.""" - from tracer.cli._ui import ( - _C, section, stat, bar_line, pair_block, success, warn, info, hr, - ) - from tracer.analysis.html_report import generate_html_report - - c = _C() - m = result.manifest - qr = result.qualitative_report - - section("Routing Policy") - method_color = {"global": "green", "l2d": "cyan", "rsb": "magenta"}.get( - m.selected_method or "", "white") - stat("method", m.selected_method or "none (parity gate blocked)", - color=method_color) - if m.coverage_cal is not None: - stat("coverage", f"{m.coverage_cal:.1%}", "handled by surrogate", color="green") - stat("teacher TA", f"{m.teacher_agreement_cal:.3f}", - "agreement on handled traffic", color="cyan") - stat("traces", f"{m.n_traces:,}") - stat("labels", str(len(m.label_space))) - stat("artifacts", str(result.artifact_dir)) - - for note in result.notes: - if note.startswith("Deployed") or note.startswith("Loaded"): - info(note) - else: - warn(note) - - if qr: - section("Per-Label Coverage") - label_slices = [s for s in qr.slices if s.slice_name.startswith("label:")] - for s in sorted(label_slices, key=lambda x: -x.handled_rate)[:15]: - bar_line(s.slice_name.replace("label:", ""), s.handled_rate, s.count) - - if qr.boundary_pairs: - print() - print(f" {c.BOLD}{c.YELLOW}Boundary Pairs{c.RESET} {c.DIM}({len(qr.boundary_pairs)} examples){c.RESET}") - print(f" {hr('·')}") - for bp in qr.boundary_pairs[:3]: - pair_block(bp.teacher_label, bp.handled_preview, bp.deferred_preview, - bp.handled_score, bp.deferred_score) - - # Generate HTML automatically - try: - html = generate_html_report(result.artifact_dir) - print() - success(f"HTML report → {c.CYAN}{html}{c.RESET}") - info(f"open {html}") - except Exception: - pass - - print() - - -def main(): - parser = argparse.ArgumentParser( - prog="tracer", - description="TRACER - Turn LLM traces into cost-efficient routing policies", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent("""\ - examples: - tracer demo Run a zero-setup demo - tracer fit traces.jsonl Fit a routing policy - tracer fit traces.jsonl --target 0.95 95%% parity target - tracer report .tracer Show policy manifest - tracer report-html .tracer Open HTML audit report - tracer sankey .tracer Generate Sankey routing diagram - tracer update new.jsonl Refit with new traces - """), - ) - sub = parser.add_subparsers(dest="command") - - sub.add_parser("demo", help="Zero-setup demo with synthetic data") - - p_scan = sub.add_parser( - "scan", - help="Day-one verdict: how much traffic is certifiably ML-shaped, with a 3D map") - p_scan.add_argument("traces", help="Path to traces JSONL file") - p_scan.add_argument("--target", type=float, default=0.90, - help="Target label-agreement to certify against (default: 0.90)") - p_scan.add_argument("--teacher-price-per-1k", type=float, default=None, - help="Your teacher $ per 1k calls, to estimate savings") - p_scan.add_argument("--monthly-calls", type=int, default=None, - help="Monthly call volume, to project monthly savings") - p_scan.add_argument("--html", default=None, - help="Also write a self-contained HTML report to this path") - p_scan.add_argument("--no-open", action="store_true", - help="Don't open the HTML report in a browser") - # Embeddings: local sentence-transformers by default; precomputed .npy or a - # custom HTTP endpoint optional. - p_scan.add_argument("--embed-model", default="all-MiniLM-L6-v2", - help="Local sentence-transformers model for embeddings " - "(default: all-MiniLM-L6-v2). Needs tracer-llm[embeddings].") - p_scan.add_argument("--embeddings", default=None, - help="Path to a precomputed embeddings .npy (skip embedding).") - p_scan.add_argument("--embed-url", default=None, - help="Use your own HTTP embedding endpoint instead of a local " - "model. POSTs JSON and reads the embedding from the response.") - p_scan.add_argument("--embed-header", action="append", default=None, metavar="K: V", - help="Header for --embed-url (repeatable), e.g. " - "--embed-header 'Authorization: Bearer ...'") - p_scan.add_argument("--embed-input-key", default="input", - help="Request JSON key for the text (default: input)") - p_scan.add_argument("--embed-output-key", default="embedding", - help="Response JSON key for the vector (default: embedding)") - p_scan.add_argument("--embed-batch-key", default=None, - help="If set, send all texts in one request under this key " - "(for batch endpoints).") - p_scan.add_argument("--viz-layout", default="pca", - choices=["pca", "umap", "tsne", "auto"], - help="3D layout for the HTML report. pca (default) is a dense " - "connected cloud; umap/tsne separate many-class data into " - "distinct clouds (umap needs umap-learn installed).") - p_scan.add_argument("--force", action="store_true", - help="Scan thin data anyway (< 1,000 traces). Coarsens the " - "clustering to concentrate held-out evidence and reports a " - "best-effort floor, not a guarantee. Warns loudly.") - - p_fit = sub.add_parser("fit", help="Fit a routing policy from traces") - p_fit.add_argument("traces", help="Path to traces JSONL file") - p_fit.add_argument("--artifact-dir", default=".tracer", - help="Output artifact directory (default: .tracer)") - p_fit.add_argument("--target", type=float, default=0.90, - help="Target teacher agreement, e.g. 0.90 (default)") - p_fit.add_argument("--trees", action="store_true", - help="Add the tree-based surrogates (decision tree, random " - "forest, extra-trees, gradient boosting). Off by default: " - "they are slower and the linear/MLP heads are usually enough.") - p_fit.add_argument("--skip", default="", - help="Comma-separated surrogate models to skip from the zoo " - "(e.g. dt,rf,et,gbt,mlp_1h).") - - p_report = sub.add_parser("report", help="Show artifact manifest as JSON") - p_report.add_argument("artifact_dir", nargs="?", default=".tracer") - - p_html = sub.add_parser("report-html", - help="Generate HTML audit report and open in browser") - p_html.add_argument("artifact_dir", nargs="?", default=".tracer") - p_html.add_argument("--output", default=None, - help="Output path (default: /report.html)") - p_html.add_argument("--no-open", action="store_true", - help="Don't open browser automatically") - - p_sankey = sub.add_parser("sankey", - help="Generate a Sankey routing diagram (requires plotly)") - p_sankey.add_argument("artifact_dir", nargs="?", default=".tracer") - p_sankey.add_argument("--output", default=None, - help="Output path (default: /sankey.)") - p_sankey.add_argument("--format", default="html", choices=["html", "png", "svg"], - help="Output format (default: html)") - p_sankey.add_argument("--top-k", type=int, default=15, - help="Number of top labels to show (default: 15)") - p_sankey.add_argument("--no-open", action="store_true", - help="Don't open browser automatically (html only)") - - p_serve = sub.add_parser("serve", - help="Start a prediction HTTP server") - p_serve.add_argument("artifact_dir", nargs="?", default=".tracer", - help="Artifact directory (default: .tracer)") - p_serve.add_argument("--host", default="0.0.0.0", help="Bind address (default: 0.0.0.0)") - p_serve.add_argument("--port", type=int, default=8000, help="Port (default: 8000)") - - p_update = sub.add_parser("update", - help="Refit with new traces (continual learning)") - p_update.add_argument("traces", help="Path to new traces JSONL file") - p_update.add_argument("--artifact-dir", default=".tracer") - - # Cloud command group: drive Tracer Cloud (app.tracerml.ai) from the CLI, - # mirroring the web UI. Defined in tracer.cloud.cli. - from ..cloud import cli as cloud_cli - p_cloud = sub.add_parser("cloud", help="Drive Tracer Cloud from the terminal (login, tracers, train, route…)") - cloud_cli.build_parser(p_cloud) - - args = parser.parse_args() - - dispatch = { - "scan": _cmd_scan, - "fit": _cmd_fit, - "report": _cmd_report, - "report-html": _cmd_report_html, - "sankey": _cmd_sankey, - "serve": _cmd_serve, - "update": _cmd_update, - "demo": _cmd_demo, - } - - if args.command == "cloud": - from ..cloud import cli as cloud_cli - cloud_cli.run(args) - elif args.command in dispatch: - dispatch[args.command](args) - else: - parser.print_help() - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/tracer/cloud/__init__.py b/src/tracer/cloud/__init__.py deleted file mode 100644 index 06c6bda..0000000 --- a/src/tracer/cloud/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Tracer Cloud client + CLI command group. - -`tracer cloud ` mirrors the Tracer Cloud web UI from the -terminal: log in once, then create/train/retrain/promote tracers, run live -routing queries, manage test batteries, models, ingest keys, billing and -analytics. Authentication is a Tracer Cloud session (the same identity the -browser uses), presented to the web API as an `Authorization: Bearer ` -header. -""" - -from .client import CloudClient, CloudError # noqa: F401 diff --git a/src/tracer/cloud/cli.py b/src/tracer/cloud/cli.py deleted file mode 100644 index b875e08..0000000 --- a/src/tracer/cloud/cli.py +++ /dev/null @@ -1,1000 +0,0 @@ -"""`tracer cloud `, drive Tracer Cloud from the terminal. - -Mirrors the web UI: login, then manage tracers, training, routing, tests, -models, observability keys, billing and analytics. Reads (list/get) use the -same read views the dashboard reads; writes (create/train/promote/…) -call the same web API routes the dashboard posts to, authenticated with the -user's Tracer Cloud session as a bearer token. -""" - -from __future__ import annotations - -import getpass -import json as _json -import os -import re -import sys -import time -from typing import Any - -from .client import CloudClient, CloudError - -# ANSI helpers (no dependency on the OSS color module to keep this standalone). -_BOLD = "\033[1m" -_DIM = "\033[2m" -_GREEN = "\033[32m" -_RED = "\033[31m" -_CYAN = "\033[36m" -_RESET = "\033[0m" - - -def _supports_color() -> bool: - return sys.stdout.isatty() and os.environ.get("NO_COLOR") is None - - -def _c(s: str, code: str) -> str: - return f"{code}{s}{_RESET}" if _supports_color() else s - - -def _print_json(obj: Any) -> None: - print(_json.dumps(obj, indent=2, default=str)) - - -def _table(rows: list[dict[str, Any]], cols: list[str]) -> None: - if not rows: - print(_c("(none)", _DIM)) - return - widths = {c: len(c) for c in cols} - cells = [] - for r in rows: - row = {} - for c in cols: - v = r.get(c) - s = "" if v is None else str(v) - if len(s) > 48: - s = s[:45] + "…" - row[c] = s - widths[c] = max(widths[c], len(s)) - cells.append(row) - header = " ".join(_c(c.ljust(widths[c]), _BOLD) for c in cols) - print(header) - print(_c(" ".join("-" * widths[c] for c in cols), _DIM)) - for row in cells: - print(" ".join(row[c].ljust(widths[c]) for c in cols)) - - -# ----- tenant / tracer resolution -------------------------------------------- -def _resolve_tenant(client: CloudClient, want: str | None) -> dict[str, Any]: - tenants = client.db("my_tenants", select="*", order="created_at.asc") - if not tenants: - raise CloudError("no workspaces found for this account") - want = want or os.environ.get("TRACER_TENANT") - if want: - for t in tenants: - if want in (t.get("id"), t.get("slug"), t.get("name")): - return t - raise CloudError(f"workspace not found: {want}") - return tenants[0] - - -_UUID_RE = re.compile( - r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I -) - - -def _resolve_tracer(client: CloudClient, ref: str) -> dict[str, Any]: - # A UUID ref is queried by id; anything else is treated as a slug. The id - # column is uuid-typed, so sending a non-uuid as an id filter makes - # PostgREST 400 ("invalid input syntax for type uuid") before any slug - # fallback could run -- so pick the right column up front. - key = "id" if _UUID_RE.match(ref or "") else "slug" - rows = client.db("tracer_summary", select="*", filters={key: f"eq.{ref}"}) - if not rows and key == "id": - rows = client.db("tracer_summary", select="*", filters={"slug": f"eq.{ref}"}) - if not rows: - raise CloudError(f"tracer not found: {ref}") - return rows[0] - - -# ===== command handlers ======================================================= -def _cmd_login(client: CloudClient, args) -> None: - if args.url: - client.cfg["base_url"] = args.url - if args.auth_url: - client.cfg["auth_url"] = args.auth_url - - # Password grant only when credentials are explicitly supplied (headless - # / CI). Otherwise use the browser device flow, which opens a tab to pick - # the account and hands the session back to a localhost callback. - if args.email or args.password: - email = args.email or input("Email: ").strip() - password = args.password or getpass.getpass("Password: ") - client.login_password(email, password) - print(_c("✓", _GREEN), f"logged in as {email}") - print(_c(f" cloud: {client.base_url}", _DIM)) - return - - _login_browser(client) - - -def _login_browser(client: CloudClient) -> None: - """Open the browser, capture the Tracer Cloud session on a localhost callback.""" - import http.server - import secrets - import threading - import urllib.parse - import webbrowser - - state = secrets.token_urlsafe(16) - captured: dict[str, Any] = {} - done = threading.Event() - - class Handler(http.server.BaseHTTPRequestHandler): - def log_message(self, *a): # silence - pass - - def do_GET(self): # noqa: N802 - q = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) - ok = q.get("state", [None])[0] == state and "session" in q - if ok: - import base64 - - try: - raw = base64.b64decode(q["session"][0]) - captured.update(_json.loads(raw)) - except Exception: # noqa: BLE001 - ok = False - self.send_response(200) - self.send_header("Content-Type", "text/html") - self.end_headers() - msg = "Connected. You can close this tab and return to the terminal." if ok \ - else "Login failed or cancelled. Return to the terminal." - self.wfile.write( - f"" - f"

Tracer CLI

{msg}

".encode() - ) - done.set() - - srv = http.server.HTTPServer(("127.0.0.1", 0), Handler) - port = srv.server_address[1] - threading.Thread(target=srv.serve_forever, daemon=True).start() - - url = ( - f"{client.base_url}/app/cli/authorize?" - + urllib.parse.urlencode({"port": port, "state": state, "mode": "session"}) - ) - print("Opening your browser to sign in…") - print(_c(f" {url}", _DIM)) - print(_c(" (if it didn't open, paste that URL into your browser)", _DIM)) - try: - webbrowser.open(url) - except Exception: # noqa: BLE001 - pass - - if not done.wait(timeout=300): - srv.shutdown() - raise CloudError("login timed out; no callback received") - srv.shutdown() - - if not captured.get("access_token"): - raise CloudError("login failed: no session returned") - client._store_session(captured, email=captured.get("email")) - print(_c("✓", _GREEN), f"logged in as {captured.get('email') or 'user'}") - print(_c(f" cloud: {client.base_url}", _DIM)) - - -def _cmd_logout(client: CloudClient, args) -> None: - client.logout() - print(_c("✓", _GREEN), "logged out") - - -def _cmd_whoami(client: CloudClient, args) -> None: - user = client.auth_user() - tenants = client.db("my_tenants", select="*", order="created_at.asc") - if args.json: - _print_json({"user": user, "tenants": tenants}) - return - print(_c("User", _BOLD), user.get("email"), _c(f"({user.get('id')})", _DIM)) - print(_c("Cloud", _BOLD), client.base_url) - print(_c("Workspaces", _BOLD)) - _table(tenants, ["id", "name", "slug", "kind"]) - - -def _cmd_tenants(client: CloudClient, args) -> None: - tenants = client.db("my_tenants", select="*", order="created_at.asc") - if args.json: - _print_json(tenants) - return - _table(tenants, ["id", "name", "slug", "kind"]) - - -def _cmd_tenant_delete(client: CloudClient, args) -> None: - tenant = _resolve_tenant(client, args.tenant) - name = tenant.get("name") or tenant.get("slug") or tenant["id"] - if tenant.get("kind") == "personal": - raise CloudError("personal workspaces cannot be deleted (only company workspaces)") - # SOTA destructive-action guard: require the exact name, typed. - if args.confirm != name: - raise CloudError( - f"refusing to delete: pass --confirm \"{name}\" to confirm deleting this workspace" - ) - ok = client.rpc("delete_company_tenant", {"_tenant": tenant["id"]}) - deleted = ok[0] if isinstance(ok, list) and ok else ok - if deleted is True: - print(_c("✓", _GREEN), "deleted workspace", name) - else: - raise CloudError("workspace not found or not deletable") - - -def _cmd_tracers(client: CloudClient, args) -> None: - rows = client.db("tracer_summary", select="*", order="created_at.desc", limit=args.limit) - if args.json: - _print_json(rows) - return - _table(rows, ["id", "name", "slug", "status", "current_version_tag"]) - print(_c(f"\n{len(rows)} tracer(s)", _DIM)) - - -def _cmd_get(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - versions = client.db( - "tracer_versions", - select="id,version_tag,status,coverage_cal,teacher_agreement_cal,selected_method,created_at", - filters={"tracer_id": f"eq.{t['id']}"}, - order="created_at.desc", - limit=20, - ) - if args.json: - _print_json({"tracer": t, "versions": versions}) - return - print(_c(t.get("name", "tracer"), _BOLD), _c(f"({t.get('slug')})", _DIM)) - print(f" id {t.get('id')}") - print(f" status {t.get('status')}") - print(f" current {t.get('current_version_tag')} ({t.get('current_version_status')})") - if t.get("current_pipeline_label"): - print(f" pipeline {t.get('current_pipeline_label')}") - print(_c("\nVersions", _BOLD)) - _table(versions, ["id", "version_tag", "status", "coverage_cal", "teacher_agreement_cal", "selected_method"]) - - -def _parse_teacher(s: str | None) -> dict[str, str] | None: - """Accept 'provider:model_id' or a JSON object.""" - if not s: - return None - s = s.strip() - if s.startswith("{"): - return _json.loads(s) - if ":" in s: - provider, model_id = s.split(":", 1) - # label matches the dashboard's ModelMenuEntry shape the create routes expect. - return {"provider": provider, "model_id": model_id, "label": model_id} - raise CloudError("teacher must be 'provider:model_id' or a JSON object") - - -def _cmd_create(client: CloudClient, args) -> None: - tenant = _resolve_tenant(client, args.tenant) - teacher = _parse_teacher(args.teacher) - label_space = [s.strip() for s in args.label_space.split(",") if s.strip()] if args.label_space else None - - if args.mode == "auto": - if not teacher: - raise CloudError("auto mode needs --teacher provider:model_id") - body = { - "tenant_id": tenant["id"], - "name": args.name, - "teacher": teacher, - "threshold": args.threshold, - } - if label_space: - body["label_space"] = label_space - if args.system_prompt: - body["system_prompt"] = args.system_prompt - res = client.api("POST", "/api/tracers/auto-create", json_body=body) - else: # labelled / unlabelled upload via quick-create (multipart) - if not args.file: - raise CloudError("quick/unlabelled mode needs --file ") - if not teacher: - raise CloudError("provide --teacher provider:model_id") - fields = { - "tenant_id": tenant["id"], - "name": args.name, - "path": "unlabelled" if args.mode == "unlabelled" else "traces", - "teacher": _json.dumps(teacher), - "modality": args.modality, - } - if label_space: - fields["label_space"] = _json.dumps(label_space) - if args.system_prompt: - fields["system_prompt"] = args.system_prompt - res = client.api_multipart("/api/tracers/quick-create", fields, args.file) - if args.json: - _print_json(res) - return - print(_c("✓", _GREEN), "tracer queued") - _print_json(res) - if args.wait and isinstance(res, dict) and res.get("tracer_id"): - _wait_for_training(client, res["tracer_id"]) - - -def _cmd_rename(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - res = client.api("PATCH", f"/api/tracers/{t['id']}", json_body={"name": args.name}) - print(_c("✓", _GREEN), "renamed ->", res.get("name", args.name)) - - -def _cmd_delete(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - confirm = args.confirm or t.get("name") - client.api("DELETE", f"/api/tracers/{t['id']}", json_body={"confirm_name": confirm}) - print(_c("✓", _GREEN), "deleted", t.get("name")) - - -def _cmd_retrain(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - body: dict[str, Any] = {"mode": args.mode} - if args.trace_ids: - body["trace_ids"] = [s.strip() for s in args.trace_ids.split(",") if s.strip()] - if args.alpha is not None: - body["alpha"] = args.alpha - res = client.api("POST", f"/api/tracers/{t['id']}/retrain", json_body=body) - print(_c("✓", _GREEN), "retrain queued:", res) - if args.wait: - _wait_for_training(client, t["id"]) - - -def _cmd_auto_retrain(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - body = {"enabled": not args.disable, "threshold": args.threshold} - res = client.api("POST", f"/api/tracers/{t['id']}/auto-retrain", json_body=body) - print(_c("✓", _GREEN), "auto-retrain:", res) - - -def _cmd_promote(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - body = {"version_id": args.version} if args.version else {} - res = client.api("POST", f"/api/tracers/{t['id']}/promote", json_body=body) - print(_c("✓", _GREEN), res.get("action", "promoted"), res) - - -def _cmd_training(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - res = client.api("GET", f"/api/tracers/{t['id']}/training/state") - _print_json(res) - - -def _cmd_training_cancel(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - res = client.api("POST", f"/api/tracers/{t['id']}/training/cancel") - print(_c("✓", _GREEN), "cancel requested:", res) - - -def _wait_for_training(client: CloudClient, tracer_id: str, timeout: float = 900.0) -> None: - print(_c("waiting for training…", _DIM)) - start = time.time() - last = None - while time.time() - start < timeout: - try: - st = client.api("GET", f"/api/tracers/{tracer_id}/training/state") - except CloudError: - time.sleep(5) - continue - status = st.get("status") - if status != last: - print(f" status: {_c(str(status), _CYAN)}") - last = status - if status in ("ready", "failed", "error"): - mark = _GREEN if status == "ready" else _RED - print(_c("✓" if status == "ready" else "✗", mark), "training", status) - return - time.sleep(6) - print(_c("(timed out waiting; check `tracer cloud training`)", _DIM)) - - -def _cmd_route(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - body: dict[str, Any] = {"input": args.input} - if args.race: - body["race"] = True - params = {"mode": "lite"} if args.lite else None - res = client.api( - "POST", f"/api/tracers/{t['id']}/route-query", json_body=body, params=params - ) - if args.json: - _print_json(res) - return - deferred = res.get("deferred") - decision = "deferred to LLM" if deferred else "handled by ML" - decision_c = _RED if deferred else _GREEN - print(_c("routing", _BOLD), _c(decision, decision_c), - _c("(OOD escalated)" if res.get("ood_escalated") else "", _DIM)) - print(_c("intent ", _BOLD), res.get("dominant_intent")) - # routed_to is an object now ({provider,model,label,tier,reason}); pull .model. - _rt = res.get("routed_to") - _model = res.get("effective_model") or (_rt.get("model") if isinstance(_rt, dict) else _rt) - print(_c("model ", _BOLD), _model, _c(f"cell={res.get('cell_id')}", _DIM)) - if res.get("conformal_upper") is not None: - print(_c("conf. ", _BOLD), f"upper={res.get('conformal_upper')}") - lat = (res.get("route_ms") or 0) + (res.get("embed_ms") or 0) - print(_c("cost ", _BOLD), f"${res.get('cost_usd')}", _c(f"{lat:.0f}ms", _DIM)) - if args.show_exemplars and res.get("exemplars"): - print(_c("exemplars", _BOLD)) - for ex in res["exemplars"][:5]: - print(" -", (ex if isinstance(ex, str) else ex.get("input", ex))) - - -def _cmd_analytics(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - res = client.api("GET", f"/api/tracers/{t['id']}/analytics", params={"range": args.range}) - _print_json(res) - - -def _cmd_label_space(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - # No mutation flags -> read-only view of the current label space. - if not args.set and args.strict is None: - rows = client.db( - "tracer_versions", - select="id,version_tag,label_space,label_space_strict,status", - filters={"tracer_id": f"eq.{t['id']}"}, - order="created_at.desc", - limit=1, - ) - cur = rows[0] if rows else {} - if args.json: - _print_json(cur) - return - ls = cur.get("label_space") or [] - print(_c("label space", _BOLD), f"({len(ls)} labels, strict={cur.get('label_space_strict')})") - for lab in ls: - print(" ", lab) - return - body: dict[str, Any] = {} - if args.set: - body["label_space"] = [s.strip() for s in args.set.split(",") if s.strip()] - if args.strict is not None: - body["label_space_strict"] = args.strict - res = client.api("PATCH", f"/api/tracers/{t['id']}/label-space", json_body=body) - print(_c("✓", _GREEN), "label space updated:", res) - - -def _cmd_batteries(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - if args.action == "list": - res = client.api("GET", f"/api/tracers/{t['id']}/batteries") - if args.json: - _print_json(res) - return - _table(res.get("batteries", []), ["id", "name", "n_cases", "created_at"]) - elif args.action == "create": - cases = _json.loads(open(args.cases).read()) if args.cases else [] - body = {"name": args.name, "cases": cases} - res = client.api("POST", f"/api/tracers/{t['id']}/batteries", json_body=body) - bid = res.get("battery_id") if isinstance(res, dict) else None - print(_c("✓", _GREEN), "battery created:", bid or res, _c(f"({res.get('n_cases')} cases)" if isinstance(res, dict) else "", _DIM)) - elif args.action == "run": - params = {"version": args.version} if args.version else None - res = client.api( - "POST", f"/api/tracers/{t['id']}/batteries/{args.battery}/run", params=params - ) - if args.json: - _print_json(res) - return - summary = res.get("summary", {}) - print(_c("battery run", _BOLD), - f"pass={summary.get('n_pass')} fail={summary.get('n_fail')} deferred={summary.get('n_deferred')}") - - -def _cmd_models(client: CloudClient, args) -> None: - if args.action == "list": - rows = client.db("model_catalog", select="*", order="provider.asc") - if args.json: - _print_json(rows) - return - _table(rows, ["provider", "model_id", "display_label", "tier_hint"]) - elif args.action == "add": - tenant = _resolve_tenant(client, args.tenant) - body = { - "tenant_id": tenant["id"], - "name": args.name, - "display_label": args.label or args.name, - "endpoint_url": args.endpoint, - "upstream_model_id": args.upstream, - "api_key": args.api_key, - "model_kind": args.kind, - "cost_input_per_1m_usd": args.cost_in, - "cost_output_per_1m_usd": args.cost_out, - "visibility": args.visibility, - } - res = client.api("POST", "/api/library/custom-models", json_body=body) - print(_c("✓", _GREEN), "custom model added:", res) - elif args.action == "delete": - client.api("DELETE", f"/api/library/custom-models?id={args.id}") - print(_c("✓", _GREEN), "deleted", args.id) - - -def _cmd_keys(client: CloudClient, args) -> None: - """Per-tracer gateway API keys (trc_*).""" - t = _resolve_tracer(client, args.tracer) - base = f"/api/tracers/{t['id']}/api-keys" - if args.action == "list": - res = client.api("GET", base) - if args.json: - _print_json(res) - return - _table(res.get("keys", []), ["id", "name", "prefix", "last4", "last_used_at", "revoked_at"]) - elif args.action == "create": - res = client.api("POST", base, json_body={"name": args.name}) - if args.json: - _print_json(res) - return - print(_c("✓ key (shown once):", _GREEN), res.get("full_key")) - elif args.action == "revoke": - client.api("DELETE", f"{base}/{args.id}") - print(_c("✓", _GREEN), "revoked", args.id) - - -def _cmd_ingest_keys(client: CloudClient, args) -> None: - """Per-tenant observability ingest keys (trobs_*).""" - if args.action == "list": - tenant = _resolve_tenant(client, args.tenant) - res = client.api("GET", "/api/observe/keys", params={"tenant": tenant["id"]}) - if args.json: - _print_json(res) - return - _table(res.get("keys", []), ["id", "name", "prefix", "last4", "last_used_at", "revoked_at"]) - elif args.action == "create": - tenant = _resolve_tenant(client, args.tenant) - res = client.api("POST", "/api/observe/keys", json_body={"tenant_id": tenant["id"], "name": args.name}) - if args.json: - _print_json(res) - return - print(_c("✓ ingest key (shown once):", _GREEN), res.get("full_key")) - elif args.action == "revoke": - client.api("DELETE", f"/api/observe/keys/{args.id}") - print(_c("✓", _GREEN), "revoked", args.id) - - -def _cmd_ingest(client: CloudClient, args) -> None: - """Send observed trace events to Tracer Cloud. - - Auto-routes by key type, matching the two UI paths: - * trobs_* (tenant ingest key) -> POST /v1/observe (tenant-wide) - * trc_* (per-tracer gateway) -> POST /v1/ingest (bound to a tracer) - """ - events = _json.loads(open(args.events).read()) - if isinstance(events, dict): - events = [events] - if args.key.startswith("trobs_"): - res = client.keyed("POST", "/v1/observe", args.key, json_body={"events": events}) - else: - res = client.keyed( - "POST", "/v1/ingest", args.key, json_body={"source": args.source, "events": events} - ) - print(_c("✓", _GREEN), "ingested:", res) - - -def _cmd_billing(client: CloudClient, args) -> None: - tenant = _resolve_tenant(client, args.tenant) - res = client.rpc("get_tenant_billing_summary", {"_tenant": tenant["id"]}) - row = res[0] if isinstance(res, list) and res else res - if args.json: - _print_json(row) - return - print(_c(f"Billing · {tenant.get('name')}", _BOLD)) - if isinstance(row, dict): - print(f" balance ${row.get('balance_usd')}") - print(f" credits ${row.get('credits_total')}") - print(f" usage ${row.get('usage_total')}") - print(f" saved ${row.get('saved_total')}") - print(f" ML queries {row.get('ml_queries')} deferred {row.get('deferred_queries')}") - - -def _cmd_scan(client: CloudClient, args) -> None: - # /api/scan is public (matches the anonymous web /scan flow), so the scan - # works logged out; the bearer is attached only if a session exists. - res = client.api_multipart("/api/scan", {}, args.file, auth="optional") - # The agent may need to disambiguate the file format. Re-submit with answers - # (CLI flags override the server's best-guess defaults) so the scan is - # non-interactive instead of silently no-opping on an ambiguous file. - if isinstance(res, dict) and res.get("needs_clarification"): - answers = dict(res.get("defaults") or {}) - if args.input_field: - answers["input_field"] = args.input_field - if args.label_field: - answers["label_field"] = args.label_field - if args.task: - answers["task"] = args.task - print(_c("clarifying format", _DIM), _json.dumps(answers)) - res = client.api_multipart( - "/api/scan", {"answers": _json.dumps(answers)}, args.file, auth="optional" - ) - job_id = res.get("job_id") if isinstance(res, dict) else None - if not job_id: - _print_json(res) - return - print(_c("scan queued", _DIM), job_id) - print(_c("link", _DIM), f"{client.base_url}/scan/{job_id}") - start = time.time() - while time.time() - start < 300: - st = client.api("GET", f"/api/scan/{job_id}", auth="optional") - if st.get("status") == "ready": - print(_c("✓", _GREEN), "scan ready") - if getattr(args, "json", False): - _print_json(st) - else: - summ = st.get("summary") or {} - proj = st.get("projection") or {} - print(f" type {st.get('data_type')}") - print(f" rows {st.get('n_rows')}") - cs = summ.get("certifiable_share") - if cs is not None: - print(f" certifiable {round(float(cs) * 100)}% of traffic") - print(f" points {len(proj.get('points') or [])}") - print(_c("view", _DIM), f"{client.base_url}/scan/{job_id}") - return - if st.get("error"): - raise CloudError(f"scan failed: {st['error']}") - time.sleep(4) - print(_c("(scan still running; re-poll later)", _DIM)) - - -def _sample_inputs(path: str, n: int = 25) -> list[str]: - """Up to n representative input strings from a jsonl/csv file, for the - onboarding planner. Tolerant of common input-field aliases.""" - keys = ("input", "query", "text", "prompt", "question") - out: list[str] = [] - try: - with open(path, "r", encoding="utf-8", errors="replace") as f: - head = f.readline() - f.seek(0) - if "," in head and not head.lstrip().startswith("{"): - import csv - for row in csv.DictReader(f): - v = next((row[k] for k in keys if row.get(k)), None) or next( - (x for x in row.values() if x and x.strip()), None - ) - if v: - out.append(str(v)[:500]) - if len(out) >= n: - break - else: - for line in f: - line = line.strip() - if not line: - continue - try: - row = _json.loads(line) - except Exception: - continue - if isinstance(row, dict): - v = next((row[k] for k in keys if isinstance(row.get(k), str)), None) - if v: - out.append(v[:500]) - elif isinstance(row, str): - out.append(row[:500]) - if len(out) >= n: - break - except OSError as e: - raise CloudError(f"could not read {path}: {e}") - return out - - -def _cmd_onboard(client: CloudClient, args) -> None: - tenant = _resolve_tenant(client, args.tenant) - samples = _sample_inputs(args.file) - body: dict[str, Any] = { - "tenant_id": tenant["id"], - "use_case_description": args.describe, - "sample_inputs": samples, - } - if args.teacher: - body["teacher_model_id"] = args.teacher - if args.labels: - body["label_space"] = [s.strip() for s in args.labels.split(",") if s.strip()] - res = client.api("POST", "/api/tracers/auto-onboard", json_body=body) - if args.json: - _print_json(res) - return - plan = res.get("plan", {}) if isinstance(res, dict) else {} - print(_c("onboarding plan", _BOLD), _c(f"({len(samples)} samples)", _DIM)) - for k in ("task", "modality", "embedder", "cascade_mode", "dual_embed", "alpha"): - if plan.get(k) is not None: - print(f" {k:<13}{plan.get(k)}") - if plan.get("label_space"): - print(f" {'label_space':<13}{', '.join(plan['label_space'])}") - if plan.get("system_prompt"): - print(f" {'system':<13}{str(plan['system_prompt'])[:200]}") - if plan.get("rationale"): - print(_c("why", _DIM), str(plan["rationale"])[:400]) - pl = res.get("planner", {}) if isinstance(res, dict) else {} - if pl.get("cost_usd") is not None: - print(_c(f"planner {pl.get('model')} · ${pl.get('cost_usd')} · {pl.get('latency_ms')}ms", _DIM)) - - -def _cmd_traces(client: CloudClient, args) -> None: - t = _resolve_tracer(client, args.tracer) - body: dict[str, Any] = {"limit": args.limit, "order": "random" if args.random else "recent"} - if args.errors: - body["errors_only"] = True - if args.tier: - body["tier"] = args.tier - if args.since_minutes: - body["since_minutes"] = args.since_minutes - if args.cell is not None: - body["cell_id"] = args.cell - res = client.api("POST", f"/api/tracers/{t['id']}/traces/select", json_body=body) - ids = res.get("trace_ids", []) if isinstance(res, dict) else [] - if args.json: - _print_json(res) - return - if args.ids_only: - # comma-separated, for: retrain --trace-ids "$(tracer cloud traces X --errors --ids-only)" - print(",".join(ids)) - return - print(_c(f"{len(ids)} trace(s)", _BOLD)) - for i in ids: - print(" ", i) - - -def _cmd_create_bulk(client: CloudClient, args) -> None: - import mimetypes - tenant = _resolve_tenant(client, args.tenant) - fname = os.path.basename(args.file) - ctype = mimetypes.guess_type(fname)[0] or ( - "application/x-ndjson" if fname.endswith((".jsonl", ".ndjson")) else "text/plain" - ) - try: - data = open(args.file, "rb").read() - except OSError as e: - raise CloudError(f"could not read {args.file}: {e}") - size = len(data) - print(_c("presigning", _DIM), f"{fname} ({size / 1_048_576:.1f} MB)") - pre = client.api("POST", "/api/tracers/uploads/presign-bulk", json_body={ - "tenant_id": tenant["id"], "filename": fname, "content_type": ctype, "size_bytes": size, - }) - url = pre.get("presigned_url") if isinstance(pre, dict) else None - storage_path = pre.get("storage_path") if isinstance(pre, dict) else None - if not url or not storage_path: - raise CloudError(f"presign failed: {pre}") - print(_c("uploading", _DIM), "to storage...") - client.put_url(url, data, ctype) - body: dict[str, Any] = { - "tenant_id": tenant["id"], - "name": args.name, - "teacher": _json.dumps(_parse_teacher(args.teacher)) if args.teacher else None, - "storage_path": storage_path, - "filename": fname, - "content_type": ctype, - } - if args.path: - body["path"] = args.path - if args.labels: - body["label_space"] = [s.strip() for s in args.labels.split(",") if s.strip()] - if args.system_prompt: - body["system_prompt"] = args.system_prompt - res = client.api("POST", "/api/tracers/quick-create-bulk", json_body=body) - tid = res.get("tracer_id") if isinstance(res, dict) else None - rows = res.get("estimated_total_rows") if isinstance(res, dict) else None - print(_c("✓", _GREEN), "tracer created:", tid or res, _c(f"(~{rows} rows)" if rows else "", _DIM)) - if args.wait and tid: - _wait_for_training(client, tid) - - -# ===== argparse wiring ======================================================== -def build_parser(p) -> None: - import argparse - - p.add_argument("--json", action="store_true", help="Raw JSON output") - # Shared parent so `--json` is accepted AFTER the subcommand too - # (e.g. `tracer cloud get --json`), not only before it. - common = argparse.ArgumentParser(add_help=False) - common.add_argument("--json", action="store_true", help="Raw JSON output") - sub = p.add_subparsers(dest="cloud_command", parser_class=lambda **kw: argparse.ArgumentParser(parents=[common], **kw)) - - pl = sub.add_parser("login", help="Log in to Tracer Cloud") - pl.add_argument("--email") - pl.add_argument("--password") - pl.add_argument("--url", help="Cloud base URL (default https://app.tracerml.ai)") - pl.add_argument("--auth-url", dest="auth_url") - - sub.add_parser("logout", help="Clear the stored session") - sub.add_parser("whoami", help="Show the logged-in user + workspaces") - sub.add_parser("tenants", help="List workspaces") - - ptd = sub.add_parser("tenant-delete", help="Delete an EMPTY company workspace (owner only)") - ptd.add_argument("tenant", help="workspace id / slug / name") - ptd.add_argument("--confirm", required=True, help="the workspace name, typed exactly, to confirm") - - pt = sub.add_parser("tracers", help="List tracers") - pt.add_argument("--limit", type=int, default=50) - - pg = sub.add_parser("get", help="Show a tracer + its versions") - pg.add_argument("tracer", help="tracer id or slug") - - pc = sub.add_parser("create", help="Create a tracer (and start training)") - pc.add_argument("name") - pc.add_argument("--mode", choices=["quick", "unlabelled", "auto"], default="quick", - help="quick=labelled upload, unlabelled=teacher labels it, auto=watch-mode") - pc.add_argument("--file", help="traces file (quick/unlabelled)") - pc.add_argument("--teacher", help="teacher model 'provider:model_id'") - pc.add_argument("--label-space", dest="label_space", help="comma-separated labels") - pc.add_argument("--system-prompt", dest="system_prompt") - pc.add_argument("--modality", default="text") - pc.add_argument("--threshold", type=int, default=200, help="auto-mode retrain threshold") - pc.add_argument("--tenant", help="workspace id/slug/name (default: first)") - pc.add_argument("--wait", action="store_true", help="block until training finishes") - - pr = sub.add_parser("rename", help="Rename a tracer") - pr.add_argument("tracer") - pr.add_argument("name") - - pd = sub.add_parser("delete", help="Delete a tracer") - pd.add_argument("tracer") - pd.add_argument("--confirm", help="confirm name (defaults to the tracer's name)") - - prt = sub.add_parser("retrain", help="Retrain a tracer") - prt.add_argument("tracer") - prt.add_argument("--mode", choices=["reuse", "merge", "replace", "consume_orphans"], default="reuse") - prt.add_argument("--trace-ids", dest="trace_ids", help="comma-separated trace ids (replace/merge)") - prt.add_argument("--alpha", type=float, default=None) - prt.add_argument("--wait", action="store_true") - - par = sub.add_parser("auto-retrain", help="Enable/disable scheduled retrain") - par.add_argument("tracer") - par.add_argument("--threshold", type=int, default=200) - par.add_argument("--disable", action="store_true") - - pp = sub.add_parser("promote", help="Promote (or roll back) a version to prod") - pp.add_argument("tracer") - pp.add_argument("--version", help="version id (rollback target); omit to promote current") - - ptr = sub.add_parser("training", help="Show training state") - ptr.add_argument("tracer") - ptc = sub.add_parser("training-cancel", help="Cancel a running training job") - ptc.add_argument("tracer") - - pq = sub.add_parser("route", help="Run a live routing query (the dashboard's Live Query)") - pq.add_argument("tracer") - pq.add_argument("input") - pq.add_argument("--race", action="store_true") - pq.add_argument("--lite", action="store_true", help="skip the teacher call (show would_call)") - pq.add_argument("--show-exemplars", dest="show_exemplars", action="store_true", - help="print the matched cell's nearest training exemplars") - - pan = sub.add_parser("analytics", help="Query-event analytics") - pan.add_argument("tracer") - pan.add_argument("--range", default="7d", choices=["24h", "7d", "30d"]) - - pls = sub.add_parser("label-space", help="View/update the label space") - pls.add_argument("tracer") - pls.add_argument("--set", help="comma-separated labels") - pls.add_argument("--strict", dest="strict", action="store_true", default=None) - - pb = sub.add_parser("batteries", help="Test batteries (list/create/run)") - pb.add_argument("action", choices=["list", "create", "run"]) - pb.add_argument("tracer") - pb.add_argument("--name") - pb.add_argument("--cases", help="JSON file of cases (create)") - pb.add_argument("--battery", help="battery id (run)") - pb.add_argument("--version", help="version id to run against") - - pm = sub.add_parser("models", help="Model library (list/add/delete)") - pm.add_argument("action", choices=["list", "add", "delete"]) - pm.add_argument("--tenant") - pm.add_argument("--name") - pm.add_argument("--label") - pm.add_argument("--endpoint") - pm.add_argument("--upstream", help="upstream model id") - pm.add_argument("--api-key", dest="api_key") - pm.add_argument("--kind", choices=["llm", "embedding"], default="llm") - pm.add_argument("--cost-in", dest="cost_in", type=float, default=0.0) - pm.add_argument("--cost-out", dest="cost_out", type=float, default=0.0) - pm.add_argument("--visibility", choices=["workspace", "private", "org"], default="workspace") - pm.add_argument("--id", help="custom model id (delete)") - - pk = sub.add_parser("keys", help="Per-tracer gateway API keys") - pk.add_argument("action", choices=["list", "create", "revoke"]) - pk.add_argument("tracer") - pk.add_argument("--name") - pk.add_argument("--id", help="key id (revoke)") - - pik = sub.add_parser("ingest-keys", help="Per-workspace observability ingest keys") - pik.add_argument("action", choices=["list", "create", "revoke"]) - pik.add_argument("--tenant") - pik.add_argument("--name") - pik.add_argument("--id") - - pin = sub.add_parser("ingest", help="Send trace events (auto-routes by key prefix)") - pin.add_argument("--key", required=True, help="trobs_* -> /v1/observe, or trc_* -> /v1/ingest") - pin.add_argument("--events", required=True, help="JSON file of events") - pin.add_argument("--source", default="cli") - - pbi = sub.add_parser("billing", help="Workspace billing summary") - pbi.add_argument("--tenant") - - psc = sub.add_parser("scan", help="Public scan of a traces file") - psc.add_argument("file") - psc.add_argument("--input-field", dest="input_field", help="input column (skips the format clarify prompt)") - psc.add_argument("--label-field", dest="label_field", help="label/teacher column") - psc.add_argument("--task", choices=["classification", "other"], help="task type override") - - pon = sub.add_parser("onboard", help="Agentic onboarding plan from a sample file") - pon.add_argument("file") - pon.add_argument("--describe", required=True, help="one sentence: what you are labelling") - pon.add_argument("--tenant", help="workspace id/slug/name") - pon.add_argument("--teacher", help="teacher model id (e.g. gpt-5-mini)") - pon.add_argument("--labels", help="comma-separated label space (optional)") - - ptr = sub.add_parser("traces", help="List trace ids (feed into retrain --trace-ids)") - ptr.add_argument("tracer") - ptr.add_argument("--errors", action="store_true", help="only errored traces") - ptr.add_argument("--tier", help="filter by routed tier (e.g. ml, teacher)") - ptr.add_argument("--since-minutes", dest="since_minutes", type=int, help="only the last N minutes") - ptr.add_argument("--cell", type=int, help="filter by cell id") - ptr.add_argument("--limit", type=int, default=500) - ptr.add_argument("--random", action="store_true", help="random sample instead of most recent") - ptr.add_argument("--ids-only", dest="ids_only", action="store_true", help="comma-separated ids only (for piping)") - - pcb = sub.add_parser("create-bulk", help="Create a tracer from a large file (presigned upload)") - pcb.add_argument("file") - pcb.add_argument("--name", required=True) - pcb.add_argument("--teacher", help="provider:model_id (or JSON object)") - pcb.add_argument("--tenant", help="workspace id/slug/name") - pcb.add_argument("--path", help="upload path kind (server default if omitted)") - pcb.add_argument("--labels", help="comma-separated label space") - pcb.add_argument("--system-prompt", dest="system_prompt") - pcb.add_argument("--wait", action="store_true", help="wait for training to finish") - - -_DISPATCH = { - "login": _cmd_login, - "logout": _cmd_logout, - "whoami": _cmd_whoami, - "tenants": _cmd_tenants, - "tenant-delete": _cmd_tenant_delete, - "tracers": _cmd_tracers, - "get": _cmd_get, - "create": _cmd_create, - "rename": _cmd_rename, - "delete": _cmd_delete, - "retrain": _cmd_retrain, - "auto-retrain": _cmd_auto_retrain, - "promote": _cmd_promote, - "training": _cmd_training, - "training-cancel": _cmd_training_cancel, - "route": _cmd_route, - "analytics": _cmd_analytics, - "label-space": _cmd_label_space, - "batteries": _cmd_batteries, - "models": _cmd_models, - "keys": _cmd_keys, - "ingest-keys": _cmd_ingest_keys, - "ingest": _cmd_ingest, - "billing": _cmd_billing, - "scan": _cmd_scan, - "onboard": _cmd_onboard, - "traces": _cmd_traces, - "create-bulk": _cmd_create_bulk, -} - - -def run(args) -> None: - cmd = getattr(args, "cloud_command", None) - if not cmd: - print("usage: tracer cloud (try `tracer cloud --help`)") - sys.exit(1) - client = CloudClient.load() - # login/logout don't require an existing session; `scan` is a public - # endpoint (mirrors the anonymous web /scan) so it runs logged out too. - if cmd not in ("login", "logout", "scan") and not client.logged_in: - print(_c("not logged in.", _RED), "run `tracer cloud login` first.") - sys.exit(1) - handler = _DISPATCH.get(cmd) - if not handler: - print(f"unknown subcommand: {cmd}") - sys.exit(1) - try: - handler(client, args) - except CloudError as e: - print(_c("error:", _RED), e) - sys.exit(1) diff --git a/src/tracer/cloud/client.py b/src/tracer/cloud/client.py deleted file mode 100644 index 6c84829..0000000 --- a/src/tracer/cloud/client.py +++ /dev/null @@ -1,355 +0,0 @@ -"""Auth + thin REST client for Tracer Cloud. - -Stdlib-only (urllib) so the OSS package gains no runtime dependency. Holds a -Tracer Cloud session in ``~/.tracer/cloud.json`` and presents it to the -Tracer Cloud web API as a bearer token, exactly as the browser session does. -""" - -from __future__ import annotations - -import io -import json -import mimetypes -import os -import time -import urllib.error -import urllib.parse -import urllib.request -import uuid -from pathlib import Path -from typing import Any - -# The package ships ZERO infrastructure identifiers. Only the public Tracer -# Cloud base URL is baked in; the auth base URL + public client key are fetched -# from {base_url}/api/cli/config at login (so they can be rotated server-side -# without breaking installed CLIs). All overridable by env to point at a local -# dev server or a different deployment. -DEFAULT_BASE_URL = os.environ.get("TRACER_CLOUD_URL", "https://app.tracerml.ai") -DEFAULT_AUTH_URL = os.environ.get("TRACER_CLOUD_AUTH_URL", "") -DEFAULT_AUTH_KEY = os.environ.get("TRACER_CLOUD_AUTH_KEY", "") - -CONFIG_PATH = Path( - os.environ.get("TRACER_CLOUD_CONFIG", str(Path.home() / ".tracer" / "cloud.json")) -) - - -class CloudError(RuntimeError): - """A Cloud request failed; message is safe to print to the user.""" - - -# A real User-Agent: the default "Python-urllib/x" trips Cloudflare's bot -# protection on app.tracerml.ai (HTTP 403, CF error 1010). -USER_AGENT = "tracer-cli/0.3.3 (+https://tracerml.ai)" - - -def _http( - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - data: bytes | None = None, - timeout: float = 60.0, -) -> tuple[int, bytes, dict[str, str]]: - req = urllib.request.Request(url, data=data, method=method) - req.add_header("User-Agent", USER_AGENT) - req.add_header("Accept", "application/json") - for k, v in (headers or {}).items(): - req.add_header(k, v) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read(), dict(resp.headers) - except urllib.error.HTTPError as e: - return e.code, e.read(), dict(e.headers or {}) - except urllib.error.URLError as e: - raise CloudError(f"network error reaching {url}: {e.reason}") from e - - -def _json_or_text(body: bytes) -> Any: - if not body: - return None - try: - return json.loads(body.decode("utf-8")) - except (ValueError, UnicodeDecodeError): - return body.decode("utf-8", "replace") - - -class CloudClient: - def __init__(self, cfg: dict[str, Any] | None = None) -> None: - self.cfg = cfg or {} - - # ----- config persistence ------------------------------------------------- - @classmethod - def load(cls) -> "CloudClient": - if CONFIG_PATH.exists(): - try: - return cls(json.loads(CONFIG_PATH.read_text())) - except (ValueError, OSError): - pass - return cls({}) - - def save(self) -> None: - CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) - CONFIG_PATH.write_text(json.dumps(self.cfg, indent=2)) - try: - os.chmod(CONFIG_PATH, 0o600) - except OSError: - pass - - @property - def base_url(self) -> str: - return (self.cfg.get("base_url") or DEFAULT_BASE_URL).rstrip("/") - - @property - def auth_url(self) -> str: - return (self.cfg.get("auth_url") or DEFAULT_AUTH_URL).rstrip("/") - - @property - def auth_key(self) -> str: - return self.cfg.get("auth_key") or DEFAULT_AUTH_KEY - - @property - def logged_in(self) -> bool: - return bool(self.cfg.get("access_token")) - - # ----- auth ---------------------------------------------------------------- - def _ensure_auth_config(self) -> None: - """Fetch the public auth base URL + client key from the Tracer Cloud - bootstrap endpoint if we don't already have them. Keeps the package free - of any hardcoded infra identifiers; the values are public (the same the - browser ships) and cached into the local config.""" - if self.cfg.get("auth_url") and self.cfg.get("auth_key"): - return - if DEFAULT_AUTH_URL and DEFAULT_AUTH_KEY: # env override - self.cfg["auth_url"], self.cfg["auth_key"] = DEFAULT_AUTH_URL, DEFAULT_AUTH_KEY - return - status, body, _ = _http("GET", f"{self.base_url}/api/cli/config") - cfg = _json_or_text(body) - if status != 200 or not isinstance(cfg, dict) or not cfg.get("auth_url") or not cfg.get("auth_key"): - raise CloudError( - f"could not fetch CLI config from {self.base_url}/api/cli/config " - f"(status {status}); set --url or TRACER_CLOUD_URL" - ) - self.cfg["auth_url"] = cfg["auth_url"] - self.cfg["auth_key"] = cfg["auth_key"] - - def login_password(self, email: str, password: str) -> dict[str, Any]: - """Password grant -> store the Tracer Cloud session.""" - self._ensure_auth_config() - url = f"{self.auth_url}/auth/v1/token?grant_type=password" - status, body, _ = _http( - "POST", - url, - headers={"apikey": self.auth_key, "Content-Type": "application/json"}, - data=json.dumps({"email": email, "password": password}).encode(), - ) - payload = _json_or_text(body) - if status != 200 or not isinstance(payload, dict) or "access_token" not in payload: - msg = payload.get("error_description") if isinstance(payload, dict) else payload - raise CloudError(f"login failed: {msg or status}") - self._store_session(payload, email=email) - return payload - - def _store_session(self, payload: dict[str, Any], email: str | None = None) -> None: - self._ensure_auth_config() # persist auth config (covers browser login) - self.cfg["base_url"] = self.base_url - self.cfg["auth_url"] = self.auth_url - self.cfg["auth_key"] = self.auth_key - self.cfg["access_token"] = payload["access_token"] - self.cfg["refresh_token"] = payload.get("refresh_token") - # expires_at may be absolute (epoch) or we derive from expires_in. - exp = payload.get("expires_at") - if not exp and payload.get("expires_in"): - exp = int(time.time()) + int(payload["expires_in"]) - self.cfg["expires_at"] = exp - user = payload.get("user") or {} - self.cfg["email"] = email or user.get("email") - self.cfg["user_id"] = user.get("id") - self.save() - - def _refresh(self) -> None: - rt = self.cfg.get("refresh_token") - if not rt: - raise CloudError("session expired and no refresh token; run `tracer cloud login`") - self._ensure_auth_config() - url = f"{self.auth_url}/auth/v1/token?grant_type=refresh_token" - status, body, _ = _http( - "POST", - url, - headers={"apikey": self.auth_key, "Content-Type": "application/json"}, - data=json.dumps({"refresh_token": rt}).encode(), - ) - payload = _json_or_text(body) - if status != 200 or not isinstance(payload, dict) or "access_token" not in payload: - raise CloudError("session refresh failed; run `tracer cloud login` again") - self._store_session(payload, email=self.cfg.get("email")) - - def _access_token(self) -> str: - if not self.logged_in: - raise CloudError("not logged in; run `tracer cloud login`") - exp = self.cfg.get("expires_at") - if isinstance(exp, (int, float)) and time.time() > float(exp) - 60: - self._refresh() - return self.cfg["access_token"] - - def _auth_header(self, auth: str) -> dict[str, str]: - """Build the Authorization header for a request. - - auth="required" (default): a session is mandatory (raises if absent). - auth="optional": attach the bearer when logged in, otherwise call - anonymously -- for public endpoints (e.g. /api/scan) that work - without an account, matching the public web flow. - auth="none": never attach a bearer. - """ - if auth == "none": - return {} - if auth == "optional" and not self.logged_in: - return {} - return {"Authorization": f"Bearer {self._access_token()}"} - - def logout(self) -> None: - for k in ("access_token", "refresh_token", "expires_at", "email", "user_id"): - self.cfg.pop(k, None) - self.save() - - # ----- web API (session-authed routes, via bearer JWT) -------------------- - def api( - self, - method: str, - path: str, - *, - json_body: Any | None = None, - params: dict[str, Any] | None = None, - auth: str = "required", - ) -> Any: - url = f"{self.base_url}{path}" - if params: - url += "?" + urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) - headers = {**self._auth_header(auth), "Origin": self.base_url} - data = None - if json_body is not None: - headers["Content-Type"] = "application/json" - data = json.dumps(json_body).encode() - status, body, _ = _http(method, url, headers=headers, data=data) - payload = _json_or_text(body) - if status >= 400: - err = payload.get("error") if isinstance(payload, dict) else payload - raise CloudError(f"{method} {path} -> {status}: {err}") - return payload - - def api_multipart( - self, path: str, fields: dict[str, str], file_path: str | None, *, auth: str = "required" - ) -> Any: - """POST multipart/form-data (used by the upload + create routes).""" - boundary = f"----tracer{uuid.uuid4().hex}" - buf = io.BytesIO() - - def w(s: str) -> None: - buf.write(s.encode()) - - for name, value in fields.items(): - if value is None: - continue - w(f"--{boundary}\r\n") - w(f'Content-Disposition: form-data; name="{name}"\r\n\r\n') - w(f"{value}\r\n") - if file_path: - fp = Path(file_path) - ctype = mimetypes.guess_type(fp.name)[0] or "application/octet-stream" - w(f"--{boundary}\r\n") - w(f'Content-Disposition: form-data; name="file"; filename="{fp.name}"\r\n') - w(f"Content-Type: {ctype}\r\n\r\n") - buf.write(fp.read_bytes()) - w("\r\n") - w(f"--{boundary}--\r\n") - - headers = { - **self._auth_header(auth), - "Origin": self.base_url, - "Content-Type": f"multipart/form-data; boundary={boundary}", - } - status, body, _ = _http( - "POST", f"{self.base_url}{path}", headers=headers, data=buf.getvalue() - ) - payload = _json_or_text(body) - if status >= 400: - err = payload.get("error") if isinstance(payload, dict) else payload - raise CloudError(f"POST {path} -> {status}: {err}") - return payload - - def put_url(self, url: str, data: bytes, content_type: str) -> None: - """Raw PUT to an external presigned URL (R2 bulk upload). No auth header; - the URL itself carries the signature.""" - status, body, _ = _http("PUT", url, headers={"Content-Type": content_type}, data=data) - if status >= 400: - raise CloudError(f"presigned PUT -> {status}: {body[:200]!r}") - - # ----- Tracer Cloud data API (the read views the dashboard uses) ---------- - def db( - self, - view: str, - *, - select: str = "*", - filters: dict[str, str] | None = None, - order: str | None = None, - limit: int | None = None, - ) -> list[dict[str, Any]]: - q: dict[str, str] = {"select": select} - if filters: - q.update(filters) - if order: - q["order"] = order - if limit: - q["limit"] = str(limit) - self._ensure_auth_config() - url = f"{self.auth_url}/rest/v1/{view}?" + urllib.parse.urlencode(q) - headers = { - "apikey": self.auth_key, - "Authorization": f"Bearer {self._access_token()}", - "Accept": "application/json", - } - status, body, _ = _http("GET", url, headers=headers) - payload = _json_or_text(body) - if status >= 400: - msg = payload.get("message") if isinstance(payload, dict) else payload - raise CloudError(f"db read {view} -> {status}: {msg}") - return payload if isinstance(payload, list) else [] - - def rpc(self, fn: str, args: dict[str, Any]) -> Any: - self._ensure_auth_config() - url = f"{self.auth_url}/rest/v1/rpc/{fn}" - headers = { - "apikey": self.auth_key, - "Authorization": f"Bearer {self._access_token()}", - "Content-Type": "application/json", - "Accept": "application/json", - } - status, body, _ = _http("POST", url, headers=headers, data=json.dumps(args).encode()) - payload = _json_or_text(body) - if status >= 400: - msg = payload.get("message") if isinstance(payload, dict) else payload - raise CloudError(f"rpc {fn} -> {status}: {msg}") - return payload - - def auth_user(self) -> dict[str, Any]: - self._ensure_auth_config() - url = f"{self.auth_url}/auth/v1/user" - headers = {"apikey": self.auth_key, "Authorization": f"Bearer {self._access_token()}"} - status, body, _ = _http("GET", url, headers=headers) - payload = _json_or_text(body) - if status >= 400: - raise CloudError(f"whoami failed: {status}") - return payload if isinstance(payload, dict) else {} - - # ----- bearer-key endpoints (gateway/ingest, no session) ------------------ - def keyed(self, method: str, path: str, key: str, json_body: Any | None = None) -> Any: - headers = {"Authorization": f"Bearer {key}"} - data = None - if json_body is not None: - headers["Content-Type"] = "application/json" - data = json.dumps(json_body).encode() - status, body, _ = _http(method, f"{self.base_url}{path}", headers=headers, data=data) - payload = _json_or_text(body) - if status >= 400: - err = payload.get("error") if isinstance(payload, dict) else payload - raise CloudError(f"{method} {path} -> {status}: {err}") - return payload diff --git a/src/tracer/embeddings/index.py b/src/tracer/embeddings/index.py index 9e75f70..ea5e95e 100644 --- a/src/tracer/embeddings/index.py +++ b/src/tracer/embeddings/index.py @@ -116,9 +116,10 @@ def embed_texts( except RuntimeError as e: if "Numpy is not available" in str(e) or "numpy" in str(e).lower(): raise RuntimeError( - "PyTorch cannot convert tensors to NumPy. This usually means " - "your NumPy version is too new for your PyTorch version.\n" - "Fix with: pip install 'numpy<2.1'\n" + "PyTorch cannot convert tensors to NumPy. Your installed " + "PyTorch and NumPy builds may be incompatible.\n" + "Install compatible builds for your Python version, for example:\n" + " python -m pip install --upgrade torch 'numpy>=1.26.4,<3'\n" "Then restart your Python process / notebook kernel." ) from e raise diff --git a/src/tracer/runtime/serve.py b/src/tracer/runtime/serve.py index 7e0bfdb..d284f88 100644 --- a/src/tracer/runtime/serve.py +++ b/src/tracer/runtime/serve.py @@ -1,6 +1,7 @@ """Lightweight prediction server for a fitted TRACER policy. - tracer serve .tracer --port 8000 + import tracer + tracer.serve(".tracer", port=8000) Exposes: POST /predict {"embedding": [0.1, 0.2, ...]} → {"label", "decision", "accept_score", "stage"} diff --git a/src/tracer/scanner.py b/src/tracer/scanner.py index 160c1fd..740d798 100644 --- a/src/tracer/scanner.py +++ b/src/tracer/scanner.py @@ -1,4 +1,4 @@ -"""tracer scan: the day-one verdict on a traces file. +"""tracer.scan(): the day-one verdict on a traces file. Answers three questions in one pass: 1. How much of this traffic is predictable enough for a near-free model? @@ -14,9 +14,11 @@ does that, learning a router with accept gates on the same traffic. Usage: - tracer scan traces.jsonl - tracer scan traces.jsonl --target 0.95 --teacher-price-per-1k 5.0 \ - --monthly-calls 3000000 --html scan_report.html + import tracer + from tracer.scanner import scan_html + result = tracer.scan("traces.jsonl", embeddings=X, target=0.95) + report = scan_html(result) + """ from __future__ import annotations @@ -36,7 +38,7 @@ _LABEL_KEYS = ("teacher", "teacher_output", "label", "intent", "output", "answer") # Data-volume policy. Below MIN the held-out evidence is too thin for the -# bounds to mean anything, so a plain scan refuses and points at --force. +# bounds to mean anything, so a plain scan refuses and points at force=True. # SUGGESTED is the volume where cells reliably carry enough held-out members # to certify at 0.90 without coarsening. MIN_SCAN_TRACES = 1_000 @@ -177,7 +179,7 @@ def scan( raise ThinDataError( f"scan needs at least {MIN_SCAN_TRACES:,} traces for a stable read " f"(we suggest ~{SUGGESTED_SCAN_TRACES:,}); this file has {n:,}. " - f"Collect more, or pass --force to scan anyway on thin data " + f"Collect more, or pass force=True to scan anyway on thin data " f"(results will be a best-effort floor, not a guarantee)." ) @@ -351,76 +353,62 @@ def scan( def format_scan(r: ScanResult) -> str: - """Coloured terminal rendering, matching the rest of the tracer CLI. - - Colours degrade to plain text automatically off a TTY (see cli._ui._C). - """ - try: - from tracer.cli._ui import _C, hr - c = _C() - rule = hr - except Exception: # pragma: no cover - cli always present - class _Plain: - def __getattr__(self, _): return "" - c = _Plain() - def rule(char="-", width=56): return char * width + """Format a scan as plain text for logs and notebooks.""" + def rule(char="-", width=56): + return char * width L: list[str] = [] pct = r.certifiable_share * 100 - head_c = c.GREEN if r.certifiable_share > 0 else c.DIM L.append("") - L.append(f" {c.BOLD}{head_c}{pct:.1f}%{c.RESET}{c.BOLD} of traffic certifiable for a near-free model{c.RESET}") - L.append(f" {c.DIM}{r.n_traces:,} traces · {r.n_classes} labels · {r.n_clusters} cells · target {r.target:.0%} agreement{c.RESET}") + L.append(f" {pct:.1f}% of traffic certifiable for a near-free model") + L.append(f" {r.n_traces:,} traces · {r.n_classes} labels · {r.n_clusters} cells · target {r.target:.0%} agreement") L.append(f" {rule()}") if r.forced: - L.append(f" {c.YELLOW}{c.BOLD}⚠ forced scan{c.RESET} {c.YELLOW}thin data: clustering coarsened to concentrate held-out evidence.{c.RESET}") - L.append(f" {c.DIM}Bounds are a best-effort floor, not a guarantee. Collect more traces or run `tracer fit` for the real number.{c.RESET}") + L.append(f" ⚠ forced scan thin data: clustering coarsened to concentrate held-out evidence.") + L.append(f" Bounds are a best-effort floor, not a guarantee. Collect more traces or run `tracer.fit()` for the real number.") if r.certifiable_share > 0: - L.append(f" {c.DIM}{'worst certified bound':<22}{c.RESET}{c.BOLD}{c.GREEN}{r.certified_floor:.3f}{c.RESET} {c.DIM}exact, held-out{c.RESET}") + L.append(f" {'worst certified bound':<22}{r.certified_floor:.3f} exact, held-out") if r.savings_per_1k_calls is not None: - L.append(f" {c.DIM}{'savings / 1k calls':<22}{c.RESET}{c.BOLD}${r.savings_per_1k_calls:.2f}{c.RESET} {c.DIM}at ${r.teacher_price_per_1k}/1k teacher{c.RESET}") + L.append(f" {'savings / 1k calls':<22}${r.savings_per_1k_calls:.2f} at ${r.teacher_price_per_1k}/1k teacher") if r.monthly_savings is not None: - L.append(f" {c.DIM}{'monthly savings':<22}{c.RESET}{c.BOLD}{c.GREEN}${r.monthly_savings:,.0f}{c.RESET} {c.DIM}at {r.monthly_calls:,} calls/mo{c.RESET}") + L.append(f" {'monthly savings':<22}${r.monthly_savings:,.0f} at {r.monthly_calls:,} calls/mo") if r.traces_needed_estimate: L.append("") - L.append(f" {c.YELLOW}⚠{c.RESET} Not enough held-out evidence yet. Collect roughly " - f"{c.BOLD}{r.traces_needed_estimate:,} more traces{c.RESET} and rescan,") - L.append(f" {c.DIM}or pass --force to certify on what you have. Exact bounds need ~22 held-out examples per cell.{c.RESET}") + L.append(f" ⚠ Not enough held-out evidence yet. Collect roughly " + f"{r.traces_needed_estimate:,} more traces and rescan,") + L.append(f" or pass force=True to certify on what you have. Exact bounds need ~22 held-out examples per cell.") if r.frontier: L.append("") - L.append(f" {c.BOLD}{c.YELLOW}Certifiable share by target{c.RESET} {c.DIM}(lightweight scan estimate){c.RESET}") + L.append(f" Certifiable share by target (lightweight scan estimate)") L.append(f" {rule('·')}") for tgt in sorted(r.frontier): share = r.frontier[tgt] - sc = c.GREEN if share >= 0.30 else (c.YELLOW if share > 0 else c.DIM) - L.append(f" {c.DIM}target {tgt:.0%}{c.RESET} {c.BOLD}{sc}{share*100:5.1f}%{c.RESET}") + L.append(f" target {tgt:.0%} {share*100:5.1f}%") L.append("") - L.append(f" {c.BOLD}{c.YELLOW}Cells{c.RESET} {c.DIM}(top {min(20, len(r.clusters))} by traffic share){c.RESET}") + L.append(f" Cells (top {min(20, len(r.clusters))} by traffic share)") L.append(f" {rule('·')}") - L.append(f" {c.DIM}{'share':>6} {'held':>5} {'bound':>6} verdict dominant label{c.RESET}") + L.append(f" {'share':>6} {'held':>5} {'bound':>6} verdict dominant label") for cl in r.clusters[:20]: if cl.certifiable: - tag = f"{c.GREEN}✔ free {c.RESET}" - bcol = c.GREEN + tag = f"✔ free " else: - tag = f"{c.RED}→ keep {c.RESET}" - bcol = c.DIM - L.append(f" {cl.share*100:>5.1f}% {cl.n_held:>5} {bcol}{cl.cp_lower:>6.3f}{c.RESET} {tag} {c.BOLD}{cl.dominant_label[:38]}{c.RESET}") + tag = f"→ keep " + L.append(f" {cl.share*100:>5.1f}% {cl.n_held:>5} {cl.cp_lower:>6.3f} {tag} {cl.dominant_label[:38]}") if cl.examples: ex = " ".join(cl.examples[0].split())[:68] - L.append(f" {c.DIM}{'':>19}e.g. “{ex}”{c.RESET}") + L.append(f" {'':>19}e.g. “{ex}”") if len(r.clusters) > 20: - L.append(f" {c.DIM}... {len(r.clusters) - 20} more cells{c.RESET}") + L.append(f" ... {len(r.clusters) - 20} more cells") L.append("") - L.append(f" {c.DIM}Certifiable = an exact binomial lower bound on held-out label agreement clears your target. No in-sample numbers.{c.RESET}") - L.append(f" {c.DIM}Next: {c.RESET}{c.CYAN}tracer fit{c.RESET}{c.DIM} trains a real router with accept gates and certifies more on the same traffic.{c.RESET}") + L.append(f" Certifiable = an exact binomial lower bound on held-out label agreement clears your target. No in-sample numbers.") + L.append(f" Next: tracer.fit() trains a real router with accept gates and certifies more on the same traffic.") L.append("") return "\n".join(L) @@ -693,7 +681,7 @@ def scan_html(r: ScanResult, source_name: str = "traces") -> str: thin = ("
Forced scan on limited data. The grouping was " "coarsened to squeeze the most evidence out of a small sample, so every number " "below is a best-effort floor, not a guarantee. Collect more requests, or run " - "tracer fit, for a number you can quote.
") + "tracer.fit(), for a number you can quote.") has_viz = bool(r.projection and r.projection.get("points")) viz_block = _VIZ_HTML if has_viz else "" @@ -707,11 +695,11 @@ def scan_html(r: ScanResult, source_name: str = "traces") -> str: - +

{pct:.0f}% of your traffic can be answered for free

A near-free model already matches your current model on {pct:.0f}% of requests, proven on held-out examples, not guessed.

{r.n_traces:,} requests · {r.n_clusters} groups of similar questions · target {r.target:.0%} agreement

- Train the real router with tracer fit → + Train the real router with tracer.fit() → {save_html} {thin}
How to read this. We grouped your requests into clusters of similar questions and laid them out in the space below. For each cluster we checked, on examples it never saw, how often a tiny free model agrees with your model. Green means it agrees at least {r.target:.0%} of the time, so it is safe to auto-answer for free. Red means we could not prove that yet, so those stay on your model.
@@ -722,11 +710,11 @@ def scan_html(r: ScanResult, source_name: str = "traces") -> str:

This is a fast, conservative estimate. Train the real router for more.

-

This 2-minute scan groups your traffic by similarity, a deliberately conservative read, so the real number is usually higher. tracer fit trains the actual router with accept gates and certifies a larger share of the same traffic.

-

pip install tracer-llm  then  tracer fit your_traces.jsonl

+

This 2-minute scan groups your traffic by similarity, a deliberately conservative read, so the real number is usually higher. tracer.fit() trains the actual router with accept gates and certifies a larger share of the same traffic.

+

pip install tracer-llm  then  tracer.fit("your_traces.jsonl", embeddings=X)

Every number here is an exact lower bound measured on held-out data the grouping never saw, no in-sample optimism. - tracerml.ai
+ TRACER on GitHub
{script} {_TEXTTIP_SCRIPT} diff --git a/src/tracer/watch.py b/src/tracer/watch.py index 6457726..18c81d4 100644 --- a/src/tracer/watch.py +++ b/src/tracer/watch.py @@ -11,16 +11,13 @@ proprietary schema. * The same span maps 1:1 to :class:`tracer.types.TraceRecord`, so watched traffic feeds ``tracer.fit()`` / ``tracer.scan()`` directly -- one object - end to end (the local recorder and the cloud optimizer consume the same - thing). + end to end. * Exporters are pluggable. ``LocalFileSink`` (default, no key) writes JSONL. - ``TracerCloudSink`` streams the same spans to Tracer Cloud's FREE - observability with one key (``cloud_key=...`` or ``TRACER_CLOUD_KEY``). ``OTLPSink`` fans the SAME spans out over OTLP to anything else that speaks it (any OTLP/HTTP backend), zero code change. -Core has zero dependencies (stdlib only) -- local recording AND Tracer Cloud -streaming both work out of the box with nothing extra to install. +Core has zero dependencies (stdlib only). Local recording works out of the box; +network export requires an explicitly configured endpoint or custom sink. """ from __future__ import annotations @@ -41,7 +38,6 @@ "Sink", "LocalFileSink", "OTLPSink", - "TracerCloudSink", "MultiSink", "Watcher", "watch", @@ -256,7 +252,7 @@ def close(self) -> None: class MultiSink: - """Fan-out to several sinks (e.g. local + cloud + your own OTLP backend).""" + """Fan-out to several sinks (e.g. local + your own OTLP backend).""" def __init__(self, sinks: Sequence[Sink]) -> None: self.sinks = list(sinks) @@ -360,134 +356,8 @@ def extract_response(span: GenAISpan, resp: Any) -> bool: return False -# Default Tracer Cloud endpoint. Override with TRACER_CLOUD_URL. -_DEFAULT_CLOUD_URL = "https://app.tracerml.ai" - - -class TracerCloudSink: - """Stream observed spans to Tracer Cloud (free observability). - - Point it at a Tracer Cloud ingest key and your watched traffic shows up in - the dashboard within seconds. No login, no SDK -- just a key: - - watch = tracer.watch("classifier", cloud_key="trobs_...") - # or, zero code change: export TRACER_CLOUD_KEY=trobs_... - - Routes by key type, matching the two product paths: - * ``trobs_*`` (workspace ingest key) -> ``/v1/observe`` (tenant-wide) - * ``trc_*`` (per-tracer gateway) -> ``/v1/ingest`` (bound to a tracer) - - Prod-safe: sends are batched on a background daemon thread, so a slow or - down endpoint never adds latency to (or crashes) the host function. Drops - silently on overflow / error; set TRACER_WATCH_DEBUG=1 to see why. - """ - - def __init__( - self, - key: str, - base_url: Optional[str] = None, - *, - source: str = "watch", - batch_size: int = 25, - flush_interval: float = 2.0, - timeout: float = 10.0, - max_queue: int = 10000, - ) -> None: - import queue as _queue - - self.key = key - self.base_url = (base_url or os.environ.get("TRACER_CLOUD_URL") or _DEFAULT_CLOUD_URL).rstrip("/") - # trobs_ = workspace ingest key -> /v1/observe; otherwise per-tracer -> /v1/ingest - self._observe = key.startswith("trobs_") - self.path = "/v1/observe" if self._observe else "/v1/ingest" - self.source = source - self.batch_size = batch_size - self.flush_interval = flush_interval - self.timeout = timeout - self._q: "Any" = _queue.Queue(maxsize=max_queue) - self._stop = threading.Event() - self._t = threading.Thread(target=self._run, name="tracer-cloud-sink", daemon=True) - self._t.start() - - def _event(self, span: GenAISpan) -> Dict[str, Any]: - if self._observe: - return { - "ts": span.start_time or None, - "system": span.system, - "model": span.response_model or span.request_model, - "input": span.input_text, - "output": span.output_text, - "prompt_tokens": span.input_tokens, - "completion_tokens": span.output_tokens, - "latency_ms": span.latency_ms, - "cost_usd": span.cost_usd, - "status": span.status, - "trace_id": span.trace_id, - "tags": span.tags, - } - # per-tracer /v1/ingest shape - return { - "input": span.input_text, - "teacher": span.output_text or None, - "output": span.output_text or None, - "model": span.response_model or span.request_model or "observed", - "cost_usd": span.cost_usd, - "ts": span.start_time or None, - } - - def emit(self, span: GenAISpan) -> None: - try: - self._q.put_nowait(self._event(span)) - except Exception: # queue full -> drop rather than block the host - if os.environ.get("TRACER_WATCH_DEBUG"): - print("[tracer.watch] cloud queue full, dropping span") - - def _post(self, events: List[Dict[str, Any]]) -> None: - import urllib.request - - body = {"events": events} if self._observe else {"source": self.source, "events": events} - data = json.dumps(body, default=str).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.key}", - # A real UA: default urllib UA trips Cloudflare bot protection (403/1010). - "User-Agent": "tracer-watch/0.2.0 (+https://tracerml.ai)", - } - req = urllib.request.Request(f"{self.base_url}{self.path}", data=data, headers=headers, method="POST") - try: - urllib.request.urlopen(req, timeout=self.timeout).read() - except Exception as e: # never let telemetry crash or block the host - if os.environ.get("TRACER_WATCH_DEBUG"): - print(f"[tracer.watch] cloud export failed: {e}") - - def _drain(self) -> List[Dict[str, Any]]: - out: List[Dict[str, Any]] = [] - while len(out) < self.batch_size: - try: - out.append(self._q.get_nowait()) - except Exception: - break - return out - - def _run(self) -> None: - while not self._stop.is_set(): - self._stop.wait(self.flush_interval) - batch = self._drain() - while batch: - self._post(batch) - batch = self._drain() - - def close(self) -> None: - # Flush whatever is queued, then stop the worker. - batch = self._drain() - while batch: - self._post(batch) - batch = self._drain() - self._stop.set() - - # --------------------------------------------------------------------------- # -# Watcher: the decorator / context manager. The one object the cloud reuses. +# Watcher: the decorator / context manager. Records calls to local or explicitly configured sinks. # --------------------------------------------------------------------------- # def _new_id(n: int = 16) -> str: return uuid.uuid4().hex[:n] @@ -538,7 +408,6 @@ def __init__( model: Optional[str] = None, operation: str = "chat", sink: Optional[Sink] = None, - cloud_key: Optional[str] = None, extract_input: Callable[[tuple, dict], str] = _default_extract_input, extract_output: Callable[[Any], str] = _default_extract_output, default_tags: Optional[List[str]] = None, @@ -547,25 +416,18 @@ def __init__( self.system = system self.model = model self.operation = operation - # Default: local-only. Free Tracer Cloud streaming turns on with a - # single key -- the cloud_key kwarg or TRACER_CLOUD_KEY env. OTLP fan-out - # to any backend stays available via TRACER_WATCH_OTLP_ENDPOINT [+ _HEADERS]. - self.sink: Sink = sink or self._sink_from_env(name, cloud_key) + # Default: local-only. Generic export is opt-in via an explicit sink + # or TRACER_WATCH_OTLP_ENDPOINT [+ _HEADERS]. + self.sink: Sink = sink or self._sink_from_env(name) self.extract_input = extract_input self.extract_output = extract_output self.default_tags = list(default_tags or []) @staticmethod - def _sink_from_env(name: str, cloud_key: Optional[str] = None) -> Sink: + def _sink_from_env(name: str) -> Sink: local = LocalFileSink(name, dir=os.environ.get("TRACER_WATCH_DIR", ".tracer/watch")) sinks: List[Sink] = [local] - # Free Tracer Cloud observability: one key, no login. Shows up in the - # dashboard within seconds. - key = cloud_key or os.environ.get("TRACER_CLOUD_KEY") - if key: - sinks.append(TracerCloudSink(key, source=name)) - # Generic OTLP fan-out (any OTLP/HTTP backend). endpoint = os.environ.get("TRACER_WATCH_OTLP_ENDPOINT") if endpoint: diff --git a/tests/test_cloud_cli.py b/tests/test_cloud_cli.py deleted file mode 100644 index 9fa9101..0000000 --- a/tests/test_cloud_cli.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Unit tests for tracer.cloud.cli helpers + command guards. No network: the -client is a fake recording double.""" -import json -from argparse import Namespace - -import pytest - -import tracer.cloud.cli as cli -from tracer.cloud.client import CloudError - - -# ----- _parse_teacher -------------------------------------------------------- # -def test_parse_teacher_colon_form(): - # label is included to match the dashboard's ModelMenuEntry shape the create routes expect. - assert cli._parse_teacher("openai:gpt-5-mini") == { - "provider": "openai", - "model_id": "gpt-5-mini", - "label": "gpt-5-mini", - } - - -def test_parse_teacher_json_form(): - out = cli._parse_teacher('{"provider":"anthropic","model_id":"claude"}') - assert out["provider"] == "anthropic" - - -def test_parse_teacher_none(): - assert cli._parse_teacher(None) is None - - -def test_parse_teacher_invalid_raises(): - with pytest.raises(CloudError): - cli._parse_teacher("just-a-model") - - -# ----- ingest auto-routes by key prefix -------------------------------------- # -class _FakeClient: - def __init__(self, tenants=None): - self.keyed_calls = [] - self.rpc_calls = [] - self._tenants = tenants or [ - {"id": "t-1", "name": "Acme", "slug": "acme", "kind": "company"}, - ] - - def keyed(self, method, path, key, json_body=None): - self.keyed_calls.append((method, path, key, json_body)) - return {"ingested": len(json_body.get("events", []))} - - def db(self, view, **kw): - return list(self._tenants) - - def rpc(self, fn, args): - self.rpc_calls.append((fn, args)) - return True - - -def test_ingest_trobs_goes_to_observe(tmp_path): - f = tmp_path / "ev.json" - f.write_text(json.dumps([{"input": "a"}, {"input": "b"}])) - c = _FakeClient() - cli._cmd_ingest(c, Namespace(key="trobs_x", events=str(f), source="cli", json=False)) - method, path, key, body = c.keyed_calls[0] - assert path == "/v1/observe" - assert "source" not in body and body["events"][0]["input"] == "a" - - -def test_ingest_trc_goes_to_ingest(tmp_path): - f = tmp_path / "ev.json" - f.write_text(json.dumps([{"input": "a"}])) - c = _FakeClient() - cli._cmd_ingest(c, Namespace(key="trc_x", events=str(f), source="myapp", json=False)) - method, path, key, body = c.keyed_calls[0] - assert path == "/v1/ingest" - assert body["source"] == "myapp" - - -# ----- tenant-delete guards -------------------------------------------------- # -def test_tenant_delete_refuses_personal(): - c = _FakeClient(tenants=[{"id": "p1", "name": "Me", "slug": "me", "kind": "personal"}]) - with pytest.raises(CloudError) as ei: - cli._cmd_tenant_delete(c, Namespace(tenant="me", confirm="Me", json=False)) - assert "personal" in str(ei.value).lower() - assert c.rpc_calls == [] # never reached the RPC - - -def test_tenant_delete_refuses_wrong_confirm(): - c = _FakeClient(tenants=[{"id": "t1", "name": "Acme", "slug": "acme", "kind": "company"}]) - with pytest.raises(CloudError) as ei: - cli._cmd_tenant_delete(c, Namespace(tenant="acme", confirm="wrong", json=False)) - assert "confirm" in str(ei.value).lower() - assert c.rpc_calls == [] - - -def test_tenant_delete_calls_rpc_when_confirmed(): - c = _FakeClient(tenants=[{"id": "t1", "name": "Acme", "slug": "acme", "kind": "company"}]) - cli._cmd_tenant_delete(c, Namespace(tenant="acme", confirm="Acme", json=False)) - assert c.rpc_calls == [("delete_company_tenant", {"_tenant": "t1"})] - - -# ----- _resolve_tracer picks id vs slug by ref shape ------------------------- # -class _ResolveClient: - def __init__(self): - self.filters = [] - - def db(self, view, **kw): - self.filters.append(kw.get("filters")) - return [{"id": "t-1", "slug": "my-tracer"}] - - -def test_resolve_tracer_uses_slug_for_nonuuid(): - # A slug must NOT be sent as an id filter (the id column is uuid-typed and - # PostgREST 400s on a non-uuid before any fallback can run). - c = _ResolveClient() - cli._resolve_tracer(c, "my-tracer") - assert c.filters[0] == {"slug": "eq.my-tracer"} - - -def test_resolve_tracer_uses_id_for_uuid(): - c = _ResolveClient() - uid = "38b9a2cf-7888-4ebd-a136-18add9c1d56a" - cli._resolve_tracer(c, uid) - assert c.filters[0] == {"id": f"eq.{uid}"} - - -# ----- _table doesn't crash on empty ----------------------------------------- # -def test_table_empty(capsys): - cli._table([], ["a", "b"]) - out = capsys.readouterr().out - assert "none" in out.lower() diff --git a/tests/test_cloud_client.py b/tests/test_cloud_client.py deleted file mode 100644 index cd3833e..0000000 --- a/tests/test_cloud_client.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Unit tests for tracer.cloud.client.CloudClient. The HTTP layer (`_http`) is -monkeypatched, so these are hermetic, no network, no real Tracer Cloud.""" -import json -import time - -import pytest - -import tracer.cloud.client as cc -from tracer.cloud.client import CloudClient, CloudError - - -def _far_future(): - return int(time.time()) + 9999 - - -def _base_cfg(**over): - cfg = { - "base_url": "http://app.test", - "auth_url": "http://sb.test", - "auth_key": "anon", - "access_token": "aaa.bbb.ccc", - "refresh_token": "refresh-1", - "expires_at": _far_future(), - "email": "u@test", - } - cfg.update(over) - return cfg - - -def _install_fake_http(monkeypatch, handler): - """handler(method, url, headers, data) -> (status, body_bytes).""" - calls = [] - - def fake_http(method, url, *, headers=None, data=None, timeout=60.0): - calls.append({"method": method, "url": url, "headers": headers or {}, "data": data}) - status, body = handler(method, url, headers or {}, data) - return status, body, {} - - monkeypatch.setattr(cc, "_http", fake_http) - return calls - - -# ----- config + resolution --------------------------------------------------- # -def test_defaults_when_cfg_empty(): - c = CloudClient({}) - assert c.base_url == cc.DEFAULT_BASE_URL - assert c.auth_url == cc.DEFAULT_AUTH_URL - assert c.auth_key == cc.DEFAULT_AUTH_KEY - assert c.logged_in is False - - -def test_save_load_roundtrip(tmp_path, monkeypatch): - p = tmp_path / "cloud.json" - monkeypatch.setattr(cc, "CONFIG_PATH", p) - c = CloudClient(_base_cfg()) - c.save() - loaded = CloudClient.load() - assert loaded.cfg["access_token"] == "aaa.bbb.ccc" - assert loaded.base_url == "http://app.test" - - -def test_logout_clears_tokens(tmp_path, monkeypatch): - monkeypatch.setattr(cc, "CONFIG_PATH", tmp_path / "c.json") - c = CloudClient(_base_cfg()) - assert c.logged_in - c.logout() - assert c.logged_in is False - assert "access_token" not in c.cfg - - -# ----- web API requests ------------------------------------------------------ # -def test_api_get_builds_bearer_request(monkeypatch): - calls = _install_fake_http(monkeypatch, lambda *a: (200, b'{"ok":true}')) - c = CloudClient(_base_cfg()) - out = c.api("GET", "/api/tracers/x") - assert out == {"ok": True} - assert calls[0]["url"] == "http://app.test/api/tracers/x" - assert calls[0]["headers"]["Authorization"] == "Bearer aaa.bbb.ccc" - - -def test_api_params_and_json_body(monkeypatch): - calls = _install_fake_http(monkeypatch, lambda *a: (200, b'{}')) - c = CloudClient(_base_cfg()) - c.api("POST", "/api/x", json_body={"a": 1}, params={"range": "7d", "skip": None}) - assert "range=7d" in calls[0]["url"] - assert "skip=" not in calls[0]["url"] # None params dropped - assert json.loads(calls[0]["data"]) == {"a": 1} - - -def test_api_raises_on_error_status(monkeypatch): - _install_fake_http(monkeypatch, lambda *a: (404, b'{"error":"nope"}')) - c = CloudClient(_base_cfg()) - with pytest.raises(CloudError) as ei: - c.api("GET", "/api/x") - assert "nope" in str(ei.value) - - -def test_db_builds_rest_query(monkeypatch): - calls = _install_fake_http(monkeypatch, lambda *a: (200, b"[]")) - c = CloudClient(_base_cfg()) - c.db("my_tenants", select="id,name", filters={"id": "eq.1"}, order="created_at.asc", limit=5) - url = calls[0]["url"] - assert url.startswith("http://sb.test/rest/v1/my_tenants?") - assert "select=id%2Cname" in url - assert "id=eq.1" in url and "order=created_at.asc" in url and "limit=5" in url - assert calls[0]["headers"]["apikey"] == "anon" - assert calls[0]["headers"]["Authorization"] == "Bearer aaa.bbb.ccc" - - -def test_rpc_posts_args(monkeypatch): - calls = _install_fake_http(monkeypatch, lambda *a: (200, b"true")) - c = CloudClient(_base_cfg()) - out = c.rpc("delete_company_tenant", {"_tenant": "t1"}) - assert out is True - assert calls[0]["url"] == "http://sb.test/rest/v1/rpc/delete_company_tenant" - assert json.loads(calls[0]["data"]) == {"_tenant": "t1"} - - -def test_keyed_uses_provided_key(monkeypatch): - calls = _install_fake_http(monkeypatch, lambda *a: (200, b'{"ingested":1}')) - c = CloudClient(_base_cfg()) - out = c.keyed("POST", "/v1/observe", "trobs_K", json_body={"events": []}) - assert out == {"ingested": 1} - assert calls[0]["headers"]["Authorization"] == "Bearer trobs_K" - - -def test_multipart_frames_file_and_fields(monkeypatch): - calls = _install_fake_http(monkeypatch, lambda *a: (200, b'{"ok":true}')) - c = CloudClient(_base_cfg()) - import tempfile, os - fd, path = tempfile.mkstemp(suffix=".jsonl") - os.write(fd, b'{"input":"x"}\n'); os.close(fd) - c.api_multipart("/api/scan", {"tenant_id": "t1"}, path) - body = calls[0]["data"] - assert b'name="tenant_id"' in body - assert b'name="file"; filename=' in body - assert b'{"input":"x"}' in body - ctype = calls[0]["headers"]["Content-Type"] - assert ctype.startswith("multipart/form-data; boundary=") - - -# ----- auth: login + refresh ------------------------------------------------- # -def test_login_password_stores_session(tmp_path, monkeypatch): - monkeypatch.setattr(cc, "CONFIG_PATH", tmp_path / "c.json") - - def handler(method, url, headers, data): - assert "grant_type=password" in url - return 200, json.dumps({ - "access_token": "tok-new", "refresh_token": "ref-new", "expires_in": 3600, - "user": {"id": "u1", "email": "u@test"}, - }).encode() - - _install_fake_http(monkeypatch, handler) - c = CloudClient({"auth_url": "http://sb.test", "auth_key": "anon"}) - c.login_password("u@test", "pw") - assert c.cfg["access_token"] == "tok-new" - assert c.cfg["user_id"] == "u1" - assert c.cfg["expires_at"] > time.time() - - -def test_access_token_refreshes_when_expired(monkeypatch): - state = {"refreshed": False} - - def handler(method, url, headers, data): - if "grant_type=refresh_token" in url: - state["refreshed"] = True - return 200, json.dumps({"access_token": "tok-2", "refresh_token": "ref-2", - "expires_in": 3600}).encode() - return 200, b"{}" - - monkeypatch.setattr(cc, "CONFIG_PATH", __import__("pathlib").Path("/tmp/_unused_cloud.json")) - _install_fake_http(monkeypatch, handler) - c = CloudClient(_base_cfg(expires_at=int(time.time()) - 10)) # already expired - tok = c._access_token() - assert state["refreshed"] is True - assert tok == "tok-2" - - -def test_access_token_no_refresh_when_fresh(monkeypatch): - state = {"refreshed": False} - - def handler(method, url, headers, data): - if "grant_type=refresh_token" in url: - state["refreshed"] = True - return 200, b"{}" - - _install_fake_http(monkeypatch, handler) - c = CloudClient(_base_cfg(expires_at=_far_future())) - assert c._access_token() == "aaa.bbb.ccc" - assert state["refreshed"] is False - - -def test_ensure_auth_config_fetches_from_bootstrap(monkeypatch): - # No auth_url/auth_key in cfg and no env override -> fetch /api/cli/config. - monkeypatch.setattr(cc, "DEFAULT_AUTH_URL", "") - monkeypatch.setattr(cc, "DEFAULT_AUTH_KEY", "") - - def handler(method, url, headers, data): - if url.endswith("/api/cli/config"): - return 200, json.dumps({"auth_url": "http://fetched", "auth_key": "fetched-key"}).encode() - return 200, b"[]" - - calls = _install_fake_http(monkeypatch, handler) - c = CloudClient({"base_url": "http://app.test", "access_token": "a.b.c", - "expires_at": _far_future()}) - c.db("my_tenants") - assert c.cfg["auth_url"] == "http://fetched" - assert c.cfg["auth_key"] == "fetched-key" - # the bootstrap endpoint was hit before the rest query - assert any(call["url"] == "http://app.test/api/cli/config" for call in calls) - - -def test_api_without_login_raises(monkeypatch): - _install_fake_http(monkeypatch, lambda *a: (200, b"{}")) - c = CloudClient({}) # no token - with pytest.raises(CloudError): - c.api("GET", "/api/x") diff --git a/tests/test_fit.py b/tests/test_fit.py index c3f4a38..c469a09 100644 --- a/tests/test_fit.py +++ b/tests/test_fit.py @@ -274,15 +274,6 @@ def test_report(): assert manifest.version == "0.1.0" -# ── CLI demo ────────────────────────────────────────────────────────────────── - -def test_demo_runs(): - """The demo CLI command should run without errors.""" - import argparse - from tracer.cli.main import _cmd_demo - _cmd_demo(argparse.Namespace()) - - # ── Progress logging & candidate skipping ──────────────────────────────────── def test_search_surrogate_invokes_on_candidate_callback(): diff --git a/tests/test_scan.py b/tests/test_scan.py index 011f9d2..85181fc 100644 --- a/tests/test_scan.py +++ b/tests/test_scan.py @@ -1,8 +1,8 @@ -"""Tests for `tracer scan` (scanner.py). +"""Tests for `tracer.scan()` (scanner.py). Covers the behaviours of the day-one scan: the data-volume gate (min 1,000 -traces unless forced, ~5,000 suggested), --force coarsening, the stable -label-colour map, and the terminal + HTML rendering. Everything here uses only +traces unless forced, ~5,000 suggested), force=True coarsening, the stable +label-colour map, and the plain-text + HTML rendering. Everything here uses only the base deps (numpy, scikit-learn), so it runs in the core CI matrix without the embeddings/viz extras and without any network access. """ @@ -54,7 +54,7 @@ def test_thin_data_gate_message_points_at_force_and_suggestion(tmp_path): msg = str(exc.value) assert f"{MIN_SCAN_TRACES:,}" in msg assert f"{SUGGESTED_SCAN_TRACES:,}" in msg - assert "--force" in msg + assert "force=True" in msg def test_scan_runs_above_minimum_without_force(tmp_path): @@ -66,7 +66,7 @@ def test_scan_runs_above_minimum_without_force(tmp_path): assert r.certifiable_share > 0.5 -# ── --force ─────────────────────────────────────────────────────────────────── +# ── force=True ─────────────────────────────────────────────────────────────────── def test_force_runs_below_minimum_and_marks_forced(tmp_path): p, X = _write_traces(tmp_path, 300) @@ -100,13 +100,13 @@ def test_format_scan_renders(tmp_path): p, X = _write_traces(tmp_path, 1200) out = format_scan(scan(p, embeddings=X)) assert "certifiable" in out.lower() - assert "tracer fit" in out + assert "tracer.fit()" in out def test_scan_html_has_branding_and_label_toggle(tmp_path): p, X = _write_traces(tmp_path, 1200) html = scan_html(scan(p, embeddings=X), "synthetic") - for needle in ("tracerml.ai", "vt-label", "vt-verdict", + for needle in ("github.com/adrida/tracer", "vt-label", "vt-verdict", "function setMode", 'id="run-fit"', 'id="texttip"'): assert needle in html, f"missing {needle!r} in scan_html output" diff --git a/tests/test_watch.py b/tests/test_watch.py index 88b9092..498b4cf 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -1,17 +1,14 @@ """Unit tests for tracer.watch: spans, sinks, the decorator/context manager, -env-driven sink composition, and the Tracer Cloud sink (mapping + prod-safety). +and opt-in generic export with local-only defaults. All hermetic: no network, temp dirs only.""" import json -import os import pytest from tracer.watch import ( GenAISpan, LocalFileSink, - MultiSink, OTLPSink, - TracerCloudSink, Watcher, watch, ) @@ -104,29 +101,6 @@ def test_sink_from_env_local_only(tmp_path, monkeypatch): assert isinstance(sink, LocalFileSink) -def test_sink_from_env_adds_cloud_when_key(tmp_path, monkeypatch): - monkeypatch.setenv("TRACER_WATCH_DIR", str(tmp_path)) - monkeypatch.setenv("TRACER_CLOUD_KEY", "trobs_abc") - monkeypatch.delenv("TRACER_WATCH_OTLP_ENDPOINT", raising=False) - sink = Watcher._sink_from_env("x") - assert isinstance(sink, MultiSink) - kinds = {type(s).__name__ for s in sink.sinks} - assert "LocalFileSink" in kinds and "TracerCloudSink" in kinds - for s in sink.sinks: - if isinstance(s, TracerCloudSink): - s.close() - - -def test_cloud_key_kwarg_beats_env(tmp_path, monkeypatch): - monkeypatch.setenv("TRACER_WATCH_DIR", str(tmp_path)) - monkeypatch.delenv("TRACER_CLOUD_KEY", raising=False) - sink = Watcher._sink_from_env("x", cloud_key="trc_xyz") - assert isinstance(sink, MultiSink) - cloud = [s for s in sink.sinks if isinstance(s, TracerCloudSink)] - assert cloud and cloud[0]._observe is False # trc_ -> per-tracer /v1/ingest - cloud[0].close() - - def test_sink_from_env_adds_otlp(tmp_path, monkeypatch): monkeypatch.setenv("TRACER_WATCH_DIR", str(tmp_path)) monkeypatch.delenv("TRACER_CLOUD_KEY", raising=False) @@ -135,84 +109,6 @@ def test_sink_from_env_adds_otlp(tmp_path, monkeypatch): assert any(isinstance(s, OTLPSink) for s in sink.sinks) -# ----- TracerCloudSink: routing + mapping + prod-safety ----------------------- # -def test_cloud_sink_routes_by_key_prefix(): - obs = TracerCloudSink("trobs_k", base_url="http://x"); obs.close() - ing = TracerCloudSink("trc_k", base_url="http://x"); ing.close() - assert obs.path == "/v1/observe" and obs._observe is True - assert ing.path == "/v1/ingest" and ing._observe is False - - -def test_cloud_sink_observe_event_shape(): - s = TracerCloudSink("trobs_k", base_url="http://x"); s.close() - span = GenAISpan(system="openai", request_model="gpt-4o", response_model="gpt-4o", - input_text="in", output_text="out", input_tokens=5, output_tokens=2, - cost_usd=0.001, status="ok", trace_id="t1") - e = s._event(span) - assert e["input"] == "in" and e["output"] == "out" - assert e["model"] == "gpt-4o" - assert e["prompt_tokens"] == 5 and e["completion_tokens"] == 2 - assert e["cost_usd"] == 0.001 - - -def test_cloud_sink_ingest_event_shape(): - s = TracerCloudSink("trc_k", base_url="http://x"); s.close() - span = GenAISpan(request_model="gpt-4o", input_text="in", output_text="out", cost_usd=0.002) - e = s._event(span) - assert e["input"] == "in" - assert e["teacher"] == "out" and e["output"] == "out" - assert e["model"] == "gpt-4o" - - -def test_cloud_sink_post_builds_request(monkeypatch): - import urllib.request - seen = {} - class FakeResp: - def read(self): return b"{}" - def __enter__(self): return self - def __exit__(self, *a): return False - def fake_urlopen(req, timeout=None): - seen["url"] = req.full_url - seen["headers"] = {k.lower(): v for k, v in req.header_items()} - seen["body"] = req.data - return FakeResp() - monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) - s = TracerCloudSink("trobs_k", base_url="http://host", source="app"); s.close() - s._post([{"input": "x"}]) - assert seen["url"] == "http://host/v1/observe" - assert seen["headers"]["authorization"] == "Bearer trobs_k" - assert "tracer-watch" in seen["headers"]["user-agent"].lower() - assert json.loads(seen["body"]) == {"events": [{"input": "x"}]} - - -def test_cloud_sink_post_swallows_errors(monkeypatch): - import urllib.request - def boom(req, timeout=None): - raise OSError("network down") - monkeypatch.setattr(urllib.request, "urlopen", boom) - s = TracerCloudSink("trobs_k", base_url="http://host"); s.close() - # Must NOT raise, telemetry can never crash the host. - s._post([{"input": "x"}]) - - -def test_cloud_sink_emit_flush_via_close(monkeypatch): - import urllib.request - posted = [] - class FakeResp: - def read(self): return b"{}" - def fake_urlopen(req, timeout=None): - posted.append(json.loads(req.data)) - return FakeResp() - monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) - s = TracerCloudSink("trobs_k", base_url="http://host", flush_interval=0.05) - s.emit(GenAISpan(input_text="a", output_text="b")) - s.emit(GenAISpan(input_text="c", output_text="d")) - s.close() # flushes the queue - flat = [ev for batch in posted for ev in batch["events"]] - assert len(flat) == 2 - assert {e["input"] for e in flat} == {"a", "c"} - - def test_watch_factory_returns_watcher(tmp_path, monkeypatch): monkeypatch.setenv("TRACER_WATCH_DIR", str(tmp_path)) monkeypatch.delenv("TRACER_CLOUD_KEY", raising=False) @@ -294,3 +190,47 @@ class _ListSink: def __init__(self, out): self.out = out def emit(self, span): self.out.append(span) def close(self): pass + + +@pytest.mark.parametrize("legacy_key", ["trobs_obsolete", "trc_obsolete"]) +def test_legacy_app_configuration_cannot_export(tmp_path, monkeypatch, legacy_key): + """Upgrades must not keep streaming data because old credentials remain set.""" + import urllib.request + monkeypatch.setenv("TRACER_WATCH_DIR", str(tmp_path)) + monkeypatch.setenv("TRACER_CLOUD_KEY", legacy_key) + monkeypatch.setenv("TRACER_CLOUD_URL", "https://unused.invalid") + monkeypatch.delenv("TRACER_WATCH_OTLP_ENDPOINT", raising=False) + calls = [] + monkeypatch.setattr(urllib.request, "urlopen", lambda *a, **kw: calls.append(a)) + recorder = watch("local") + with recorder.span("ticket") as span: + span.set_output("billing") + recorder.sink.close() + assert calls == [] + saved = json.loads((tmp_path / "local.jsonl").read_text()) + assert saved["input_text"] == "ticket" + assert saved["output_text"] == "billing" + + +def test_explicit_otlp_export_still_works(tmp_path, monkeypatch): + import urllib.request + monkeypatch.setenv("TRACER_WATCH_DIR", str(tmp_path)) + monkeypatch.setenv("TRACER_WATCH_OTLP_ENDPOINT", "http://collector/v1/traces") + monkeypatch.setenv("TRACER_WATCH_OTLP_HEADERS", "Authorization=Bearer test") + monkeypatch.setenv("TRACER_CLOUD_KEY", "trobs_obsolete") + seen = [] + class Response: + def read(self): return b"{}" + def capture(req, **kwargs): + seen.append(req) + return Response() + monkeypatch.setattr(urllib.request, "urlopen", capture) + recorder = watch("export") + with recorder.span("ticket") as span: + span.set_output("billing") + recorder.sink.close() + assert len(seen) == 1 + assert seen[0].full_url == "http://collector/v1/traces" + assert seen[0].get_header("Authorization") == "Bearer test" + assert json.loads(seen[0].data)["attributes"]["gen_ai.operation.name"] == "chat" + assert (tmp_path / "export.jsonl").is_file()