Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,15 @@ notebooks/data/

# Node (JS package)
node_modules/

# Secrets / credentials (never commit)
.env
.env.*
!.env.example
*.pem
*.key
credentials.json
cloud.json

# Local agent/security tooling
.superstack/
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,9 @@ router = tracer.load_router(".tracer", embedder=my_embedder)

### Pattern 4: HTTP server (zero deps)
```bash
tracer serve .tracer --port 8000
tracer serve .tracer --port 8000 # binds 127.0.0.1 by default (no auth)
# POST /predict {"embedding": [...]} → {"label", "decision", "accept_score"}
# Only use --host 0.0.0.0 behind a reverse proxy with TLS/auth. .tracer artifacts are trusted code (joblib/pickle).
```

---
Expand Down
14 changes: 14 additions & 0 deletions SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,18 @@ own API keys; issues in third-party dependencies, model providers, or your own
deployment configuration are out of scope here — please report those to the
relevant vendor.

## Trust model (operators)

A few boundaries that matter when deploying Tracer:

- **`.tracer` artifacts are trusted code.** `pipeline.joblib` is loaded with
joblib/pickle. Only load artifacts you produced (or otherwise fully trust).
Treat a foreign `.tracer/` directory like an untrusted binary.
- **`tracer serve` has no authentication.** It binds to `127.0.0.1` by default.
If you bind to `0.0.0.0` (or otherwise expose the port), put a reverse proxy
with TLS and auth in front. CORS is off unless you pass `--cors-origin`.
- **`tracer.watch` / `@tracer-llm/watch` may exfiltrate prompts.** When
`TRACER_CLOUD_KEY` (or an OTLP endpoint) is set, full input/output text is
sent to that destination. Keep those env vars under your control.

Thank you for helping keep Tracer and its users safe.
5 changes: 4 additions & 1 deletion docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,10 @@ tracer serve [<artifact-dir>] [--host <addr>] [--port <int>]
| Argument | Default | Description |
|----------|---------|-------------|
| `artifact-dir` | `.tracer` | Artifact directory |
| `--host` | `0.0.0.0` | Bind address |
| `--host` | `127.0.0.1` | Bind address (loopback; use `0.0.0.0` only behind a reverse proxy with auth) |
| `--cors-origin` | off | If set, emit `Access-Control-Allow-Origin` (e.g. `https://app.example.com` or `*`) |
| `--max-body-bytes` | `16777216` | Reject larger request bodies |
| `--max-batch` | `10000` | Reject `/predict_batch` with more rows |
| `--port` | `8000` | Listen port |

**Endpoints:**
Expand Down
2 changes: 1 addition & 1 deletion docs/javascript.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ TRACER reads `traces.jsonl` and auto-discovers `traces.npy` (same stem, `.npy` e
tracer serve .tracer --port 8000
```

Run this as a sidecar next to your Node app, same server, same docker-compose, same machine. It binds to `0.0.0.0:8000` by default.
Run this as a sidecar next to your Node app, same server, same docker-compose, same machine. It binds to `127.0.0.1:8000` by default (pass `--host 0.0.0.0` only behind a reverse proxy with auth).

---

Expand Down
25 changes: 22 additions & 3 deletions js/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -372,16 +372,35 @@ function getEnv(name: string): string | undefined {
return g.process?.env?.[name];
}

const WATCH_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;

/** Allowlist watcher names used as filenames (no path separators / traversal). */
export function safeWatchName(name: string): string {
if (typeof name !== "string" || !WATCH_NAME_RE.test(name)) {
throw new Error(
`invalid watch name ${JSON.stringify(name)}: use 1-128 chars of [A-Za-z0-9._-], starting with alphanumeric`,
);
}
return name;
}

/** Append spans as JSONL to `<dir>/<name>.jsonl`. No network, no key. */
export class LocalFileSink implements Sink {
readonly path: string;
constructor(name: string, dir: string = ".tracer/watch") {
this.path = path.join(dir, `${name}.jsonl`);
const safe = safeWatchName(name);
const root = path.resolve(dir);
try {
fs.mkdirSync(dir, { recursive: true });
fs.mkdirSync(root, { recursive: true });
} catch (e) {
debugLog(`could not create dir ${dir}: ${String(e)}`);
debugLog(`could not create dir ${root}: ${String(e)}`);
}
const full = path.resolve(root, `${safe}.jsonl`);
const rel = path.relative(root, full);
if (rel.startsWith("..") || path.isAbsolute(rel)) {
throw new Error(`watch path escapes directory: ${JSON.stringify(name)}`);
}
this.path = full;
}

emit(span: GenAISpan): void {
Expand Down
7 changes: 7 additions & 0 deletions js/test/watch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@ describe("GenAISpan", () => {

// ----- LocalFileSink --------------------------------------------------------- //
describe("LocalFileSink", () => {
it("rejects unsafe watch names (allowlist)", () => {
const dir = tmpDir();
for (const bad of ["../evil", "..\\evil", "a/b", "..", "", "has space", "-leading"]) {
expect(() => new LocalFileSink(bad, dir)).toThrow(/invalid watch name/);
}
});

it("writes JSONL", () => {
const dir = tmpDir();
const sink = new LocalFileSink("unit", dir);
Expand Down
68 changes: 45 additions & 23 deletions src/tracer/analysis/html_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,18 @@
from __future__ import annotations

import json
from html import escape as _html_escape
from pathlib import Path
from typing import Union

# Pin Plotly (no "latest"); only injected when a sankey div is rendered.
_PLOTLY_CDN = "https://cdn.plot.ly/plotly-2.35.2.min.js"


def _esc(value) -> str:
"""HTML-escape dynamic report fields (XSS-safe)."""
return _html_escape("" if value is None else str(value), quote=True)


_CSS = """
* { box-sizing: border-box; margin: 0; padding: 0; }
Expand Down Expand Up @@ -222,11 +231,26 @@ def generate_html_report(
method = manifest_raw.get("selected_method") or "none"
coverage = manifest_raw.get("coverage_cal")
ta = manifest_raw.get("teacher_agreement_cal")
n_traces = manifest_raw.get("n_traces", 0)
n_labels = len(manifest_raw.get("label_space", []))
emb_dim = manifest_raw.get("embedding_dim")
target_ta = manifest_raw.get("target_teacher_agreement", 0.90)
method_cls = _METHOD_BADGE.get(method, "b-gray")
# Coerce numerics — manifest fields are untrusted if the artifact is.
try:
n_traces = int(manifest_raw.get("n_traces", 0) or 0)
except (TypeError, ValueError):
n_traces = 0
label_space = manifest_raw.get("label_space") or []
n_labels = len(label_space) if isinstance(label_space, list) else 0
try:
emb_dim = int(manifest_raw["embedding_dim"]) if manifest_raw.get("embedding_dim") is not None else None
except (TypeError, ValueError):
emb_dim = None
try:
target_ta = float(manifest_raw.get("target_teacher_agreement", 0.90) or 0.90)
except (TypeError, ValueError):
target_ta = 0.90
# Only allow known badge classes (manifest method is untrusted input).
method_cls = _METHOD_BADGE.get(method) or "b-gray"
if method_cls not in ("b-green", "b-blue", "b-purple", "b-gray"):
method_cls = "b-gray"
emb_dim_disp = str(emb_dim) if emb_dim is not None else "?"

cov_exact = (coverage or 0) * 100
defer_exact = 100 - cov_exact
Expand Down Expand Up @@ -261,10 +285,7 @@ def generate_html_report(
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Tracer routing report</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;700&family=Manrope:wght@500;700;800&display=swap" rel="stylesheet">
<style>{_CSS}</style>
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
</head>
<body>
<div class="page">
Expand All @@ -278,7 +299,7 @@ def generate_html_report(
<h1>Routing Policy</h1>
<div class="subtitle">
{n_traces:,} traces &nbsp;·&nbsp; {n_labels} labels
&nbsp;·&nbsp; {emb_dim}-dim embeddings
&nbsp;·&nbsp; {_esc(emb_dim_disp)}-dim embeddings
&nbsp;·&nbsp; target TA = {target_ta:.0%}
</div>

Expand All @@ -296,7 +317,7 @@ def generate_html_report(
<div class="card purple">
<div class="card-label">Method</div>
<div class="card-value" style="font-size:1.2rem;margin-top:10px">
<span class="badge {method_cls}">{method.upper()}</span>
<span class="badge {method_cls}">{_esc(method.upper())}</span>
</div>
<div class="card-sub">selected pipeline</div>
</div>
Expand Down Expand Up @@ -355,6 +376,7 @@ def generate_html_report(
from tracer.analysis.sankey import generate_sankey_div
sankey_div = generate_sankey_div(artifact_dir)
if sankey_div:
html += f'<script src="{_PLOTLY_CDN}"></script>\n'
html += f"""
<div class="section">
<h2>Routing Flow</h2>
Expand Down Expand Up @@ -389,16 +411,16 @@ def generate_html_report(
<tbody>
"""
for s in sorted(label_slices, key=lambda x: -x["handled_rate"]):
label = s["slice_name"].replace("label:", "")
label = _esc(s["slice_name"].replace("label:", ""))
hr_v = s["handled_rate"]
ta_s = s.get("teacher_agreement_handled")
ta_str = f"{ta_s:.1%}" if ta_s is not None else "-"
html += (
f'<tr><td><code style="color:#8b949e;font-size:.82rem">{label}</code></td>'
f'<td><b style="color:#f0f6fc">{hr_v:.1%}</b></td>'
f'<td>{_bar_html(hr_v)}</td>'
f'<td style="color:#8b949e">{s["count"]}</td>'
f'<td style="color:#8b949e">{ta_str}</td></tr>\n'
f'<td style="color:#8b949e">{int(s["count"])}</td>'
f'<td style="color:#8b949e">{_esc(ta_str)}</td></tr>\n'
Comment on lines +422 to +423
)
html += " </tbody>\n </table>\n</div>\n"

Expand All @@ -408,10 +430,10 @@ def generate_html_report(
html += '<tr><th>Bucket</th><th>Coverage</th><th style="width:130px"></th><th>Count</th></tr>\n'
for s in length_slices:
html += (
f'<tr><td>{s["slice_name"]}</td>'
f'<tr><td>{_esc(s["slice_name"])}</td>'
f'<td><b style="color:#f0f6fc">{s["handled_rate"]:.1%}</b></td>'
f'<td>{_bar_html(s["handled_rate"])}</td>'
f'<td style="color:#8b949e">{s["count"]}</td></tr>\n'
f'<td style="color:#8b949e">{int(s["count"])}</td></tr>\n'
)
html += '</table>\n</div>\n'

Expand All @@ -423,14 +445,14 @@ def generate_html_report(
hs = _score_str(p.get("handled_score"))
ds = _score_str(p.get("deferred_score"))
html += f"""<div class="pair">
<div class="pair-intent">{p["teacher_label"]}</div>
<div class="pair-intent">{_esc(p["teacher_label"])}</div>
<div class="pair-row">
<span class="pair-tag pt-local">SURROGATE</span>
<span class="pair-text">{p["handled_preview"]}</span>{hs}
<span class="pair-text">{_esc(p["handled_preview"])}</span>{hs}
</div>
<div class="pair-row">
<span class="pair-tag pt-deferred">→ LLM</span>
<span class="pair-text">{p["deferred_preview"]}</span>{ds}
<span class="pair-text">{_esc(p["deferred_preview"])}</span>{ds}
</div>
</div>\n"""
html += '</div>\n'
Expand All @@ -443,18 +465,18 @@ def generate_html_report(
for ex in handled_ex[:6]:
score_str = f'<span class="ex-score">score {ex["accept_score"]:.2f}</span>' if ex.get("accept_score") else ""
html += (f'<div class="ex-item handled">'
f'<div class="ex-text">{ex["input_preview"]}</div>'
f'<div class="ex-text">{_esc(ex["input_preview"])}</div>'
f'<div class="ex-meta">'
f'<span class="ex-label">{ex["teacher_label"]}</span>'
f'<span class="ex-label">{_esc(ex["teacher_label"])}</span>'
f'{score_str}</div></div>\n')
html += '</div>\n'

html += '<div class="ex-col">\n<h3>Deferred to teacher</h3>\n'
for ex in deferred_ex[:6]:
html += (f'<div class="ex-item deferred">'
f'<div class="ex-text">{ex["input_preview"]}</div>'
f'<div class="ex-text">{_esc(ex["input_preview"])}</div>'
f'<div class="ex-meta">'
f'<span class="ex-label">{ex["teacher_label"]}</span>'
f'<span class="ex-label">{_esc(ex["teacher_label"])}</span>'
f'</div></div>\n')
html += '</div>\n'

Expand All @@ -470,7 +492,7 @@ def generate_html_report(
prev_bar = int(d["previous_handled_rate"] * 60)
cur_bar = int(d["current_handled_rate"] * 60)
html += (
f'<tr><td><code style="color:#8b949e;font-size:.82rem">{d["label"]}</code></td>'
f'<tr><td><code style="color:#8b949e;font-size:.82rem">{_esc(d["label"])}</code></td>'
f'<td style="color:#8b949e">{d["previous_handled_rate"]:.1%}'
f'<div class="delta-bar-bg"><div class="delta-bar-fill" style="width:{prev_bar}px"></div></div></td>'
f'<td style="color:#f0f6fc">{d["current_handled_rate"]:.1%}'
Expand Down
22 changes: 18 additions & 4 deletions src/tracer/analysis/sankey.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,20 @@
from __future__ import annotations

import json
import re
from pathlib import Path
from typing import Optional, Union

# Labels are embedded into Plotly HTML/JS. Strip markup metacharacters so a
# poisoned teacher label cannot break out of the inline <script> payload.
_UNSAFE_LABEL = re.compile(r"""[<>&"'`\\]""")


def _safe_node_label(text: str, *, max_len: int = 120) -> str:
cleaned = _UNSAFE_LABEL.sub("", str(text))
cleaned = "".join(ch for ch in cleaned if ch.isprintable() or ch in " \t")
return cleaned[:max_len] if cleaned else "(label)"


_PALETTE = [
"#4f7cac", "#7fb069", "#e0a458", "#c97b84", "#8f7aa7",
Expand Down Expand Up @@ -131,15 +142,18 @@ def _build_sankey_figure(manifest: dict, qr: dict, top_k: int, dark: bool):
n_other_labels = len(manifest.get("label_space", [])) - n_top
has_other = (other_handled + other_deferred) > 0.5

node_labels = [s["slice_name"].replace("label:", "").replace("_", " ") for s in top]
node_labels = [
_safe_node_label(s["slice_name"].replace("label:", "").replace("_", " "))
for s in top
]
if has_other:
node_labels.append(f"other ({n_other_labels} labels)")
node_labels.append(_safe_node_label(f"other ({n_other_labels} labels)"))
surrogate_idx = len(node_labels)
teacher_idx = surrogate_idx + 1
pct_h = coverage * 100
pct_d = (1 - coverage) * 100
node_labels.append(f"Surrogate ({pct_h:.0f}%)")
node_labels.append(f"Teacher / LLM ({pct_d:.0f}%)")
node_labels.append(_safe_node_label(f"Surrogate ({pct_h:.0f}%)"))
node_labels.append(_safe_node_label(f"Teacher / LLM ({pct_d:.0f}%)"))

if dark:
_palette_dark = [
Expand Down
32 changes: 28 additions & 4 deletions src/tracer/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,14 @@ def _cmd_sankey(args):

def _cmd_serve(args):
from tracer.runtime.serve import serve
serve(artifact_dir=args.artifact_dir, host=args.host, port=args.port)
serve(
artifact_dir=args.artifact_dir,
host=args.host,
port=args.port,
cors_origin=args.cors_origin,
max_body_bytes=args.max_body_bytes,
max_batch=args.max_batch,
)


def _cmd_update(args):
Expand Down Expand Up @@ -691,12 +698,29 @@ def main():
p_sankey.add_argument("--no-open", action="store_true",
help="Don't open browser automatically (html only)")

p_serve = sub.add_parser("serve",
help="Start a prediction HTTP server")
p_serve = sub.add_parser(
"serve",
help="Start a prediction HTTP server (loopback by default, no auth)",
)
p_serve.add_argument("artifact_dir", nargs="?", default=".tracer",
help="Artifact directory (default: .tracer)")
p_serve.add_argument("--host", default="0.0.0.0", help="Bind address (default: 0.0.0.0)")
p_serve.add_argument(
"--host", default="127.0.0.1",
help="Bind address (default: 127.0.0.1; use 0.0.0.0 only behind a reverse proxy)",
)
p_serve.add_argument("--port", type=int, default=8000, help="Port (default: 8000)")
p_serve.add_argument(
"--cors-origin", default=None,
help="If set, emit Access-Control-Allow-Origin (e.g. https://app.example.com or *). Off by default.",
)
p_serve.add_argument(
"--max-body-bytes", type=int, default=16 * 1024 * 1024,
help="Reject request bodies larger than this (default: 16MiB)",
)
p_serve.add_argument(
"--max-batch", type=int, default=10_000,
help="Reject /predict_batch with more rows than this (default: 10000)",
)

p_update = sub.add_parser("update",
help="Refit with new traces (continual learning)")
Expand Down
8 changes: 8 additions & 0 deletions src/tracer/policy/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ def save_pipeline(artifact_dir: Path, pipeline: dict, label_space: list) -> str:


def load_pipeline(artifact_dir: Path) -> dict:
"""Load ``pipeline.joblib`` via joblib (pickle).

Warning
-------
Joblib uses pickle under the hood. Loading an untrusted ``.tracer``
directory can execute arbitrary code. Only load artifacts produced by
your own ``tracer.fit`` / ``tracer.update`` (or another fully trusted source).
"""
return joblib.load(artifact_dir / "pipeline.joblib")


Expand Down
Loading