build_reference.py

#!/usr/bin/env python3
"""(Re)build the tokenizer reference table used by the single-endpoint tokenizer probe.

This is a DEV-ONLY script. The tool itself has zero runtime dependencies; this
script optionally uses `tiktoken` to compute *real* token counts for OpenAI's
public tokenizers so we never hand-write (i.e. fabricate) reference numbers.

Usage:
    pip install tiktoken
    python scripts/build_reference.py

It writes llm_honesty_probe/data/tokenizers.json. Each family's "deltas" are the
raw token count of each probe string under that tokenizer, which is what the probe
compares its measured deltas against. Regenerate this yourself to trust it — that
is the whole point of an honesty tool.
"""

from __future__ import annotations

import json
import os
import sys

# Keep this list byte-for-byte identical to PROBE_STRINGS in probes/tokenizer.py.
PROBE_STRINGS = [
    ("digits", "1234567890" * 6),
    ("spaces", " " * 40),
    ("tabs", "\t" * 20),
    ("word_rep", "banana " * 20),
    ("cjk", "\u4f60\u597d\u4e16\u754c" * 12),
    ("emoji", "\U0001f600" * 16),
    ("accents", "caf\u00e9 r\u00e9sum\u00e9 na\u00efve Stra\u00dfe \u00der" * 3),
    ("code", "def f(x):\n    return x * x\n" * 4),
    ("url", "https://example.com/a/b?q=1&r=2 " * 4),
    ("mixed", "AbC123_xyz-\u6d4b\u8bd5-\U0001f680-END " * 5),
]

# family -> tiktoken encoding name
TIKTOKEN_FAMILIES = {
    "o200k_base": "o200k_base",
    "cl100k_base": "cl100k_base",
}

OUT = os.path.join(os.path.dirname(__file__), "..", "llm_honesty_probe", "data", "tokenizers.json")


def main() -> int:
    try:
        import tiktoken
    except ImportError:
        sys.stderr.write("tiktoken is not installed. Run: pip install tiktoken\n")
        return 1

    families = {}
    for family, enc_name in TIKTOKEN_FAMILIES.items():
        enc = tiktoken.get_encoding(enc_name)
        deltas = {pid: len(enc.encode(s)) for pid, s in PROBE_STRINGS}
        families[family] = {"deltas": deltas, "source": "tiktoken:%s" % enc_name}

    payload = {
        "_meta": {
            "note": "Regenerated by scripts/build_reference.py. deltas = raw token "
                    "count of each probe string under the named tokenizer.",
            "anchor": ".",
            "probe_battery": [pid for pid, _ in PROBE_STRINGS],
        },
        "families": families,
    }

    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(os.path.abspath(OUT), "w", encoding="utf-8") as fh:
        json.dump(payload, fh, indent=2, ensure_ascii=False)
    print("wrote %s (%d families)" % (os.path.abspath(OUT), len(families)))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())