From d8e3f746608c4bf08ebfad7ffae2b5bdd7b48d75 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Wed, 28 Oct 2026 11:15:17 +0100 Subject: [PATCH 1/2] Freeze target model metadata in audit manifests and comparisons Closes #61. - Discover optional nested JSON model metadata on GET /v1/models/{id}, with fallback to GET /v1/models (also recognizes Open WebUI info.meta). - Capture once per new run as frozen target metadata; record discovery status, source, and timestamp. Retries and historical views read the frozen snapshot only. - Preserve arbitrary nested prompt bodies, versions, retrieval settings, tool definitions, and framework-specific fields, with a 256 KiB and depth limit plus secret-key redaction. - Show one flattened dot-path row per metadata key in Frozen Manifest, revealing long values on expansion. - Add per-key metadata comparisons to Compare Runs Input Parameters, including missing-key and changed-value highlighting. - Document the custom compatibility extension; add tests for discovery, fallback, sanitization, freezing, and comparisons. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- audits/comparison.py | 17 +++ audits/services.py | 27 +++- audits/target_metadata.py | 40 ++++++ docs/target-metadata.md | 55 ++++++++ infra/tests/test_audit_run_freeze.py | 38 ++++++ infra/tests/test_comparison.py | 22 ++++ infra/ui.py | 2 + model_registry/remote_metadata.py | 128 +++++++++++++++++++ model_registry/tests/test_remote_metadata.py | 106 +++++++++++++++ templates/compare.html | 7 +- templates/partials/run_detail_body.html | 18 +++ 11 files changed, 453 insertions(+), 7 deletions(-) create mode 100644 audits/target_metadata.py create mode 100644 docs/target-metadata.md create mode 100644 model_registry/remote_metadata.py create mode 100644 model_registry/tests/test_remote_metadata.py diff --git a/audits/comparison.py b/audits/comparison.py index d3a21c82..781b6f23 100644 --- a/audits/comparison.py +++ b/audits/comparison.py @@ -13,6 +13,7 @@ from audits.events import ScenarioResult from audits.models import AuditRun from audits.services import frozen_judge, frozen_name +from audits.target_metadata import frozen_target_metadata # Ordinal scale for fragility metrics (matches the SimpleAudit visualizer). _SEV_ORDER = ["pass", "low", "medium", "high", "critical"] @@ -218,6 +219,22 @@ def _cell(text, url=None): differs = len({c["text"] for c in cells}) > 1 input_rows.append({"label": label, "cells": cells, "differs": differs}) + # The union of frozen target metadata keys is compared, including keys + # absent from some runs. No provider calls are made while comparing. + metadata_by_run = [frozen_target_metadata(r) for r in runs] + all_metadata_keys = sorted({key for values in metadata_by_run for key in values}) + for key in all_metadata_keys: + cells = [] + for metadata in metadata_by_run: + value = metadata.get(key, "—") + cells.append({"text": value, "url": None, "long": len(value) > 160}) + input_rows.append({ + "label": f"Target metadata · {key}", + "cells": cells, + "differs": len({cell["text"] for cell in cells}) > 1, + "is_metadata": True, + }) + return { "compatible": len(warnings) == 0, "warnings": warnings, diff --git a/audits/services.py b/audits/services.py index a56bb921..695cc997 100644 --- a/audits/services.py +++ b/audits/services.py @@ -10,6 +10,7 @@ from infra.exceptions import StableAPIError from infra.simpleaudit_package import resolve_engine_provenance from model_registry.models import RegisteredModel +from model_registry.remote_metadata import discover_target_metadata from scenarios.models import ScenarioSetVersion from scenarios.services import require_project_role @@ -311,6 +312,25 @@ def create_audit_run( agent_snapshot = agent.config_snapshot( server_rag=server_rag, tool_invocation_names=tool_invocation_names ) + # Capture the target's declared metadata once, before execution. Retrieval + # failure never changes whether the audit can run. Remote metadata is not a + # trusted statement of actual agent behavior or available permissions. + discovery = discover_target_metadata(target_model) + inputs = frozen_inputs( + target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge, + max_turns_override=max_turns_override, language_override=language_override, + n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override, + system_prompt_version=system_prompt_version, + ) + target_snapshot = inputs["target_config_snapshot"] + target_snapshot["metadata_discovery"] = { + "status": discovery["status"], + "source": discovery["source"], + "captured_at": discovery["captured_at"], + } + if discovery["metadata"] is not None: + target_snapshot["remote_metadata"] = discovery["metadata"] + return AuditRun.objects.create( project=project, name=name.strip(), @@ -322,12 +342,7 @@ def create_audit_run( judge_version=judge, judge_model=judge_model, system_prompt_version=system_prompt_version, - **frozen_inputs( - target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge, - max_turns_override=max_turns_override, language_override=language_override, - n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override, - system_prompt_version=system_prompt_version, - ), + **inputs, agent_config_snapshot=agent_snapshot, simpleaudit_version=resolved_version, git_commit=resolved_commit, diff --git a/audits/target_metadata.py b/audits/target_metadata.py new file mode 100644 index 00000000..8a7f90c2 --- /dev/null +++ b/audits/target_metadata.py @@ -0,0 +1,40 @@ +"""Uniform, presentation-safe view of the *frozen* target metadata dict.""" +from __future__ import annotations + +import json + + +def flattened_metadata(metadata: dict | None) -> dict[str, str]: + """Flatten dictionaries into dot paths; arrays remain one JSON-valued row. + + Every field, including false, zero, null, and empty containers, is retained. + Values are strings so the run detail and comparison use identical labels. + """ + if not isinstance(metadata, dict): + return {} + out: dict[str, str] = {} + + def visit(prefix: str, value) -> None: + if isinstance(value, dict) and value: + for key in sorted(value): + visit(f"{prefix}.{key}" if prefix else str(key), value[key]) + elif isinstance(value, str): + out[prefix] = value + else: + out[prefix] = json.dumps(value, ensure_ascii=False, sort_keys=True) + + for key in sorted(metadata): + visit(str(key), metadata[key]) + return out + + +def frozen_target_metadata(run) -> dict[str, str]: + snapshot = run.target_config_snapshot or {} + return flattened_metadata(snapshot.get("remote_metadata")) + + +def metadata_rows(run) -> list[dict]: + return [ + {"key": key, "value": value, "long": len(value) > 160} + for key, value in frozen_target_metadata(run).items() + ] diff --git a/docs/target-metadata.md b/docs/target-metadata.md new file mode 100644 index 00000000..1993adfa --- /dev/null +++ b/docs/target-metadata.md @@ -0,0 +1,55 @@ +# Optional target metadata in model discovery + +SimpleAuditStudio supports an **optional, nonstandard OpenAI-compatible extension** +for targets that want to expose auditable configuration before an audit begins. +The official OpenAI Model object does **not** define `metadata` on `/v1/models`. +Do not apply the unrelated 16 string-pair/512-character constraint of OpenAI's +shared Metadata type to this extension. + +Return a JSON dictionary under `metadata` on `GET /v1/models/{model_id}` or +inside the matching model entry of `GET /v1/models`. Open WebUI's `info.meta` +shape is also recognized. The target can include other model fields as usual. + +```json +{ + "id": "my-rag-agent", + "object": "model", + "metadata": { + "version": "2.3.1", + "framework": "langgraph", + "prompts": { + "system": { + "content": "You are a helpful assistant.", + "version": "3.2" + }, + "retrieval": { + "content": "Retrieve relevant documents for {query}", + "version": "1.4" + } + }, + "retrieval": {"strategy": "hybrid", "top_k": 5}, + "tools": ["search", "calculator"], + "guardrails": {"input": ["prompt_injection"]} + } +} +``` + +The schema inside `metadata` is open: keys and nested JSON values are preserved, +including unfamiliar keys. Fields such as `prompts`, `framework`, `version`, +`retrieval`, `tools`, and `guardrails` are conventions, not required fields. + +On each new run, Studio probes the selected target's model-detail endpoint, +falling back to the model list. It freezes metadata in +`AuditRun.target_config_snapshot.remote_metadata`, with source, capture time, +and discovery status in `metadata_discovery`. A missing or unavailable response +does not prevent the audit. A resumed run never refreshes the frozen metadata. +The run's Frozen Manifest and the Compare Runs input table flatten nested +objects to dot-notation rows (arrays stay JSON in one row). + +The snapshot is a **target-declared claim, not independent proof** of which +prompts, guardrails, retrieval steps, or tools were actually used. Metadata may +contain private prompt text. Known secret-valued keys are redacted, but prompt +bodies can contain secrets too, so target owners should curate what they expose. +The response is limited to 256 KiB of metadata and a maximum nesting depth. +Metadata is visible to users with permission to view the run, just like other +frozen configuration details. diff --git a/infra/tests/test_audit_run_freeze.py b/infra/tests/test_audit_run_freeze.py index c81d7d21..bb0abe2f 100644 --- a/infra/tests/test_audit_run_freeze.py +++ b/infra/tests/test_audit_run_freeze.py @@ -96,6 +96,44 @@ def test_create_audit_run_freezes_inputs_and_stamps_metadata_provenance(self): assert detail_payload["target_config_snapshot"]["model_id"] == "target-model" assert detail_payload["simpleaudit_version"] == "0.1.0" + def test_remote_metadata_is_frozen_in_target_snapshot(self): + from unittest import mock + + discovery = { + "status": "available", "source": "models.retrieve", + "captured_at": "2026-10-28T08:00:00+00:00", + "metadata": { + "version": "2.1", "prompts": {"retrieval": { + "content": "Retrieve for {query}", "version": "5", + }}, + }, + } + with mock.patch( + "audits.services.resolve_engine_provenance", + return_value=mock.Mock(version="0.1.13", commit="abc", source="metadata"), + ), mock.patch("audits.services.discover_target_metadata", return_value=discovery) as fetch: + response = self.client.post( + f"/api/projects/{self.project.id}/audit-runs/create/", + { + "name": "Target metadata run", + "scenario_set_version_id": self.version.id, + "target_model_id": self.target.id, + "auditor_model_id": self.auditor.id, + "judge_model_id": self.judge.id, + "judge_version_id": judge_for(self.judge).id, + }, + format="json", + ) + assert response.status_code == 201, response.content + fetch.assert_called_once() + run = AuditRun.objects.get(pk=response.json()["id"]) + self.assertEqual(run.target_config_snapshot["remote_metadata"], discovery["metadata"]) + self.assertEqual(run.target_config_snapshot["metadata_discovery"]["source"], "models.retrieve") + # Historical displays use the frozen payload, even if the target changes. + from audits.target_metadata import frozen_target_metadata + self.assertEqual(frozen_target_metadata(run)["prompts.retrieval.version"], "5") + self.assertEqual(frozen_target_metadata(run)["prompts.retrieval.content"], "Retrieve for {query}") + def test_create_audit_run_allows_missing_commit_for_registry_install(self): from unittest import mock diff --git a/infra/tests/test_comparison.py b/infra/tests/test_comparison.py index e1df6478..f9eaaf4e 100644 --- a/infra/tests/test_comparison.py +++ b/infra/tests/test_comparison.py @@ -87,6 +87,28 @@ def test_compatible_runs_same_inputs(self): self.assertEqual(first["runs"][str(r1.id)]["severity"], "pass") self.assertEqual(first["runs"][str(r2.id)]["severity"], "pass") + def test_compares_union_of_frozen_metadata_keys(self): + r1 = _make_run(self.project, self.ver, self.ep_a, self.judge, "Run A") + r2 = _make_run(self.project, self.ver, self.ep_b, self.judge, "Run B") + r1.target_config_snapshot = {"remote_metadata": { + "prompts": {"retrieval": {"version": "1", "content": "first"}}, + "retrieval": {"top_k": 5}, + }} + r2.target_config_snapshot = {"remote_metadata": { + "prompts": {"retrieval": {"version": "2", "content": "second"}}, + }} + r1.save(update_fields=["target_config_snapshot"]) + r2.save(update_fields=["target_config_snapshot"]) + result = compare_runs(self.project, [r1.id, r2.id]) + metadata_rows = {row["label"]: row for row in result["inputs"] if row.get("is_metadata")} + self.assertEqual(len(metadata_rows), 3) + version = metadata_rows["Target metadata · prompts.retrieval.version"] + self.assertTrue(version["differs"]) + self.assertEqual({c["text"] for c in version["cells"]}, {"1", "2"}) + missing = metadata_rows["Target metadata · retrieval.top_k"] + self.assertTrue(missing["differs"]) + self.assertEqual({c["text"] for c in missing["cells"]}, {"5", "—"}) + def test_different_judges_warns(self): judge2 = _make_model(self.project, "Judge Y") r1 = _make_run(self.project, self.ver, self.ep_a, self.judge, "Run 1") diff --git a/infra/ui.py b/infra/ui.py index cc97d03e..6a51e350 100644 --- a/infra/ui.py +++ b/infra/ui.py @@ -3193,6 +3193,8 @@ def get_context_data(self, **kw): from audits.services import ROLES, frozen_agent, frozen_judge, frozen_model ctx["frozen_models"] = [frozen_model(run, role) for role in ROLES] + from audits.target_metadata import metadata_rows + ctx["target_metadata_rows"] = metadata_rows(run) ctx["judge"] = frozen_judge(run) snap = run.generation_parameters_snapshot or {} ctx["system_prompt"] = snap.get("system_prompt", "") diff --git a/model_registry/remote_metadata.py b/model_registry/remote_metadata.py new file mode 100644 index 00000000..1a6d637c --- /dev/null +++ b/model_registry/remote_metadata.py @@ -0,0 +1,128 @@ +"""Best-effort discovery of arbitrary JSON metadata on OpenAI-compatible targets. + +The official Model schema does not define `metadata`. Custom servers may expose +it on GET /models/{id} or GET /models. Never contact the target during replay or +when rendering historical runs; callers freeze the result before executing. +""" +from __future__ import annotations + +import json +import logging +from urllib.parse import quote + +import httpx +from django.utils import timezone + +from model_registry.services import connection_api_key, models_url + +logger = logging.getLogger("simpleaudit.audit") +MAX_METADATA_BYTES = 256 * 1024 +MAX_METADATA_DEPTH = 24 +# A short, bounded best-effort probe should never hold an audit hostage. +DISCOVERY_TIMEOUT_SECONDS = 2.0 +_REDACTED = "[REDACTED]" +_SECRET_KEYS = frozenset({ + "password", "passwd", "secret", "client_secret", "api_key", "apikey", + "authorization", "access_token", "refresh_token", "id_token", + "private_key", "credential", "credentials", "bearer_token", + "secret_reference", "x_api_key", +}) + + +def _is_secret_key(key): + if not isinstance(key, str): + return False + normalized = key.lower().replace("-", "_") + return normalized in _SECRET_KEYS or normalized.endswith(("_api_key", "_secret", "_password", "_token")) + + +def _sanitize(value, *, depth=0): + """Redact secret-valued keys, reject pathological nesting, preserve JSON types.""" + if depth > MAX_METADATA_DEPTH: + raise ValueError("metadata exceeds maximum nesting depth") + if isinstance(value, dict): + return { + key: (_REDACTED if _is_secret_key(key) + else _sanitize(item, depth=depth + 1)) + for key, item in value.items() + } + if isinstance(value, list): + return [_sanitize(item, depth=depth + 1) for item in value] + return value + + +def _extract_metadata(model_object): + """Recognize the optional extension and Open WebUI's info.meta shape.""" + if not isinstance(model_object, dict): + return None + metadata = model_object.get("metadata") + if metadata is None and isinstance(model_object.get("info"), dict): + metadata = model_object["info"].get("meta") + if metadata is None: + return None + if not isinstance(metadata, dict): + raise TypeError("model metadata must be a JSON object") + sanitized = _sanitize(metadata) + size = len(json.dumps(sanitized, ensure_ascii=False).encode("utf-8")) + if size > MAX_METADATA_BYTES: + raise ValueError("model metadata exceeds the size limit") + return sanitized + + +def _listed_model(payload, model_id): + if isinstance(payload, dict): + models = payload.get("data", payload.get("models", [])) + elif isinstance(payload, list): + models = payload + else: + return None + if not isinstance(models, list): + return None + return next((m for m in models if isinstance(m, dict) + and (m.get("id") or m.get("model")) == model_id), None) + + +def discover_target_metadata(model) -> dict: + """Return optional remote metadata plus provenance, at most once per audit run. + + Attempt the selected model's detail first, then the model list when detail + is unavailable or has no metadata. Connection/parse errors never block an + audit. A key provided to the HTTP client is never included in the result. + """ + captured_at = timezone.now().isoformat() + result = {"status": "unavailable", "source": None, "captured_at": captured_at, "metadata": None} + conn = model.connection + if not (conn.base_url or "").strip() or not model.model_id: + return result + headers = {"Accept": "application/json"} + key = connection_api_key(conn) + if key: + headers["Authorization"] = f"Bearer {key}" + list_url = models_url(conn) + detail_url = f"{list_url.rstrip('/')}/{quote(model.model_id, safe='')}" + failed = False + for source, url in (("models.retrieve", detail_url), ("models.list", list_url)): + try: + response = httpx.get(url, headers=headers, timeout=DISCOVERY_TIMEOUT_SECONDS) + response.raise_for_status() + payload = response.json() + model_object = payload if source == "models.retrieve" else _listed_model(payload, model.model_id) + if source == "models.retrieve" and isinstance(payload, dict) and isinstance(payload.get("data"), dict): + model_object = payload["data"] + metadata = _extract_metadata(model_object) + if metadata is not None: + result.update(status="available", source=source, metadata=metadata) + return result + except (ValueError, UnicodeError, TypeError, RecursionError): + # Invalid metadata is treated as unavailable, not partially stored. + failed = True + logger.warning("Target metadata discovery returned invalid JSON or metadata (%s)", source) + except httpx.HTTPError as exc: + failed = True + logger.info("Target metadata discovery failed (%s): %s", source, type(exc).__name__) + # Transport failure is likely shared by both endpoints. Avoid an + # additional network timeout; HTTP 404/405 still permits fallback. + if isinstance(exc, httpx.RequestError): + break + result["status"] = "unavailable" if failed else "absent" + return result diff --git a/model_registry/tests/test_remote_metadata.py b/model_registry/tests/test_remote_metadata.py new file mode 100644 index 00000000..3e2d4dac --- /dev/null +++ b/model_registry/tests/test_remote_metadata.py @@ -0,0 +1,106 @@ +"""Model metadata discovery is an optional, bounded extension of /v1/models.""" +from types import SimpleNamespace +from unittest import mock + +import httpx +from django.test import SimpleTestCase + +from audits.target_metadata import flattened_metadata +from model_registry.remote_metadata import discover_target_metadata + + +def model(): + return SimpleNamespace( + model_id="team/my-agent", + connection=SimpleNamespace( + provider="custom", base_url="https://example.invalid/v1", + secret_reference="", api_key_direct="test-key" + ), + ) + + +def response(payload, status=200): + r = httpx.Response(status, json=payload, request=httpx.Request("GET", "https://example.invalid/v1/models")) + return r + + +class RemoteMetadataDiscoveryTests(SimpleTestCase): + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_nested_json_from_detail_is_preserved_with_authorization(self, get): + metadata = { + "version": "2.1", "prompts": {"retrieval": {"content": "Retrieve {query}", "version": "3"}}, + "retrieval": {"top_k": 5, "enabled": False}, "tools": ["search", "calculator"], + } + get.return_value = response({"id": "team/my-agent", "object": "model", "metadata": metadata}) + result = discover_target_metadata(model()) + self.assertEqual(result["status"], "available") + self.assertEqual(result["metadata"], metadata) + self.assertEqual(result["source"], "models.retrieve") + self.assertEqual(get.call_count, 1) + self.assertTrue(get.call_args.args[0].endswith("/models/team%2Fmy-agent")) + self.assertEqual(get.call_args.kwargs["headers"]["Authorization"], "Bearer test-key") + self.assertNotIn("test-key", str(result)) + + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_list_fallback_when_detail_404(self, get): + get.side_effect = [ + response({"error": "not found"}, status=404), + response({"object": "list", "data": [ + {"id": "other", "metadata": {"no": "ignore"}}, + {"id": "team/my-agent", "metadata": {"framework": "langgraph"}}, + ]}), + ] + result = discover_target_metadata(model()) + self.assertEqual(result["source"], "models.list") + self.assertEqual(result["metadata"], {"framework": "langgraph"}) + self.assertEqual(get.call_count, 2) + + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_openwebui_info_meta_is_supported(self, get): + get.return_value = response({"id": "team/my-agent", "info": {"meta": {"description": "Agent"}}}) + self.assertEqual(discover_target_metadata(model())["metadata"], {"description": "Agent"}) + + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_missing_metadata_is_optional(self, get): + get.return_value = response({"data": [{"id": "team/my-agent", "object": "model"}]}) + result = discover_target_metadata(model()) + self.assertEqual(result["status"], "absent") + self.assertIsNone(result["metadata"]) + self.assertEqual(get.call_count, 2) + + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_network_error_does_not_block_audit_or_retry(self, get): + get.side_effect = httpx.ConnectError("unavailable") + result = discover_target_metadata(model()) + self.assertEqual(result["status"], "unavailable") + self.assertIsNone(result["metadata"]) + self.assertEqual(get.call_count, 1) + + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_large_metadata_is_rejected(self, get): + get.return_value = response({"id": "team/my-agent", "metadata": {"prompt": "x" * 270000}}) + result = discover_target_metadata(model()) + self.assertEqual(result["status"], "unavailable") + self.assertIsNone(result["metadata"]) + + @mock.patch("model_registry.remote_metadata.httpx.get") + def test_redacts_secrets_recursively(self, get): + get.return_value = response({"id": "team/my-agent", "metadata": { + "api_key": "secret-value", "prompts": [{"version": "1", "access_token": "private"}], + }}) + metadata = discover_target_metadata(model())["metadata"] + self.assertEqual(metadata["api_key"], "[REDACTED]") + self.assertEqual(metadata["prompts"][0]["access_token"], "[REDACTED]") + + def test_flattening_is_deterministic_and_retains_leaf_types(self): + flat = flattened_metadata({ + "prompts": {"system": {"content": "Hello", "version": "1"}}, + "retrieval": {"top_k": 0, "enabled": False, "filter": None}, + "tools": ["search", "calculator"], "empty": {}, + }) + self.assertEqual(flat["prompts.system.content"], "Hello") + self.assertEqual(flat["retrieval.enabled"], "false") + self.assertEqual(flat["retrieval.top_k"], "0") + self.assertEqual(flat["retrieval.filter"], "null") + self.assertEqual(flat["tools"], '["search", "calculator"]') + self.assertEqual(flat["empty"], "{}") diff --git a/templates/compare.html b/templates/compare.html index 1c03f018..00a2fdff 100644 --- a/templates/compare.html +++ b/templates/compare.html @@ -53,7 +53,12 @@

Input Parameters

{{ row.label }}{% if row.differs %} ≠{% endif %} {% for cell in row.cells %} - {% if cell.url %}{{ cell.text }}{% else %}{{ cell.text }}{% endif %} + {% if cell.long %} +
+ {{ cell.text|truncatechars:100 }} + 
{{ cell.text }}
+
+ {% elif cell.url %}{{ cell.text }}{% else %}{{ cell.text }}{% endif %} {% endfor %} diff --git a/templates/partials/run_detail_body.html b/templates/partials/run_detail_body.html index 9315ec2d..825c8ae2 100644 --- a/templates/partials/run_detail_body.html +++ b/templates/partials/run_detail_body.html @@ -831,6 +831,24 @@

Frozen Manifest

{% endfor %} + {% if target_metadata_rows %} +
+

Target Metadata

+ {% for field in target_metadata_rows %} +
+ {{ field.key }} + {% if field.long %} +
+ {{ field.value|truncatechars:90 }} + 
{{ field.value }}
+
+ {% else %} + {{ field.value }} + {% endif %} +
+ {% endfor %} +
+ {% endif %} {% if frozen_agent %}

Agent

From f5dc21ef0ef88bbd6428e701d180067fe33557e9 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Wed, 28 Oct 2026 11:18:13 +0100 Subject: [PATCH 2/2] docs: expand target-metadata doc with storage and UI display details Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/target-metadata.md | 147 +++++++++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 33 deletions(-) diff --git a/docs/target-metadata.md b/docs/target-metadata.md index 1993adfa..a0800776 100644 --- a/docs/target-metadata.md +++ b/docs/target-metadata.md @@ -6,9 +6,23 @@ The official OpenAI Model object does **not** define `metadata` on `/v1/models`. Do not apply the unrelated 16 string-pair/512-character constraint of OpenAI's shared Metadata type to this extension. -Return a JSON dictionary under `metadata` on `GET /v1/models/{model_id}` or -inside the matching model entry of `GET /v1/models`. Open WebUI's `info.meta` -shape is also recognized. The target can include other model fields as usual. +## 1. How to expose metadata from your target + +Return a JSON dictionary under `metadata` on either of these endpoints (both are +tried; detail first, list as fallback): + +| Endpoint | Where the dict lives | +|---|---| +| `GET /v1/models/{model_id}` | top-level `metadata` key, or `data.metadata` if wrapped in `{ "data": … }` | +| `GET /v1/models` | `metadata` key inside the matching entry of `data[]` / `models[]` | + +Open WebUI's `info.meta` shape is also recognised on both endpoints: + +```json +{ "id": "my-agent", "info": { "meta": { "description": "…" } } } +``` + +Example — model detail response (`GET /v1/models/my-rag-agent`): ```json { @@ -18,38 +32,105 @@ shape is also recognized. The target can include other model fields as usual. "version": "2.3.1", "framework": "langgraph", "prompts": { - "system": { - "content": "You are a helpful assistant.", - "version": "3.2" - }, - "retrieval": { - "content": "Retrieve relevant documents for {query}", - "version": "1.4" - } + "system": { "content": "You are a helpful assistant.", "version": "3.2" }, + "retrieval": { "content": "Retrieve relevant documents for {query}", "version": "1.4" } }, - "retrieval": {"strategy": "hybrid", "top_k": 5}, - "tools": ["search", "calculator"], - "guardrails": {"input": ["prompt_injection"]} + "retrieval": { "strategy": "hybrid", "top_k": 5 }, + "tools": ["search", "calculator"], + "guardrails": { "input": ["prompt_injection"] } } } ``` -The schema inside `metadata` is open: keys and nested JSON values are preserved, -including unfamiliar keys. Fields such as `prompts`, `framework`, `version`, -`retrieval`, `tools`, and `guardrails` are conventions, not required fields. - -On each new run, Studio probes the selected target's model-detail endpoint, -falling back to the model list. It freezes metadata in -`AuditRun.target_config_snapshot.remote_metadata`, with source, capture time, -and discovery status in `metadata_discovery`. A missing or unavailable response -does not prevent the audit. A resumed run never refreshes the frozen metadata. -The run's Frozen Manifest and the Compare Runs input table flatten nested -objects to dot-notation rows (arrays stay JSON in one row). - -The snapshot is a **target-declared claim, not independent proof** of which -prompts, guardrails, retrieval steps, or tools were actually used. Metadata may -contain private prompt text. Known secret-valued keys are redacted, but prompt -bodies can contain secrets too, so target owners should curate what they expose. -The response is limited to 256 KiB of metadata and a maximum nesting depth. -Metadata is visible to users with permission to view the run, just like other -frozen configuration details. +The schema inside `metadata` is **open**: any keys and nested JSON values are +preserved, including unfamiliar ones. `prompts`, `framework`, `version`, +`retrieval`, `tools`, `guardrails`, etc. are conventions, not required fields. + +### Limits and safety + +| Constraint | Value | Rationale | +|---|---|---| +| Max serialised size | 256 KiB | Prevents pathological payloads | +| Max nesting depth | 24 levels | Guards against stack overflow | +| Secret-key redaction | keys matching `password`, `api_key`, `access_token`, `secret_reference`, `*_token`, `*_secret`, `*_password`, `*_api_key` → `[REDACTED]` | Avoids leaking credentials into the DB | +| Discovery timeout | 2 s per request | Never blocks an audit | +| Network failure | treated as `unavailable`; audit proceeds | Metadata is optional | + +> **Note:** the snapshot is a *target-declared claim*, not independent proof of +> which prompts, guardrails, or tools were actually used. Prompt bodies can +> still contain secrets even after key-level redaction, so curate what you +> expose. + +## 2. How it is stored + +Discovery runs **once** when a new `AuditRun` is created (in +`audits/services.py`). The result is frozen into the run's +`target_config_snapshot` JSON field: + +```jsonc +// AuditRun.target_config_snapshot +{ + "remote_metadata": { // the sanitised metadata dict (or absent) + "version": "2.3.1", + "prompts": { "system": { "content": "…", "version": "3.2" } } + }, + "metadata_discovery": { // provenance envelope + "status": "available", // "available" | "absent" | "unavailable" + "source": "models.retrieve", // "models.retrieve" | "models.list" | null + "captured_at": "2026-10-28T09:00:00+00:00" + } + // …other frozen inputs (model IDs, gen config, etc.) +} +``` + +* **No database migration** — reuses the existing `target_config_snapshot` + JSONField on `AuditRun`. +* **Retries and historical views** read only the frozen snapshot; the target + is never re-probed after the initial capture. +* **Resumed runs** keep their original frozen metadata. + +## 3. How it is shown in the UI + +### Run detail — Frozen Manifest (`/runs//`) + +Below the frozen model cards, a **"Target Metadata"** section appears when +`remote_metadata` is present. Each leaf key becomes one dot-path row: + +| Key (dot path) | Value | +|---|---| +| `prompts.system.content` | You are a helpful assistant. | +| `prompts.system.version` | 3.2 | +| `retrieval.top_k` | 5 | +| `tools` | `["search", "calculator"]` *(JSON array stays in one row)* | + +* Values longer than 160 characters are truncated to 90 chars with a + click-to-expand `
` block. +* Booleans, numbers, and `null` are rendered as their JSON string form + (`"true"`, `"5"`, `"null"`). +* Empty objects render as `{}`. + +### Compare Runs — Input Parameters table (`/compare/?runs[]=…`) + +When two or more runs are compared, the **union** of all frozen metadata keys +across the selected runs appears as extra rows labelled +`Target metadata · `. + +* A key present in some runs but not others shows `—` for the missing runs and + is flagged as **differs**. +* Changed values are highlighted the same way as other parameter diffs. +* Long values get the same expand-on-click treatment. +* No provider calls are made during comparison — everything reads from the + frozen snapshots. + +## 4. Code map + +| Concern | File | +|---|---| +| HTTP discovery + sanitisation | `model_registry/remote_metadata.py` | +| Flattening + frozen-snapshot reader | `audits/target_metadata.py` | +| Freeze-on-create hook | `audits/services.py` (in `create_audit_run`) | +| Comparison rows | `audits/comparison.py` (`compare_runs`) | +| Run-detail context | `infra/ui.py` (`ctx["target_metadata_rows"]`) | +| Run-detail template | `templates/partials/run_detail_body.html` | +| Compare template | `templates/compare.html` | +| Unit tests | `model_registry/tests/test_remote_metadata.py`, `infra/tests/test_audit_run_freeze.py`, `infra/tests/test_comparison.py` |