Freeze target metadata in audit manifests and comparisons

audits/comparison.py

@@ -13,6 +13,7 @@
 from audits.events import ScenarioResult
 from audits.models import AuditRun
 from audits.services import frozen_judge, frozen_name
+from audits.target_metadata import frozen_target_metadata
 
 # Ordinal scale for fragility metrics (matches the SimpleAudit visualizer).
 _SEV_ORDER = ["pass", "low", "medium", "high", "critical"]
@@ -218,6 +219,22 @@ def _cell(text, url=None):
         differs = len({c["text"] for c in cells}) > 1
         input_rows.append({"label": label, "cells": cells, "differs": differs})
 
+    # The union of frozen target metadata keys is compared, including keys
+    # absent from some runs. No provider calls are made while comparing.
+    metadata_by_run = [frozen_target_metadata(r) for r in runs]
+    all_metadata_keys = sorted({key for values in metadata_by_run for key in values})
+    for key in all_metadata_keys:
+        cells = []
+        for metadata in metadata_by_run:
+            value = metadata.get(key, "—")
+            cells.append({"text": value, "url": None, "long": len(value) > 160})
+        input_rows.append({
+            "label": f"Target metadata · {key}",
+            "cells": cells,
+            "differs": len({cell["text"] for cell in cells}) > 1,
+            "is_metadata": True,
+        })
+
     return {
         "compatible": len(warnings) == 0,
         "warnings": warnings,

audits/services.py

@@ -10,6 +10,7 @@
 from infra.exceptions import StableAPIError
 from infra.simpleaudit_package import resolve_engine_provenance
 from model_registry.models import RegisteredModel
+from model_registry.remote_metadata import discover_target_metadata
 from scenarios.models import ScenarioSetVersion
 from scenarios.services import require_project_role
 
@@ -311,6 +312,25 @@ def create_audit_run(
         agent_snapshot = agent.config_snapshot(
             server_rag=server_rag, tool_invocation_names=tool_invocation_names
         )
+    # Capture the target's declared metadata once, before execution. Retrieval
+    # failure never changes whether the audit can run. Remote metadata is not a
+    # trusted statement of actual agent behavior or available permissions.
+    discovery = discover_target_metadata(target_model)
+    inputs = frozen_inputs(
+        target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge,
+        max_turns_override=max_turns_override, language_override=language_override,
+        n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override,
+        system_prompt_version=system_prompt_version,
+    )
+    target_snapshot = inputs["target_config_snapshot"]
+    target_snapshot["metadata_discovery"] = {
+        "status": discovery["status"],
+        "source": discovery["source"],
+        "captured_at": discovery["captured_at"],
+    }
+    if discovery["metadata"] is not None:
+        target_snapshot["remote_metadata"] = discovery["metadata"]
+
     return AuditRun.objects.create(
         project=project,
         name=name.strip(),
@@ -322,12 +342,7 @@ def create_audit_run(
         judge_version=judge,
         judge_model=judge_model,
         system_prompt_version=system_prompt_version,
-        **frozen_inputs(
-            target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge,
-            max_turns_override=max_turns_override, language_override=language_override,
-            n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override,
-            system_prompt_version=system_prompt_version,
-        ),
+        **inputs,
         agent_config_snapshot=agent_snapshot,
         simpleaudit_version=resolved_version,
         git_commit=resolved_commit,

audits/target_metadata.py

@@ -0,0 +1,40 @@
+"""Uniform, presentation-safe view of the *frozen* target metadata dict."""
+from __future__ import annotations
+
+import json
+
+
+def flattened_metadata(metadata: dict | None) -> dict[str, str]:
+    """Flatten dictionaries into dot paths; arrays remain one JSON-valued row.
+
+    Every field, including false, zero, null, and empty containers, is retained.
+    Values are strings so the run detail and comparison use identical labels.
+    """
+    if not isinstance(metadata, dict):
+        return {}
+    out: dict[str, str] = {}
+
+    def visit(prefix: str, value) -> None:
+        if isinstance(value, dict) and value:
+            for key in sorted(value):
+                visit(f"{prefix}.{key}" if prefix else str(key), value[key])
+        elif isinstance(value, str):
+            out[prefix] = value
+        else:
+            out[prefix] = json.dumps(value, ensure_ascii=False, sort_keys=True)
+
+    for key in sorted(metadata):
+        visit(str(key), metadata[key])
+    return out
+
+
+def frozen_target_metadata(run) -> dict[str, str]:
+    snapshot = run.target_config_snapshot or {}
+    return flattened_metadata(snapshot.get("remote_metadata"))
+
+
+def metadata_rows(run) -> list[dict]:
+    return [
+        {"key": key, "value": value, "long": len(value) > 160}
+        for key, value in frozen_target_metadata(run).items()
+    ]

docs/target-metadata.md

@@ -0,0 +1,136 @@
+# Optional target metadata in model discovery
+
+SimpleAuditStudio supports an **optional, nonstandard OpenAI-compatible extension**
+for targets that want to expose auditable configuration before an audit begins.
+The official OpenAI Model object does **not** define `metadata` on `/v1/models`.
+Do not apply the unrelated 16 string-pair/512-character constraint of OpenAI's
+shared Metadata type to this extension.
+
+## 1. How to expose metadata from your target
+
+Return a JSON dictionary under `metadata` on either of these endpoints (both are
+tried; detail first, list as fallback):
+
+| Endpoint | Where the dict lives |
+|---|---|
+| `GET /v1/models/{model_id}` | top-level `metadata` key, or `data.metadata` if wrapped in `{ "data": … }` |
+| `GET /v1/models` | `metadata` key inside the matching entry of `data[]` / `models[]` |
+
+Open WebUI's `info.meta` shape is also recognised on both endpoints:
+
+```json
+{ "id": "my-agent", "info": { "meta": { "description": "…" } } }
+```
+
+Example — model detail response (`GET /v1/models/my-rag-agent`):
+
+```json
+{
+  "id": "my-rag-agent",
+  "object": "model",
+  "metadata": {
+    "version": "2.3.1",
+    "framework": "langgraph",
+    "prompts": {
+      "system":    { "content": "You are a helpful assistant.", "version": "3.2" },
+      "retrieval": { "content": "Retrieve relevant documents for {query}", "version": "1.4" }
+    },
+    "retrieval":  { "strategy": "hybrid", "top_k": 5 },
+    "tools":      ["search", "calculator"],
+    "guardrails": { "input": ["prompt_injection"] }
+  }
+}
+```
+
+The schema inside `metadata` is **open**: any keys and nested JSON values are
+preserved, including unfamiliar ones. `prompts`, `framework`, `version`,
+`retrieval`, `tools`, `guardrails`, etc. are conventions, not required fields.
+
+### Limits and safety
+
+| Constraint | Value | Rationale |
+|---|---|---|
+| Max serialised size | 256 KiB | Prevents pathological payloads |
+| Max nesting depth | 24 levels | Guards against stack overflow |
+| Secret-key redaction | keys matching `password`, `api_key`, `access_token`, `secret_reference`, `*_token`, `*_secret`, `*_password`, `*_api_key` → `[REDACTED]` | Avoids leaking credentials into the DB |
+| Discovery timeout | 2 s per request | Never blocks an audit |
+| Network failure | treated as `unavailable`; audit proceeds | Metadata is optional |
+
+> **Note:** the snapshot is a *target-declared claim*, not independent proof of
+> which prompts, guardrails, or tools were actually used. Prompt bodies can
+> still contain secrets even after key-level redaction, so curate what you
+> expose.
+
+## 2. How it is stored
+
+Discovery runs **once** when a new `AuditRun` is created (in
+`audits/services.py`). The result is frozen into the run's
+`target_config_snapshot` JSON field:
+
+```jsonc
+// AuditRun.target_config_snapshot
+{
+  "remote_metadata": {              // the sanitised metadata dict (or absent)
+    "version": "2.3.1",
+    "prompts": { "system": { "content": "…", "version": "3.2" } }
+  },
+  "metadata_discovery": {           // provenance envelope
+    "status": "available",          // "available" | "absent" | "unavailable"
+    "source": "models.retrieve",   // "models.retrieve" | "models.list" | null
+    "captured_at": "2026-10-28T09:00:00+00:00"
+  }
+  // …other frozen inputs (model IDs, gen config, etc.)
+}
+```
+
+* **No database migration** — reuses the existing `target_config_snapshot`
+  JSONField on `AuditRun`.
+* **Retries and historical views** read only the frozen snapshot; the target
+  is never re-probed after the initial capture.
+* **Resumed runs** keep their original frozen metadata.
+
+## 3. How it is shown in the UI
+
+### Run detail — Frozen Manifest (`/runs/<id>/`)
+
+Below the frozen model cards, a **"Target Metadata"** section appears when
+`remote_metadata` is present. Each leaf key becomes one dot-path row:
+
+| Key (dot path) | Value |
+|---|---|
+| `prompts.system.content` | You are a helpful assistant. |
+| `prompts.system.version` | 3.2 |
+| `retrieval.top_k` | 5 |
+| `tools` | `["search", "calculator"]` *(JSON array stays in one row)* |
+
+* Values longer than 160 characters are truncated to 90 chars with a
+  click-to-expand `<details>` block.
+* Booleans, numbers, and `null` are rendered as their JSON string form
+  (`"true"`, `"5"`, `"null"`).
+* Empty objects render as `{}`.
+
+### Compare Runs — Input Parameters table (`/compare/?runs[]=…`)
+
+When two or more runs are compared, the **union** of all frozen metadata keys
+across the selected runs appears as extra rows labelled
+`Target metadata · <dot.path>`.
+
+* A key present in some runs but not others shows `—` for the missing runs and
+  is flagged as **differs**.
+* Changed values are highlighted the same way as other parameter diffs.
+* Long values get the same expand-on-click treatment.
+* No provider calls are made during comparison — everything reads from the
+  frozen snapshots.
+
+## 4. Code map
+
+| Concern | File |
+|---|---|
+| HTTP discovery + sanitisation | `model_registry/remote_metadata.py` |
+| Flattening + frozen-snapshot reader | `audits/target_metadata.py` |
+| Freeze-on-create hook | `audits/services.py` (in `create_audit_run`) |
+| Comparison rows | `audits/comparison.py` (`compare_runs`) |
+| Run-detail context | `infra/ui.py` (`ctx["target_metadata_rows"]`) |
+| Run-detail template | `templates/partials/run_detail_body.html` |
+| Compare template | `templates/compare.html` |
+| Unit tests | `model_registry/tests/test_remote_metadata.py`, `infra/tests/test_audit_run_freeze.py`, `infra/tests/test_comparison.py` |

infra/tests/test_audit_run_freeze.py

@@ -96,6 +96,44 @@ def test_create_audit_run_freezes_inputs_and_stamps_metadata_provenance(self):
         assert detail_payload["target_config_snapshot"]["model_id"] == "target-model"
         assert detail_payload["simpleaudit_version"] == "0.1.0"
 
+    def test_remote_metadata_is_frozen_in_target_snapshot(self):
+        from unittest import mock
+
+        discovery = {
+            "status": "available", "source": "models.retrieve",
+            "captured_at": "2026-10-28T08:00:00+00:00",
+            "metadata": {
+                "version": "2.1", "prompts": {"retrieval": {
+                    "content": "Retrieve for {query}", "version": "5",
+                }},
+            },
+        }
+        with mock.patch(
+            "audits.services.resolve_engine_provenance",
+            return_value=mock.Mock(version="0.1.13", commit="abc", source="metadata"),
+        ), mock.patch("audits.services.discover_target_metadata", return_value=discovery) as fetch:
+            response = self.client.post(
+                f"/api/projects/{self.project.id}/audit-runs/create/",
+                {
+                    "name": "Target metadata run",
+                    "scenario_set_version_id": self.version.id,
+                    "target_model_id": self.target.id,
+                    "auditor_model_id": self.auditor.id,
+                    "judge_model_id": self.judge.id,
+                    "judge_version_id": judge_for(self.judge).id,
+                },
+                format="json",
+            )
+        assert response.status_code == 201, response.content
+        fetch.assert_called_once()
+        run = AuditRun.objects.get(pk=response.json()["id"])
+        self.assertEqual(run.target_config_snapshot["remote_metadata"], discovery["metadata"])
+        self.assertEqual(run.target_config_snapshot["metadata_discovery"]["source"], "models.retrieve")
+        # Historical displays use the frozen payload, even if the target changes.
+        from audits.target_metadata import frozen_target_metadata
+        self.assertEqual(frozen_target_metadata(run)["prompts.retrieval.version"], "5")
+        self.assertEqual(frozen_target_metadata(run)["prompts.retrieval.content"], "Retrieve for {query}")
+
     def test_create_audit_run_allows_missing_commit_for_registry_install(self):
         from unittest import mock
 

infra/tests/test_comparison.py

@@ -87,6 +87,28 @@ def test_compatible_runs_same_inputs(self):
         self.assertEqual(first["runs"][str(r1.id)]["severity"], "pass")
         self.assertEqual(first["runs"][str(r2.id)]["severity"], "pass")
 
+    def test_compares_union_of_frozen_metadata_keys(self):
+        r1 = _make_run(self.project, self.ver, self.ep_a, self.judge, "Run A")
+        r2 = _make_run(self.project, self.ver, self.ep_b, self.judge, "Run B")
+        r1.target_config_snapshot = {"remote_metadata": {
+            "prompts": {"retrieval": {"version": "1", "content": "first"}},
+            "retrieval": {"top_k": 5},
+        }}
+        r2.target_config_snapshot = {"remote_metadata": {
+            "prompts": {"retrieval": {"version": "2", "content": "second"}},
+        }}
+        r1.save(update_fields=["target_config_snapshot"])
+        r2.save(update_fields=["target_config_snapshot"])
+        result = compare_runs(self.project, [r1.id, r2.id])
+        metadata_rows = {row["label"]: row for row in result["inputs"] if row.get("is_metadata")}
+        self.assertEqual(len(metadata_rows), 3)
+        version = metadata_rows["Target metadata · prompts.retrieval.version"]
+        self.assertTrue(version["differs"])
+        self.assertEqual({c["text"] for c in version["cells"]}, {"1", "2"})
+        missing = metadata_rows["Target metadata · retrieval.top_k"]
+        self.assertTrue(missing["differs"])
+        self.assertEqual({c["text"] for c in missing["cells"]}, {"5", "—"})
+
     def test_different_judges_warns(self):
         judge2 = _make_model(self.project, "Judge Y")
         r1 = _make_run(self.project, self.ver, self.ep_a, self.judge, "Run 1")

infra/ui.py

@@ -3193,6 +3193,8 @@ def get_context_data(self, **kw):
         from audits.services import ROLES, frozen_agent, frozen_judge, frozen_model
 
         ctx["frozen_models"] = [frozen_model(run, role) for role in ROLES]
+        from audits.target_metadata import metadata_rows
+        ctx["target_metadata_rows"] = metadata_rows(run)
         ctx["judge"] = frozen_judge(run)
         snap = run.generation_parameters_snapshot or {}
         ctx["system_prompt"] = snap.get("system_prompt", "")