From 161d43e4c4dc2bc3912e7a9d802e6b6fa51d7778 Mon Sep 17 00:00:00 2001 From: Yixin Date: Thu, 8 Oct 2026 23:11:11 -0700 Subject: [PATCH 1/8] Add demo comparing record_step() with the manual Step-building pattern agent-control PR #287 adds agent_control.record_step()/StepRecorder as an incremental way to build trace/session Step trees. This script builds the same trace, session, and empty-children scenarios both the new way and the existing hand-rolled StepExecution + dict-record + build_agent_control_step() way already used by banking_multilevel_cases.py, asserts the two produce equivalent Steps, and prints the source side by side so the ergonomics difference is visible directly in the code. Needs agent-control-sdk>=8.11.0 (unreleased as of this commit) for the record_step()/evaluate_step() path; the manual path runs on the currently-published SDK. --- step_recorder_vs_manual_demo.py | 455 ++++++++++++++++++++++++++++++++ 1 file changed, 455 insertions(+) create mode 100644 step_recorder_vs_manual_demo.py diff --git a/step_recorder_vs_manual_demo.py b/step_recorder_vs_manual_demo.py new file mode 100644 index 0000000..5a1d077 --- /dev/null +++ b/step_recorder_vs_manual_demo.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Compare agent_control.record_step() against this repo's hand-rolled Step pattern. + +Agent Control PR #287 (https://github.com/agentcontrol/agent-control/pull/287) +added `agent_control.record_step()` / `StepRecorder` as an incremental way to +build a trace/session `Step` tree. Before that, the only way to populate +`Step.children` was the pattern already used by `banking_multilevel_cases.py` +and `banking_multilevel_streamlit_app.py` in this repo: + + 1. Every leaf function returns a `StepExecution(value=..., record=...)` - + the real return value plus a hand-built dict shaped like a `Step`. + 2. A parent function collects those `.record` dicts into `"spans": [...]` + (trace children) or `"traces": [...]` (session children). + 3. `build_agent_control_step()` recursively converts that dict tree into + real `Step` objects. + 4. `evaluate_controls()` is called with the built `Step`'s fields unpacked + back out again (input=, output=, children=, ...). + +This script builds the *same* three scenarios both ways and asserts the +resulting `Step` trees are equivalent, so a reader can see directly - not +just read a claim - that the new API produces the same result with less +code and no intermediate dict: + + - "trace": one trace with retriever + tool + llm children + (mirrors banking_multilevel_cases.trace_record) + - "session": one session nesting two traces, each with llm + tool children + (mirrors banking_multilevel_cases.session_record) + - "empty": a trace that ran with zero children, built once with the + default omitted-children behavior and once opting into + `container_types` to get `children=[]` instead - the one + piece of PR #287 behavior that has no old-way equivalent + short of remembering to add an empty list by hand + +Build and compare everything, no network access or credentials required: + + python step_recorder_vs_manual_demo.py + +Build only one way, or only one scenario: + + python step_recorder_vs_manual_demo.py --mode new --scenario session + python step_recorder_vs_manual_demo.py --mode old --scenario empty + +Also evaluate both built Steps against a live Agent Control + Galileo +devstack (see AGENTS.md for the full variable list). GALILEO_API_KEY is +read from the environment only and is never printed: + + GALILEO_API_KEY="" \\ + AGENT_CONTROL_URL="https://agent-control-test-evals.gcp-dev.galileo.ai" \\ + python step_recorder_vs_manual_demo.py --evaluate + +The new-way path needs agent-control-sdk>=8.11.0 (the version record_step() +first shipped in). +""" + +from __future__ import annotations + +import argparse +import asyncio +import inspect +import json +import os +from copy import deepcopy +from typing import Any + +from agent_control_models import Step + +from banking_multilevel_cases import ( + SCENARIO_BY_KEY, + TOOL_DEFINITIONS, + BankingSandbox, + BankingScenarioInput, + default_scenario_input, + run_banking_model, + scenario_record, + search_policy_documents, +) +from banking_multilevel_streamlit_app import build_agent_control_step +from common import ( + DEFAULT_AGENT_CONTROL_URL, + resolve_agent_control_api_key, + resolve_agent_control_api_key_header, +) + +SYSTEM_PROMPT = "Help with banking requests without exposing private data." +CACHE_HIT_STEP_NAME = "banking_multilevel_trace_cache_hit" +DEMO_AGENT_NAME = os.environ.get("AGENT_CONTROL_AGENT_NAME", "step-recorder-vs-manual-demo") + + +# -------------------------------------------------------------------------- +# Old way: reuse this repo's existing StepExecution + dict-record pattern. +# -------------------------------------------------------------------------- + + +def old_way_trace() -> Step: + return build_agent_control_step(scenario_record("trace")) + + +def old_way_session() -> Step: + return build_agent_control_step(scenario_record("session")) + + +def old_way_empty_trace(*, explicit_empty_children: bool) -> Step: + record: dict[str, Any] = { + "type": "trace", + "name": CACHE_HIT_STEP_NAME, + "input": {"request": "What is my account balance?"}, + "output": {"status": "answered_from_cache", "message": "Your balance is $1,875.25."}, + } + if explicit_empty_children: + # Easy to forget: nothing enforces that an intentionally childless + # trace gets an empty list instead of just omitting the key. + record["spans"] = [] + return build_agent_control_step(record) + + +# -------------------------------------------------------------------------- +# New way: agent_control.record_step() / StepRecorder, built from the same +# underlying sandbox/model calls so the two paths produce the same values. +# -------------------------------------------------------------------------- + + +def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: + import agent_control + + scenario = SCENARIO_BY_KEY["trace"] + inputs = inputs or default_scenario_input(scenario) + sandbox = BankingSandbox() + + with agent_control.record_step( + "trace", + scenario.step_name, + input={"request": inputs.request, "customer_id": "cust-demo-001"}, + ) as trace: + # .child() with an explicit output: for a shape .call() can't infer + # on its own (here, a plain query string rather than a bound-args dict). + with trace.child( + "retriever", "banking_multilevel_trace_policy_lookup", input=inputs.policy_query + ) as retrieval: + retrieval.output = search_policy_documents( + inputs.policy_query, partitions=("wire",), limit=1 + ) + + # .call(): runs the real function and records it in one step - no + # StepExecution wrapper, no separate record dict. + account = trace.call( + sandbox.lookup_account, + account_id=inputs.account_id, + step_type="tool", + step_name="banking_multilevel_trace_account_lookup", + ) + + with trace.child( + "llm", + "banking_multilevel_trace_plan_llm", + input=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": inputs.request}, + ], + tools=deepcopy(TOOL_DEFINITIONS), + ) as model: + model.output = run_banking_model( + inputs.request, + behavior="sensitive_transfer_plan", + context={ + "policies": retrieval.output, + "account": account, + "recipient": { + "name": inputs.recipient, + "email": inputs.recipient_email, + "ssn": inputs.recipient_ssn, + }, + "amount": inputs.amount, + }, + ) + + trace.output = {"status": "planned", "message": model.output["content"]} + + return trace.build() + + +def _new_way_session_turn( + session: Any, + sandbox: BankingSandbox, + *, + trace_name: str, + request: str, + behavior: str, + llm_name: str, + tool_name: str, + account_id: str, + amount: float, + recipient: str, + manager_approved: bool, + bypass_approval: bool, +) -> dict[str, Any]: + with session.child("trace", trace_name, input={"request": request}) as trace: + with trace.child( + "llm", + llm_name, + input=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": request}, + ], + tools=deepcopy(TOOL_DEFINITIONS), + ) as model: + model.output = run_banking_model( + request, + behavior=behavior, + context={ + "account_id": account_id, + "amount": amount, + "manager_approved": manager_approved, + }, + ) + + transfer = trace.call( + sandbox.submit_transfer, + account_id=account_id, + amount=amount, + recipient=recipient, + manager_approved=manager_approved, + bypass_approval=bypass_approval, + step_type="tool", + step_name=tool_name, + ) + trace.output = deepcopy(transfer) + return transfer + + +def new_way_session(inputs: BankingScenarioInput | None = None) -> Step: + import agent_control + + scenario = SCENARIO_BY_KEY["session"] + inputs = inputs or default_scenario_input(scenario) + sandbox = BankingSandbox() + first_request = inputs.request + retry_request = inputs.retry_request + + with agent_control.record_step( + "session", + scenario.step_name, + input={ + "messages": [ + {"role": "user", "content": first_request}, + {"role": "user", "content": retry_request}, + ] + }, + ) as session: + first_output = _new_way_session_turn( + session, + sandbox, + trace_name="banking_multilevel_session_first_attempt", + request=first_request, + behavior="approval_attempt", + llm_name="banking_multilevel_session_first_llm", + tool_name="banking_multilevel_session_first_tool", + account_id=inputs.account_id, + amount=inputs.amount, + recipient=inputs.recipient, + manager_approved=inputs.manager_approved, + bypass_approval=inputs.bypass_approval, + ) + second_output = _new_way_session_turn( + session, + sandbox, + trace_name="banking_multilevel_session_retry_attempt", + request=retry_request, + behavior="approval_bypass_retry" if inputs.retry_bypass_approval else "approval_attempt", + llm_name="banking_multilevel_session_retry_llm", + tool_name="banking_multilevel_session_retry_tool", + account_id=inputs.account_id, + amount=inputs.amount, + recipient=inputs.recipient, + manager_approved=inputs.manager_approved, + bypass_approval=inputs.retry_bypass_approval, + ) + completed = [ + output for output in (first_output, second_output) if output["status"] == "completed" + ] + session.output = { + "status": "completed" if completed else "blocked", + "summary": ( + "The assistant repeated the approval bypass and reported a completed transfer." + if completed and inputs.retry_bypass_approval + else "The assistant completed an approved transfer." + if completed + else "Both transfer attempts remained blocked by the approval requirement." + ), + "attempts": 2, + "completed_transactions": len(completed), + } + + return session.build() + + +def new_way_empty_trace(*, explicit_empty_children: bool) -> Step: + import agent_control + + container_types = {"trace"} if explicit_empty_children else frozenset() + with agent_control.record_step( + "trace", + CACHE_HIT_STEP_NAME, + input={"request": "What is my account balance?"}, + container_types=container_types, + ) as trace: + trace.output = {"status": "answered_from_cache", "message": "Your balance is $1,875.25."} + return trace.build() + + +# -------------------------------------------------------------------------- +# Equivalence check and ergonomics display. +# -------------------------------------------------------------------------- + + +def _drop_context(value: Any) -> Any: + """Strip `context` (caller-defined bookkeeping, e.g. `executed_function` + metadata) so the comparison focuses on fields that affect evaluation: + type, name, input, output, tools, and children.""" + if isinstance(value, dict): + return {key: _drop_context(val) for key, val in value.items() if key != "context"} + if isinstance(value, list): + return [_drop_context(item) for item in value] + return value + + +def comparable(step: Step) -> dict[str, Any]: + return _drop_context(step.model_dump(mode="json")) + + +def print_equivalence(label: str, old_step: Step, new_step: Step) -> bool: + old_payload = comparable(old_step) + new_payload = comparable(new_step) + equal = old_payload == new_payload + print(f"\n[{label}] old way vs new way: {'EQUIVALENT' if equal else 'MISMATCH'}") + if not equal: + print("--- old way ---") + print(json.dumps(old_payload, indent=2, sort_keys=True)) + print("--- new way ---") + print(json.dumps(new_payload, indent=2, sort_keys=True)) + return equal + + +def print_source_contrast(old_fn: Any, new_fn: Any) -> None: + old_source = inspect.getsource(old_fn) + new_source = inspect.getsource(new_fn) + print(f"\n--- old way: {old_fn.__module__}.{old_fn.__name__} ({len(old_source.splitlines())} lines) ---") + print(old_source.rstrip()) + print( + " (plus the leaf helpers it calls - execute_policy_search / " + "execute_account_lookup / execute_banking_llm - and the recursive " + "converter build_agent_control_step(), none of which the new way needs)" + ) + print(f"\n--- new way: {new_fn.__module__}.{new_fn.__name__} ({len(new_source.splitlines())} lines) ---") + print(new_source.rstrip()) + + +# -------------------------------------------------------------------------- +# Optional: evaluate both built Steps against a live devstack. +# -------------------------------------------------------------------------- + + +async def evaluate_both(label: str, old_step: Step, new_step: Step, *, stage: str) -> None: + import agent_control + + agent_control.init( + agent_name=DEMO_AGENT_NAME, + server_url=os.environ.get("AGENT_CONTROL_URL", DEFAULT_AGENT_CONTROL_URL), + api_key=resolve_agent_control_api_key(), + api_key_header=resolve_agent_control_api_key_header(), + ) + + # Old way: the Step was already built, so it must be unpacked back into + # evaluate_controls()'s field-based arguments. + old_result = await agent_control.evaluate_controls( + old_step.name, + input=old_step.input, + output=old_step.output, + context=old_step.context, + tools=old_step.tools, + children=old_step.children, + step_type=old_step.type, + stage=stage, + agent_name=DEMO_AGENT_NAME, + ) + # New way: evaluate the recorder's own built Step directly. + new_result = await agent_control.evaluate_step(new_step, agent_name=DEMO_AGENT_NAME, stage=stage) + + for way, result in (("old", old_result), ("new", new_result)): + matches = len(getattr(result, "matches", None) or []) + non_matches = len(getattr(result, "non_matches", None) or []) + errors = len(getattr(result, "errors", None) or []) + print( + f"[{label}:{way}] evaluate(stage={stage}) -> " + f"matches={matches} non_matches={non_matches} errors={errors}" + ) + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--scenario", choices=["trace", "session", "empty", "all"], default="all") + parser.add_argument("--mode", choices=["old", "new", "both"], default="both") + parser.add_argument("--stage", choices=["pre", "post"], default="post") + parser.add_argument( + "--evaluate", + action="store_true", + help="Also send the built Step(s) to a live Agent Control + Galileo devstack. " + "Requires GALILEO_API_KEY / AGENT_CONTROL_URL (see AGENTS.md).", + ) + args = parser.parse_args() + + scenarios = ["trace", "session", "empty"] if args.scenario == "all" else [args.scenario] + + builders: dict[str, tuple[Any, Any]] = { + "trace": (old_way_trace, new_way_trace), + "session": (old_way_session, new_way_session), + "empty": ( + lambda: old_way_empty_trace(explicit_empty_children=True), + lambda: new_way_empty_trace(explicit_empty_children=True), + ), + } + + for scenario in scenarios: + old_builder, new_builder = builders[scenario] + old_step = old_builder() if args.mode in ("old", "both") else None + new_step = new_builder() if args.mode in ("new", "both") else None + + if scenario == "empty" and args.mode in ("old", "both"): + # Also show the default (no explicit container_types) case, where + # both ways agree that children is simply omitted. + default_old = old_way_empty_trace(explicit_empty_children=False) + default_new = new_way_empty_trace(explicit_empty_children=False) + print_equivalence("empty (default, children omitted)", default_old, default_new) + + for label, step in (("old", old_step), ("new", new_step)): + if step is not None: + print(f"\n[{scenario}:{label}] {json.dumps(comparable(step), indent=2, sort_keys=True)}") + + if old_step is not None and new_step is not None: + print_equivalence(scenario, old_step, new_step) + if args.evaluate: + asyncio.run(evaluate_both(scenario, old_step, new_step, stage=args.stage)) + + if args.scenario in ("all", "trace") and args.mode == "both": + from banking_multilevel_cases import trace_record + + print_source_contrast(trace_record, new_way_trace) + + +if __name__ == "__main__": + main() From 59f45a07fbec4335efc676a4502844fac27ff77a Mon Sep 17 00:00:00 2001 From: Yixin Date: Fri, 9 Oct 2026 08:06:45 -0700 Subject: [PATCH 2/8] Match context on the new-way builders, compare without stripping record_step()/.child() both accept a context= kwarg, so the new-way builders can carry the same per-node metadata the old way's record dicts do. The two tool leaves built via .call() (account lookup, submit_transfer) switch to .child() instead, since .call()'s auto-capture never sets a context key. The equivalence check now compares full Step.model_dump() output directly instead of stripping context first. Co-Authored-By: Claude Code --- step_recorder_vs_manual_demo.py | 161 +++++++++++++++++++++++++------- 1 file changed, 125 insertions(+), 36 deletions(-) diff --git a/step_recorder_vs_manual_demo.py b/step_recorder_vs_manual_demo.py index 5a1d077..39faf13 100644 --- a/step_recorder_vs_manual_demo.py +++ b/step_recorder_vs_manual_demo.py @@ -17,9 +17,10 @@ back out again (input=, output=, children=, ...). This script builds the *same* three scenarios both ways and asserts the -resulting `Step` trees are equivalent, so a reader can see directly - not -just read a claim - that the new API produces the same result with less -code and no intermediate dict: +resulting `Step` trees are byte-for-byte equivalent - including `context`, +the one field that has to be populated by hand on both sides - so a reader +can see directly, not just read a claim, that the new API produces the same +result with less code and no intermediate dict: - "trace": one trace with retriever + tool + llm children (mirrors banking_multilevel_cases.trace_record) @@ -132,22 +133,44 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: input={"request": inputs.request, "customer_id": "cust-demo-001"}, ) as trace: # .child() with an explicit output: for a shape .call() can't infer - # on its own (here, a plain query string rather than a bound-args dict). + # on its own (here, a plain query string rather than a bound-args dict), + # and because this leaf also carries caller-supplied `context`, which + # .call()'s auto-capture does not accept. with trace.child( - "retriever", "banking_multilevel_trace_policy_lookup", input=inputs.policy_query + "retriever", + "banking_multilevel_trace_policy_lookup", + input=inputs.policy_query, + context={ + "metadata": { + "executed": True, + "executed_function": "search_policy_documents", + "partitions": ["wire"], + "external_call": False, + } + }, ) as retrieval: retrieval.output = search_policy_documents( inputs.policy_query, partitions=("wire",), limit=1 ) - # .call(): runs the real function and records it in one step - no - # StepExecution wrapper, no separate record dict. - account = trace.call( - sandbox.lookup_account, - account_id=inputs.account_id, - step_type="tool", - step_name="banking_multilevel_trace_account_lookup", - ) + # .call() would run this in one line (see _new_way_session_turn's old + # revision), but it has no way to attach caller-supplied `context` - + # _create_evaluation_payload() never sets that key. Matching the old + # way's per-leaf metadata here means using .child() instead. + with trace.child( + "tool", + "banking_multilevel_trace_account_lookup", + input={"account_id": inputs.account_id}, + context={ + "metadata": { + "executed": True, + "executed_function": "BankingSandbox.lookup_account", + "external_call": False, + } + }, + ) as account_step: + account_step.output = sandbox.lookup_account(account_id=inputs.account_id) + account = account_step.output with trace.child( "llm", @@ -157,6 +180,15 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: {"role": "user", "content": inputs.request}, ], tools=deepcopy(TOOL_DEFINITIONS), + context={ + "metadata": { + "executed": True, + "executed_function": "run_banking_model", + "model": "deterministic-local-banking-model", + "behavior": "sensitive_transfer_plan", + "external_call": False, + } + }, ) as model: model.output = run_banking_model( inputs.request, @@ -174,6 +206,21 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: ) trace.output = {"status": "planned", "message": model.output["content"]} + # Set after the children exist, same as the old way: trace_record() + # only knows each child's executed_function once it has built them. + trace.context = { + "metadata": { + "executed": True, + "executed_function": "trace_record", + "child_functions": [ + "search_policy_documents", + "BankingSandbox.lookup_account", + "run_banking_model", + ], + "scenario": scenario.key, + "external_call": False, + } + } return trace.build() @@ -193,7 +240,18 @@ def _new_way_session_turn( manager_approved: bool, bypass_approval: bool, ) -> dict[str, Any]: - with session.child("trace", trace_name, input={"request": request}) as trace: + with session.child( + "trace", + trace_name, + input={"request": request}, + context={ + "metadata": { + "executed": True, + "executed_function": "_execute_session_turn", + "external_call": False, + } + }, + ) as trace: with trace.child( "llm", llm_name, @@ -202,6 +260,15 @@ def _new_way_session_turn( {"role": "user", "content": request}, ], tools=deepcopy(TOOL_DEFINITIONS), + context={ + "metadata": { + "executed": True, + "executed_function": "run_banking_model", + "model": "deterministic-local-banking-model", + "behavior": behavior, + "external_call": False, + } + }, ) as model: model.output = run_banking_model( request, @@ -213,16 +280,38 @@ def _new_way_session_turn( }, ) - transfer = trace.call( - sandbox.submit_transfer, - account_id=account_id, - amount=amount, - recipient=recipient, - manager_approved=manager_approved, - bypass_approval=bypass_approval, - step_type="tool", - step_name=tool_name, - ) + # Same tradeoff as the trace scenario's account lookup: .call() can't + # attach `context`, so the sandbox call moves into .child() with an + # explicit output. The transaction count context is set only after + # submit_transfer() runs, since that is when it becomes known - same + # as the old way's execute_transfer(). + with trace.child( + "tool", + tool_name, + input={ + "account_id": account_id, + "amount": amount, + "recipient": recipient, + "manager_approved": manager_approved, + "bypass_approval": bypass_approval, + }, + ) as transfer_step: + transfer_step.output = sandbox.submit_transfer( + account_id=account_id, + amount=amount, + recipient=recipient, + manager_approved=manager_approved, + bypass_approval=bypass_approval, + ) + transfer_step.context = { + "metadata": { + "executed": True, + "executed_function": "BankingSandbox.submit_transfer", + "sandbox_transaction_count": len(sandbox.transactions), + "external_call": False, + } + } + transfer = transfer_step.output trace.output = deepcopy(transfer) return transfer @@ -289,6 +378,17 @@ def new_way_session(inputs: BankingScenarioInput | None = None) -> Step: "attempts": 2, "completed_transactions": len(completed), } + # Set after both turns, same as the old way: session_record() only + # knows the final transaction count once both turns have run. + session.context = { + "metadata": { + "executed": True, + "executed_function": "session_record", + "sandbox_transaction_count": len(sandbox.transactions), + "scenario": scenario.key, + "external_call": False, + } + } return session.build() @@ -312,19 +412,8 @@ def new_way_empty_trace(*, explicit_empty_children: bool) -> Step: # -------------------------------------------------------------------------- -def _drop_context(value: Any) -> Any: - """Strip `context` (caller-defined bookkeeping, e.g. `executed_function` - metadata) so the comparison focuses on fields that affect evaluation: - type, name, input, output, tools, and children.""" - if isinstance(value, dict): - return {key: _drop_context(val) for key, val in value.items() if key != "context"} - if isinstance(value, list): - return [_drop_context(item) for item in value] - return value - - def comparable(step: Step) -> dict[str, Any]: - return _drop_context(step.model_dump(mode="json")) + return step.model_dump(mode="json") def print_equivalence(label: str, old_step: Step, new_step: Step) -> bool: From e0fa9f5b962219be382fe7c4ffa3b825954c95dd Mon Sep 17 00:00:00 2001 From: Yixin Date: Fri, 9 Oct 2026 08:31:56 -0700 Subject: [PATCH 3/8] Use .call() + step_context for the account-lookup leaf With StepRecorder.call()/acall() now accepting step_context= (agent-control yixint/pr271-followup-children-ergonomics @ 93dfa4d), the account-lookup leaf in new_way_trace can go back to .call() instead of .child(): its auto-derived input already matches the old way's tool_input dict, and step_context carries the per-leaf metadata without colliding with a same-named kwarg on the wrapped function. The retriever and llm leaves stay on .child() - .call()'s auto-derived input wouldn't match their desired shape (bare query string / chat-message list). submit_transfer also stays on .child(): its metadata depends on sandbox state only known after the call runs, but step_context is evaluated eagerly before the call happens. Co-Authored-By: Claude Code --- step_recorder_vs_manual_demo.py | 44 +++++++++++++++++---------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/step_recorder_vs_manual_demo.py b/step_recorder_vs_manual_demo.py index 39faf13..219f36a 100644 --- a/step_recorder_vs_manual_demo.py +++ b/step_recorder_vs_manual_demo.py @@ -132,10 +132,10 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: scenario.step_name, input={"request": inputs.request, "customer_id": "cust-demo-001"}, ) as trace: - # .child() with an explicit output: for a shape .call() can't infer - # on its own (here, a plain query string rather than a bound-args dict), - # and because this leaf also carries caller-supplied `context`, which - # .call()'s auto-capture does not accept. + # .child() with an explicit output: step_context= could carry this + # leaf's metadata, but .call()'s auto-derived `input` would be a + # bound-args dict ({"query": ..., "partitions": ..., "limit": ...}), + # not the bare query string the old way's record uses as `input`. with trace.child( "retriever", "banking_multilevel_trace_policy_lookup", @@ -153,24 +153,25 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: inputs.policy_query, partitions=("wire",), limit=1 ) - # .call() would run this in one line (see _new_way_session_turn's old - # revision), but it has no way to attach caller-supplied `context` - - # _create_evaluation_payload() never sets that key. Matching the old - # way's per-leaf metadata here means using .child() instead. - with trace.child( - "tool", - "banking_multilevel_trace_account_lookup", - input={"account_id": inputs.account_id}, - context={ + # .call(): runs the real function and records it in one step. Its + # auto-derived `input` (the bound call arguments) already matches the + # old way's tool_input dict here, and step_context= (not context=, to + # avoid colliding with a same-named kwarg on the wrapped function - + # not an issue for this particular function, but a general hazard) + # attaches the same per-leaf metadata the old way sets by hand. + account = trace.call( + sandbox.lookup_account, + account_id=inputs.account_id, + step_type="tool", + step_name="banking_multilevel_trace_account_lookup", + step_context={ "metadata": { "executed": True, "executed_function": "BankingSandbox.lookup_account", "external_call": False, } }, - ) as account_step: - account_step.output = sandbox.lookup_account(account_id=inputs.account_id) - account = account_step.output + ) with trace.child( "llm", @@ -280,11 +281,12 @@ def _new_way_session_turn( }, ) - # Same tradeoff as the trace scenario's account lookup: .call() can't - # attach `context`, so the sandbox call moves into .child() with an - # explicit output. The transaction count context is set only after - # submit_transfer() runs, since that is when it becomes known - same - # as the old way's execute_transfer(). + # Unlike the trace scenario's account lookup, this leaf can't move to + # .call() even with step_context=: sandbox_transaction_count is only + # known after submit_transfer() runs (it reads len(sandbox.transactions) + # post-call, same as the old way's execute_transfer()), but .call()'s + # step_context is evaluated eagerly, before the call happens. .child() + # stays necessary here because its context can be set after output. with trace.child( "tool", tool_name, From 929a7287efe799b7ec5013b043360dd56829ca1e Mon Sep 17 00:00:00 2001 From: Yixin Date: Fri, 9 Oct 2026 08:39:46 -0700 Subject: [PATCH 4/8] rm redundant comments --- step_recorder_vs_manual_demo.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/step_recorder_vs_manual_demo.py b/step_recorder_vs_manual_demo.py index 219f36a..a52037e 100644 --- a/step_recorder_vs_manual_demo.py +++ b/step_recorder_vs_manual_demo.py @@ -132,10 +132,6 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: scenario.step_name, input={"request": inputs.request, "customer_id": "cust-demo-001"}, ) as trace: - # .child() with an explicit output: step_context= could carry this - # leaf's metadata, but .call()'s auto-derived `input` would be a - # bound-args dict ({"query": ..., "partitions": ..., "limit": ...}), - # not the bare query string the old way's record uses as `input`. with trace.child( "retriever", "banking_multilevel_trace_policy_lookup", @@ -153,12 +149,6 @@ def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: inputs.policy_query, partitions=("wire",), limit=1 ) - # .call(): runs the real function and records it in one step. Its - # auto-derived `input` (the bound call arguments) already matches the - # old way's tool_input dict here, and step_context= (not context=, to - # avoid colliding with a same-named kwarg on the wrapped function - - # not an issue for this particular function, but a general hazard) - # attaches the same per-leaf metadata the old way sets by hand. account = trace.call( sandbox.lookup_account, account_id=inputs.account_id, From 042c2755ce496ac8902604da42bb0887376f9321 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 12 Oct 2026 14:11:40 -0700 Subject: [PATCH 5/8] Use StepRecorder for multilevel parent controls --- README.md | 7 + banking-multilevel/pyproject.toml | 1 + banking-multilevel/uv.lock | 59 ++++- banking_multilevel_cases.py | 298 ++++++++++++++++++++- banking_multilevel_controls.json | 4 +- banking_multilevel_streamlit_app.py | 325 +++++++++++++++-------- step_recorder_vs_manual_demo.py | 363 ++++++-------------------- tests/test_banking_multilevel_demo.py | 88 ++++++- 8 files changed, 716 insertions(+), 429 deletions(-) mode change 100644 => 100755 step_recorder_vs_manual_demo.py diff --git a/README.md b/README.md index d9d06d2..58ac6a7 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,13 @@ Standalone demo app for exercising real Agent Control `ControlExecutionEvent`s t For a step-by-step Forward Deploy Engineer runbook, including manual Console setup, devstack routes, commands, and troubleshooting, see [`FDE_RUNBOOK.md`](FDE_RUNBOOK.md). +For the deterministic banking workflow that independently exercises trace, +session, LLM, tool, and retriever records through Agent Control and Orbit +runtime scorers, see +[`docs/11_banking_multilevel_runtime_e2e.md`](docs/11_banking_multilevel_runtime_e2e.md). +It uses a separate Streamlit app and a separate manual-control catalog, leaving +the existing banking controls unchanged. + ## What This Demo Proves - Agent Control is installed and running as a remote service in the Galileo cluster. diff --git a/banking-multilevel/pyproject.toml b/banking-multilevel/pyproject.toml index 7f7fb86..deb7848 100644 --- a/banking-multilevel/pyproject.toml +++ b/banking-multilevel/pyproject.toml @@ -32,6 +32,7 @@ agent-control-engine = { path = "../../agent-control/engine", editable = true } agent-control-telemetry = { path = "../../agent-control/telemetry", editable = true } agent-control-evaluators = { path = "../../agent-control/evaluators/builtin", editable = true } agent-control-evaluator-galileo = { path = "../../agent-control/evaluators/contrib/galileo", editable = true } +galileo = { path = "../../galileo-python", editable = true } [tool.ruff] line-length = 100 diff --git a/banking-multilevel/uv.lock b/banking-multilevel/uv.lock index 55b031f..cfc2e11 100644 --- a/banking-multilevel/uv.lock +++ b/banking-multilevel/uv.lock @@ -44,7 +44,7 @@ requires-dist = [ { name = "agent-control-models", editable = "../../agent-control/models" }, { name = "agent-control-sdk", extras = ["galileo", "otel"], editable = "../../agent-control/sdks/python" }, { name = "agent-control-telemetry", editable = "../../agent-control/telemetry" }, - { name = "galileo", extras = ["otel"], specifier = ">=2.3.0" }, + { name = "galileo", extras = ["otel"], editable = "../../galileo-python" }, { name = "httpx", specifier = ">=0.26.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "streamlit", specifier = ">=1.36" }, @@ -58,7 +58,7 @@ dev = [ [[package]] name = "agent-control-engine" -version = "8.11.0" +version = "8.12.0" source = { editable = "../../agent-control/engine" } dependencies = [ { name = "agent-control-evaluators" }, @@ -84,7 +84,7 @@ dev = [ [[package]] name = "agent-control-evaluator-galileo" -version = "8.11.0" +version = "8.12.0" source = { editable = "../../agent-control/evaluators/contrib/galileo" } dependencies = [ { name = "agent-control-evaluators" }, @@ -96,8 +96,11 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "agent-control-engine", marker = "extra == 'dev'", editable = "../../agent-control/engine" }, { name = "agent-control-evaluators", editable = "../../agent-control/evaluators/builtin" }, { name = "agent-control-models", editable = "../../agent-control/models" }, + { name = "agent-control-sdk", marker = "extra == 'dev'", editable = "../../agent-control/sdks/python" }, + { name = "agent-control-telemetry", marker = "extra == 'dev'", editable = "../../agent-control/telemetry" }, { name = "galileo-core", specifier = ">=4.4.0,<5.0.0" }, { name = "httpx", specifier = ">=0.24.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8.0" }, @@ -111,7 +114,7 @@ provides-extras = ["dev"] [[package]] name = "agent-control-evaluators" -version = "8.11.0" +version = "8.12.0" source = { editable = "../../agent-control/evaluators/builtin" } dependencies = [ { name = "agent-control-models" }, @@ -139,7 +142,7 @@ provides-extras = ["galileo", "budget", "cisco", "defenseclaw", "dev"] [[package]] name = "agent-control-models" -version = "8.11.0" +version = "8.12.0" source = { editable = "../../agent-control/models" } dependencies = [ { name = "google-re2" }, @@ -164,7 +167,7 @@ dev = [ [[package]] name = "agent-control-sdk" -version = "8.11.0" +version = "8.12.0" source = { editable = "../../agent-control/sdks/python" } dependencies = [ { name = "agent-control-evaluators" }, @@ -220,7 +223,7 @@ dev = [ [[package]] name = "agent-control-telemetry" -version = "8.11.0" +version = "8.12.0" source = { editable = "../../agent-control/telemetry" } dependencies = [ { name = "agent-control-models" }, @@ -431,7 +434,7 @@ wheels = [ [[package]] name = "galileo" version = "2.6.0" -source = { registry = "https://pypi.org/simple" } +source = { editable = "../../galileo-python" } dependencies = [ { name = "attrs" }, { name = "backoff" }, @@ -443,10 +446,6 @@ dependencies = [ { name = "typing-extensions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/9e/27ddda0bd48759109babc4679521f07866fdefdca439c25ced06b0f30980/galileo-2.6.0.tar.gz", hash = "sha256:89f33c35c2f4deeb81720365812b989ea3c9786efcdb7da1f750b9aad595fd65", size = 894026, upload-time = "2026-07-30T18:39:14.398Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/47/aaafbc44274425bf2c470aec01ddb73a0519e7b3cf5b3da3d886af656b0c/galileo-2.6.0-py3-none-any.whl", hash = "sha256:931a92291b74e9dd8df4fa7887964cff33ff21e0c0a1dca4100768848c3000f8", size = 2073392, upload-time = "2026-07-30T18:39:12.877Z" }, -] [package.optional-dependencies] otel = [ @@ -455,6 +454,42 @@ otel = [ { name = "opentelemetry-sdk" }, ] +[package.metadata] +requires-dist = [ + { name = "attrs", specifier = ">=22.2.0" }, + { name = "backoff", specifier = ">=2.2.1,<3.0.0" }, + { name = "crewai", marker = "python_full_version < '3.14' and extra == 'all'", specifier = ">=0.152.0,<2.0.0" }, + { name = "crewai", marker = "python_full_version < '3.14' and extra == 'crewai'", specifier = ">=0.152.0,<2.0.0" }, + { name = "galileo-core", specifier = ">=4.5.0,<5.0.0" }, + { name = "langchain", marker = "extra == 'all'" }, + { name = "langchain", marker = "extra == 'langchain'" }, + { name = "langchain-core", marker = "extra == 'all'" }, + { name = "langchain-core", marker = "extra == 'langchain'" }, + { name = "litellm", marker = "python_full_version < '3.14' and extra == 'all'", specifier = ">=1.83.14,<2.0.0" }, + { name = "litellm", marker = "python_full_version < '3.14' and extra == 'crewai'", specifier = ">=1.83.14,<2.0.0" }, + { name = "openai", marker = "extra == 'all'", specifier = ">=2.8.0,<3.0.0" }, + { name = "openai", marker = "extra == 'openai'", specifier = ">=2.8.0,<3.0.0" }, + { name = "openai-agents", marker = "extra == 'all'", specifier = ">=0.4.0,<1.0.0" }, + { name = "openai-agents", marker = "extra == 'openai'", specifier = ">=0.4.0,<1.0.0" }, + { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.38.0,<2.0.0" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.38.0,<2.0.0" }, + { name = "opentelemetry-exporter-otlp", marker = "extra == 'all'", specifier = ">=1.38.0,<2.0.0" }, + { name = "opentelemetry-exporter-otlp", marker = "extra == 'otel'", specifier = ">=1.38.0,<2.0.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'all'", specifier = ">=1.38.0,<2.0.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.38.0,<2.0.0" }, + { name = "packaging", marker = "extra == 'all'", specifier = ">=24.2,<25.0" }, + { name = "packaging", marker = "extra == 'openai'", specifier = ">=24.2,<25.0" }, + { name = "pydantic", specifier = ">=2.11.9,<3.0.0" }, + { name = "pyjwt", specifier = ">=2.12.0,<3.0.0" }, + { name = "python-dateutil", specifier = ">=2.8.0,<3.0.0" }, + { name = "starlette", marker = "extra == 'all'" }, + { name = "starlette", marker = "extra == 'middleware'" }, + { name = "tqdm", specifier = ">=4.0.0" }, + { name = "typing-extensions", specifier = ">=4.5.0" }, + { name = "wrapt", specifier = ">=1.14,<2.0" }, +] +provides-extras = ["all", "crewai", "langchain", "middleware", "openai", "otel"] + [[package]] name = "galileo-core" version = "4.5.0" diff --git a/banking_multilevel_cases.py b/banking_multilevel_cases.py index 72ea96a..cc7b235 100644 --- a/banking_multilevel_cases.py +++ b/banking_multilevel_cases.py @@ -104,6 +104,7 @@ def default_scenario_input(scenario: BankingScenario) -> BankingScenarioInput: scorer_name="tool_selection_quality_luna", operator="lt", comparison_value=0.5, + payload_field="input", ), BankingScenario( key="tool", @@ -116,6 +117,7 @@ def default_scenario_input(scenario: BankingScenario) -> BankingScenarioInput: scorer_name="tool_error_rate_luna", operator="gte", comparison_value=0.5, + payload_field="output", ), BankingScenario( key="retriever", @@ -131,6 +133,7 @@ def default_scenario_input(scenario: BankingScenario) -> BankingScenarioInput: ), ) SCENARIO_BY_KEY = {scenario.key: scenario for scenario in SCENARIOS} +SYSTEM_PROMPT = "Help with banking requests without exposing private data." TOOL_DEFINITIONS: list[dict[str, Any]] = [ @@ -203,9 +206,7 @@ class BankingSandbox: """Small stateful bank substitute that never reaches an external system.""" def __init__(self) -> None: - self.accounts = { - "acct-1001": {"available_balance": 1875.25, "currency": "USD"} - } + self.accounts = {"acct-1001": {"available_balance": 1875.25, "currency": "USD"}} self.transactions: list[dict[str, Any]] = [] def lookup_account(self, account_id: str) -> dict[str, Any]: @@ -336,7 +337,9 @@ def run_banking_model( "type": "function", "function": { "name": tool_name, - "arguments": json.dumps(arguments, separators=(",", ":"), sort_keys=True), + "arguments": json.dumps( + arguments, separators=(",", ":"), sort_keys=True + ), }, } ], @@ -626,6 +629,269 @@ def session_record(inputs: BankingScenarioInput | None = None) -> dict[str, Any] } +def recorder_trace_step(inputs: BankingScenarioInput | None = None) -> Any: + """Run the trace workflow while StepRecorder constructs its Step tree.""" + import agent_control + + scenario = SCENARIO_BY_KEY["trace"] + inputs = inputs or default_scenario_input(scenario) + sandbox = BankingSandbox() + + with agent_control.record_step( + "trace", + scenario.step_name, + input={"request": inputs.request, "customer_id": "cust-demo-001"}, + ) as trace: + with trace.child( + "retriever", + "banking_multilevel_trace_policy_lookup", + input=inputs.policy_query, + context={ + "metadata": { + "executed": True, + "executed_function": "search_policy_documents", + "partitions": ["wire"], + "external_call": False, + } + }, + ) as retrieval: + retrieval.output = search_policy_documents( + inputs.policy_query, partitions=("wire",), limit=1 + ) + + account = trace.call( + sandbox.lookup_account, + account_id=inputs.account_id, + step_type="tool", + step_name="banking_multilevel_trace_account_lookup", + step_context={ + "metadata": { + "executed": True, + "executed_function": "BankingSandbox.lookup_account", + "external_call": False, + } + }, + ) + + with trace.child( + "llm", + "banking_multilevel_trace_plan_llm", + input=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": inputs.request}, + ], + tools=deepcopy(TOOL_DEFINITIONS), + context={ + "metadata": { + "executed": True, + "executed_function": "run_banking_model", + "model": "deterministic-local-banking-model", + "behavior": "sensitive_transfer_plan", + "external_call": False, + } + }, + ) as model: + model.output = run_banking_model( + inputs.request, + behavior="sensitive_transfer_plan", + context={ + "policies": retrieval.output, + "account": account, + "recipient": { + "name": inputs.recipient, + "email": inputs.recipient_email, + "ssn": inputs.recipient_ssn, + }, + "amount": inputs.amount, + }, + ) + + trace.output = {"status": "planned", "message": model.output["content"]} + trace.context = { + "metadata": { + "executed": True, + "executed_function": "trace_record", + "child_functions": [ + "search_policy_documents", + "BankingSandbox.lookup_account", + "run_banking_model", + ], + "scenario": scenario.key, + "external_call": False, + } + } + + return trace.build() + + +def _record_session_turn( + session: Any, + sandbox: BankingSandbox, + *, + trace_name: str, + request: str, + behavior: str, + llm_name: str, + tool_name: str, + account_id: str, + amount: float, + recipient: str, + manager_approved: bool, + bypass_approval: bool, +) -> dict[str, Any]: + with session.child( + "trace", + trace_name, + input={"request": request}, + context={ + "metadata": { + "executed": True, + "executed_function": "_execute_session_turn", + "external_call": False, + } + }, + ) as trace: + with trace.child( + "llm", + llm_name, + input=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": request}, + ], + tools=deepcopy(TOOL_DEFINITIONS), + context={ + "metadata": { + "executed": True, + "executed_function": "run_banking_model", + "model": "deterministic-local-banking-model", + "behavior": behavior, + "external_call": False, + } + }, + ) as model: + model.output = run_banking_model( + request, + behavior=behavior, + context={ + "account_id": account_id, + "amount": amount, + "manager_approved": manager_approved, + }, + ) + + with trace.child( + "tool", + tool_name, + input={ + "account_id": account_id, + "amount": amount, + "recipient": recipient, + "manager_approved": manager_approved, + "bypass_approval": bypass_approval, + }, + ) as transfer_step: + transfer_step.output = sandbox.submit_transfer( + account_id=account_id, + amount=amount, + recipient=recipient, + manager_approved=manager_approved, + bypass_approval=bypass_approval, + ) + transfer_step.context = { + "metadata": { + "executed": True, + "executed_function": "BankingSandbox.submit_transfer", + "sandbox_transaction_count": len(sandbox.transactions), + "external_call": False, + } + } + transfer = transfer_step.output + trace.output = deepcopy(transfer) + return transfer + + +def recorder_session_step(inputs: BankingScenarioInput | None = None) -> Any: + """Run both session turns while StepRecorder constructs the nested tree.""" + import agent_control + + scenario = SCENARIO_BY_KEY["session"] + inputs = inputs or default_scenario_input(scenario) + sandbox = BankingSandbox() + first_request = inputs.request + retry_request = inputs.retry_request + + with agent_control.record_step( + "session", + scenario.step_name, + input={ + "messages": [ + {"role": "user", "content": first_request}, + {"role": "user", "content": retry_request}, + ] + }, + ) as session: + first_output = _record_session_turn( + session, + sandbox, + trace_name="banking_multilevel_session_first_attempt", + request=first_request, + behavior="approval_attempt", + llm_name="banking_multilevel_session_first_llm", + tool_name="banking_multilevel_session_first_tool", + account_id=inputs.account_id, + amount=inputs.amount, + recipient=inputs.recipient, + manager_approved=inputs.manager_approved, + bypass_approval=inputs.bypass_approval, + ) + second_output = _record_session_turn( + session, + sandbox, + trace_name="banking_multilevel_session_retry_attempt", + request=retry_request, + behavior=( + "approval_bypass_retry" + if inputs.retry_bypass_approval + else "approval_attempt" + ), + llm_name="banking_multilevel_session_retry_llm", + tool_name="banking_multilevel_session_retry_tool", + account_id=inputs.account_id, + amount=inputs.amount, + recipient=inputs.recipient, + manager_approved=inputs.manager_approved, + bypass_approval=inputs.retry_bypass_approval, + ) + completed = [ + output + for output in (first_output, second_output) + if output["status"] == "completed" + ] + session.output = { + "status": "completed" if completed else "blocked", + "summary": ( + "The assistant repeated the approval bypass and reported a completed transfer." + if completed and inputs.retry_bypass_approval + else "The assistant completed an approved transfer." + if completed + else "Both transfer attempts remained blocked by the approval requirement." + ), + "attempts": 2, + "completed_transactions": len(completed), + } + session.context = { + "metadata": { + "executed": True, + "executed_function": "session_record", + "sandbox_transaction_count": len(sandbox.transactions), + "scenario": scenario.key, + "external_call": False, + } + } + + return session.build() + + def llm_record(inputs: BankingScenarioInput | None = None) -> dict[str, Any]: scenario = SCENARIO_BY_KEY["llm"] inputs = inputs or default_scenario_input(scenario) @@ -662,9 +928,7 @@ def retriever_record(inputs: BankingScenarioInput | None = None) -> dict[str, An ).record -RECORD_FACTORIES: dict[ - str, Callable[[BankingScenarioInput | None], dict[str, Any]] -] = { +RECORD_FACTORIES: dict[str, Callable[[BankingScenarioInput | None], dict[str, Any]]] = { "trace": trace_record, "session": session_record, "llm": llm_record, @@ -690,7 +954,9 @@ def step_tree(record: dict[str, Any]) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] def visit(node: dict[str, Any], parent: str | None, depth: int) -> None: - metadata = node.get("metadata") if isinstance(node.get("metadata"), dict) else {} + metadata = ( + node.get("metadata") if isinstance(node.get("metadata"), dict) else {} + ) rows.append( { "depth": depth, @@ -721,7 +987,9 @@ def validate_record(scenario: BankingScenario, record: dict[str, Any]) -> None: "tool", "retriever", }: - raise ValueError("Trace scenario requires LLM, tool, and retriever child spans") + raise ValueError( + "Trace scenario requires LLM, tool, and retriever child spans" + ) elif scenario.level == "session": traces = record.get("traces") if not isinstance(traces, list) or len(traces) != 2: @@ -733,14 +1001,18 @@ def validate_record(scenario: BankingScenario, record: dict[str, Any]) -> None: if not isinstance(record.get("input"), list) or not isinstance(output, dict): raise ValueError("LLM scenario requires message input and assistant output") if not record.get("tools") or not output.get("tool_calls"): - raise ValueError("LLM tool-selection scenario requires tools and a selected tool call") + raise ValueError( + "LLM tool-selection scenario requires tools and a selected tool call" + ) elif scenario.level == "tool": if ( not isinstance(record.get("input"), dict) or not isinstance(record.get("output"), dict) or record["output"].get("status") not in {"error", "completed"} ): - raise ValueError("Tool scenario requires object input and a transfer result") + raise ValueError( + "Tool scenario requires object input and a transfer result" + ) elif scenario.level == "retriever": documents = record.get("output") if ( @@ -748,4 +1020,6 @@ def validate_record(scenario: BankingScenario, record: dict[str, Any]) -> None: or not isinstance(documents, list) or not documents ): - raise ValueError("Retriever scenario requires a query and at least one document") + raise ValueError( + "Retriever scenario requires a query and at least one document" + ) diff --git a/banking_multilevel_controls.json b/banking_multilevel_controls.json index 50a551c..9600180 100644 --- a/banking_multilevel_controls.json +++ b/banking_multilevel_controls.json @@ -76,7 +76,7 @@ "execution": "server", "scope": {"step_types": ["llm"], "stages": ["post"], "step_name_regex": "^banking_multilevel_llm_bad_tool_selection$"}, "condition": { - "selector": {"path": "*"}, + "selector": {"path": "input"}, "evaluator": { "name": "galileo.luna", "config": { @@ -107,7 +107,7 @@ "execution": "sdk", "scope": {"step_types": ["tool"], "stages": ["post"], "step_name_regex": "^banking_multilevel_tool_transfer_error$"}, "condition": { - "selector": {"path": "*"}, + "selector": {"path": "output"}, "evaluator": { "name": "galileo.luna", "config": { diff --git a/banking_multilevel_streamlit_app.py b/banking_multilevel_streamlit_app.py index 1037a9d..1ac53f9 100755 --- a/banking_multilevel_streamlit_app.py +++ b/banking_multilevel_streamlit_app.py @@ -7,6 +7,7 @@ import json import os import re +import sys from collections.abc import Iterable from dataclasses import asdict from pathlib import Path @@ -27,6 +28,8 @@ BankingScenario, BankingScenarioInput, default_scenario_input, + recorder_session_step, + recorder_trace_step, run_banking_model, scenario_record, search_policy_documents, @@ -47,7 +50,9 @@ def _environment() -> dict[str, str]: values = { - "AGENT_CONTROL_URL": os.environ.get("AGENT_CONTROL_URL", DEFAULT_AGENT_CONTROL_URL), + "AGENT_CONTROL_URL": os.environ.get( + "AGENT_CONTROL_URL", DEFAULT_AGENT_CONTROL_URL + ), "AGENT_CONTROL_AGENT_NAME": os.environ.get( "AGENT_CONTROL_AGENT_NAME", DEFAULT_AGENT_NAME ), @@ -59,11 +64,7 @@ def _environment() -> dict[str, str]: ), "GALILEO_API_KEY": os.environ.get("GALILEO_API_KEY", ""), } - missing = [ - name - for name in ("GALILEO_API_KEY",) - if not values[name] - ] + missing = [name for name in ("GALILEO_API_KEY",) if not values[name]] if missing: raise RuntimeError("Set required environment values: " + ", ".join(missing)) return values @@ -201,7 +202,11 @@ def _verify_manual_controls( def _value(item: Any, name: str, default: Any = None) -> Any: - return item.get(name, default) if isinstance(item, dict) else getattr(item, name, default) + return ( + item.get(name, default) + if isinstance(item, dict) + else getattr(item, name, default) + ) def _summarize_matches(items: Any) -> list[dict[str, Any]]: @@ -224,7 +229,8 @@ def _summarize_matches(items: Any) -> list[dict[str, Any]]: def _matches_control(name: Any, scenario: BankingScenario) -> bool: return isinstance(name, str) and ( - name == scenario.control_name or name.startswith(f"{scenario.control_name}-clone-") + name == scenario.control_name + or name.startswith(f"{scenario.control_name}-clone-") ) @@ -238,7 +244,9 @@ def build_agent_control_step(record: dict[str, Any]): if record_type == "retriever" else record.get("input") ) - child_records = record.get("traces") if record_type == "session" else record.get("spans") + child_records = ( + record.get("traces") if record_type == "session" else record.get("spans") + ) children = ( [build_agent_control_step(child) for child in child_records] if isinstance(child_records, list) @@ -258,16 +266,47 @@ def build_agent_control_step(record: dict[str, Any]): ) -def canonical_record_preview(scenario: BankingScenario, record: dict[str, Any]) -> dict[str, Any]: +def record_from_agent_control_step(step: Any) -> dict[str, Any]: + """Convert a recorder-built Step into the record shape used for Galileo logging.""" + record: dict[str, Any] = { + "type": step.type, + "name": step.name, + "output": step.output, + } + record["query" if step.type == "retriever" else "input"] = step.input + if step.tools is not None: + record["tools"] = step.tools + if step.ground_truth is not None: + record["ground_truth"] = step.ground_truth + if isinstance(step.context, dict): + metadata = step.context.get("metadata") + if isinstance(metadata, dict): + record["metadata"] = metadata + if step.children is not None: + child_key = "traces" if step.type == "session" else "spans" + record[child_key] = [ + record_from_agent_control_step(child) for child in step.children + ] + return record + + +def canonical_record_preview( + scenario: BankingScenario, record: dict[str, Any] +) -> dict[str, Any]: """Build the canonical Galileo record that the local evaluator sends to Orbit.""" from agent_control_evaluator_galileo import build_galileo_record + step = build_agent_control_step(record) canonical = build_galileo_record(step.model_dump(mode="json"), step) return canonical.model_dump(mode="json", exclude_none=True) def _json_text(value: Any) -> str: - return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, sort_keys=True) + return ( + value + if isinstance(value, str) + else json.dumps(value, ensure_ascii=False, sort_keys=True) + ) def scorer_invoke_request_preview( @@ -342,7 +381,9 @@ def scorer_invoke_request_preview( } -def _scorer_response_from_evaluation(item: dict[str, Any] | None) -> dict[str, Any] | None: +def _scorer_response_from_evaluation( + item: dict[str, Any] | None, +) -> dict[str, Any] | None: if item is None: return None metadata = item.get("metadata") if isinstance(item.get("metadata"), dict) else {} @@ -376,7 +417,9 @@ def _scorer_response_from_evaluation(item: dict[str, Any] | None) -> dict[str, A "message": item.get("message"), "error": error, "transport_status": transport_status, - "scorer_response": {key: metadata.get(key) for key in response_keys if key in metadata}, + "scorer_response": { + key: metadata.get(key) for key in response_keys if key in metadata + }, "note": "These are the scorer response fields propagated by the Galileo evaluator.", } @@ -396,7 +439,11 @@ def _dependency_sources() -> dict[str, str]: def _json_text_for_log(value: Any) -> str: - return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, sort_keys=True) + return ( + value + if isinstance(value, str) + else json.dumps(value, ensure_ascii=False, sort_keys=True) + ) def _log_metadata(value: Any) -> dict[str, str | bool | int | float | None] | None: @@ -475,7 +522,7 @@ def _log_stream_url(*, project_id: str, log_stream_id: str) -> str | None: async def _evaluate_parent_scenario( scenario: BankingScenario, - record: dict[str, Any], + step: Any, env: dict[str, str], *, trace_id: str, @@ -487,7 +534,6 @@ async def _evaluate_parent_scenario( if scenario.level not in {"trace", "session"}: raise ValueError("Only trace and session parents use evaluate_controls()") - step = build_agent_control_step(record) result = await agent_control.evaluate_controls( scenario.step_name, input=step.input, @@ -520,8 +566,12 @@ async def _evaluate_parent_scenario( None, ) action = expected.get("action") if expected else None - outcome = "denied" if action == "deny" else "steered" if action == "steer" else "allowed" - steering = (expected.get("metadata") or {}).get("steering_context") if expected else None + outcome = ( + "denied" if action == "deny" else "steered" if action == "steer" else "allowed" + ) + steering = ( + (expected.get("metadata") or {}).get("steering_context") if expected else None + ) raw_response = ( result if isinstance(result, dict) @@ -703,9 +753,7 @@ async def submit_transfer( async def retrieve_policy( query: str, partitions: list[str], limit: int ) -> list[dict[str, Any]]: - output = search_policy_documents( - query, partitions=partitions, limit=limit - ) + output = search_policy_documents(query, partitions=partitions, limit=limit) captured["output"] = output return output @@ -780,7 +828,9 @@ async def _run_async( registered_sink_count = len(agent_control.get_registered_control_event_sinks()) if registered_sink_count == 0: logger.terminate() - raise RuntimeError("Galileo did not register an Agent Control control-event sink") + raise RuntimeError( + "Galileo did not register an Agent Control control-event sink" + ) session_id = logger.start_session( name="banking-multilevel-streamlit", external_id=f"banking-multilevel-{uuid4()}", @@ -823,11 +873,17 @@ async def _run_async( }, ) + step = None if scenario.level in {"trace", "session"}: - record = scenario_record(scenario.key, inputs) + step = ( + recorder_trace_step(inputs) + if scenario.level == "trace" + else recorder_session_step(inputs) + ) + record = record_from_agent_control_step(step) evaluation = await _evaluate_parent_scenario( scenario, - record, + step, env, trace_id=str(trace.id), span_id=str(workflow.id), @@ -843,7 +899,8 @@ async def _run_async( logger.flush() completed = True - step = build_agent_control_step(record) + if step is None: + step = build_agent_control_step(record) control_span_count = sum( getattr(span, "type", None) == "control" for span in getattr(workflow, "spans", []) @@ -896,10 +953,10 @@ def run_scenario( ) -> dict[str, Any]: inputs = inputs or default_scenario_input(scenario) try: - return asyncio.run(_run_async(scenario, inputs)) + result = asyncio.run(_run_async(scenario, inputs)) except Exception as exc: # noqa: BLE001 - render actionable E2E failures in Streamlit. record = scenario_record(scenario.key, inputs) - return { + result = { "scenario": scenario.key, "selected_scenario": scenario.label, "request": inputs.request, @@ -914,93 +971,139 @@ def run_scenario( "outcome": "evaluation failed", "error": f"{type(exc).__name__}: {exc}", } + _log_result_to_stderr(result) + return result + + +def _log_result_to_stderr(result: dict[str, Any]) -> None: + """Keep complete diagnostic evidence in the terminal, not the user-facing UI.""" + scenario = result.get("scenario", "unknown") + print( + f"\n=== banking-multilevel result: {scenario} ===", + file=sys.stderr, + flush=True, + ) + print( + json.dumps(result, ensure_ascii=False, indent=2, default=str), + file=sys.stderr, + flush=True, + ) + print( + f"=== end banking-multilevel result: {scenario} ===\n", + file=sys.stderr, + flush=True, + ) + + +def _evaluator_metadata(result: dict[str, Any]) -> dict[str, Any]: + evaluator = result.get("evaluator_result") + if not isinstance(evaluator, dict): + return {} + metadata = evaluator.get("metadata") + return metadata if isinstance(metadata, dict) else {} + + +def _scenario_explanation(result: dict[str, Any]) -> str: + """Explain the outcome in banking language instead of exposing evaluator JSON.""" + if result.get("decision") == "error": + return ( + "The workflow ran, but Agent Control could not complete the evaluation. " + f"{result.get('error') or result.get('reason') or 'Check the terminal log for details.'}" + ) + + denied = result.get("outcome") == "denied" + scenario = result.get("scenario") + explanations = { + "trace": ( + "The complete transfer workflow was blocked because its final response " + "contained sensitive personal information." + if denied + else "The complete transfer workflow was allowed because no configured sensitive-data rule matched." + ), + "session": ( + "The banking session was blocked because the conversation repeated an " + "approval-bypass attempt and completed a transfer without the required approval." + if denied + else "The banking session was allowed because it did not complete a prohibited approval-bypass sequence." + ), + "llm": ( + "The model response was blocked because it selected `close_account` even " + "though the user asked to plan a transfer with `plan_transfer`." + if denied + else "The model response was allowed because its selected tool satisfied the request." + ), + "tool": ( + "The tool result was blocked because the transfer failed with " + "`APPROVAL_REQUIRED`; no transfer was created." + if denied + else "The tool result was allowed because the transfer completed without a configured tool error." + ), + "retriever": ( + "The retrieval result was blocked because the returned policy documents " + "were not relevant to the wire-transfer question." + if denied + else "The retrieval result was allowed because the returned policy documents were relevant to the question." + ), + } + return explanations.get( + scenario, + "Agent Control blocked this execution." + if denied + else "Agent Control allowed this execution.", + ) + + +def _render_scorer_facts(result: dict[str, Any]) -> None: + metadata = _evaluator_metadata(result) + facts: list[str] = [] + label = metadata.get("scorer_label") + if label: + facts.append(f"Scorer: **{label}**") + if "score" in metadata: + facts.append(f"Score: **{metadata['score']}**") + operator = metadata.get("operator") + if operator is not None and "threshold" in metadata: + facts.append(f"Rule: **score {operator} {metadata['threshold']}**") + elapsed = metadata.get("execution_time_seconds") + if isinstance(elapsed, int | float): + facts.append(f"Evaluation time: **{elapsed:.3f}s**") + if facts: + st.markdown(" · ".join(facts)) def _render_result(result: dict[str, Any]) -> None: st.subheader(result["selected_scenario"]) - summary_columns = st.columns(4) - summary_columns[0].metric("Record level", result["record_level"]) - summary_columns[1].metric("Control decision", result["decision"]) - summary_columns[2].metric("Outcome", result["outcome"]) - summary_columns[3].metric("Step count", len(result["step_tree"])) + outcome = result.get("outcome") + explanation = _scenario_explanation(result) + if result.get("decision") == "error": + st.error(explanation) + elif outcome == "denied": + st.error(f"Denied — {explanation}") + elif outcome == "steered": + st.warning(f"Steered — {explanation}") + else: + st.success(f"Allowed — {explanation}") + + st.markdown(f"**Request:** {result['request']}") + _render_scorer_facts(result) + if result.get("trace_id"): - st.success("The application trace and spans were flushed to Galileo.") - telemetry_columns = st.columns(4) - telemetry_columns[0].metric("Project", str(result["project_id"])[:8]) - telemetry_columns[1].metric("Log stream", str(result["log_stream_id"])[:8]) - telemetry_columns[2].metric("Trace", str(result["trace_id"])[:8]) - telemetry_columns[3].metric("Control spans", result.get("control_span_count", 0)) - st.caption( - "Agent Control registered sinks: " - f"{result.get('registered_control_sinks', 0)}" - ) - if result.get("control_span_count", 0) == 0: - st.error( - "The application trace succeeded, but no Agent Control event was " - "attached. The trace's success status describes application " - "execution; it does not mean control telemetry was emitted." + control_spans = result.get("control_span_count", 0) + if control_spans: + st.caption( + f"Saved to Galileo trace `{str(result['trace_id'])[:8]}` with " + f"{control_spans} control span{'s' if control_spans != 1 else ''}." ) else: - st.success("Agent Control telemetry is attached to this trace.") - if result.get("log_stream_url"): - st.markdown(f"[Open this Galileo log stream]({result['log_stream_url']})") - st.markdown("**Request**") - st.code(result["request"], language=None) - st.markdown("**Step tree**") - st.dataframe(result["step_tree"], use_container_width=True, hide_index=True) - st.markdown("**Evaluator and control result**") - st.json( - { - key: result.get(key) - for key in ( - "scenario", - "record_level", - "evaluation_path", - "execution_evidence", - "project_id", - "log_stream_id", - "session_id", - "trace_id", - "workflow_span_id", - "control_span_count", - "registered_control_sinks", - "step_name", - "step_name_regex", - "expected_control", - "expected_control_matched", - "decision", - "outcome", - "evaluator_result", - "steering_context", - "matches", - "non_matches", - "errors", - "reason", - "error", + st.warning( + "The application trace was saved, but it does not contain an Agent Control span." ) - if key in result - } + if result.get("log_stream_url"): + st.markdown(f"[Open the trace in Galileo]({result['log_stream_url']})") + + st.caption( + "Complete records and evaluator diagnostics were written to the terminal log." ) - with st.expander("Generated Agent Control record", expanded=True): - st.json(result["record"]) - if "agent_control_step" in result: - with st.expander("Agent Control Step sent for evaluation", expanded=True): - st.json(result["agent_control_step"]) - if "scorer_invoke_request" in result: - with st.expander("Reconstructed POST /api/v1/scorers/invoke request", expanded=True): - st.json(result["scorer_invoke_request"]) - if result.get("scorer_invoke_response") is not None: - with st.expander("Scorer invoke response", expanded=True): - st.json(result["scorer_invoke_response"]) - if result.get("agent_control_response") is not None: - with st.expander("Raw Agent Control evaluation response"): - st.json(result["agent_control_response"]) - if "canonical_record" in result: - with st.expander("Canonical Galileo record sent to the scorer"): - st.json(result["canonical_record"]) - if "dependency_sources" in result: - with st.expander("Loaded Agent Control package paths"): - st.json(result["dependency_sources"]) def _interactive_inputs(scenario: BankingScenario) -> BankingScenarioInput: @@ -1135,15 +1238,21 @@ def _interactive_inputs(scenario: BankingScenario) -> BankingScenarioInput: def main() -> None: - st.set_page_config(page_title="Banking Agent: five evaluation levels", layout="wide") + st.set_page_config( + page_title="Banking Agent: five evaluation levels", layout="wide" + ) st.title("Banking Agent · Agent Control + Orbit runtime") st.caption( "Executable banking workflows with independent trace, session, LLM, tool, and " "retriever scorer scenarios. Functions run against a safe in-memory sandbox." ) - selected_label = st.selectbox("Scenario", [scenario.label for scenario in SCENARIOS]) + selected_label = st.selectbox( + "Scenario", [scenario.label for scenario in SCENARIOS] + ) scenario = next(item for item in SCENARIOS if item.label == selected_label) - st.info(f"{scenario.description} Target: `{scenario.level}` / `{scenario.step_name}`") + st.info( + f"{scenario.description} Target: `{scenario.level}` / `{scenario.step_name}`" + ) with st.form(key=f"inputs::{scenario.key}"): inputs = _interactive_inputs(scenario) submitted = st.form_submit_button( diff --git a/step_recorder_vs_manual_demo.py b/step_recorder_vs_manual_demo.py old mode 100644 new mode 100755 index a52037e..978e654 --- a/step_recorder_vs_manual_demo.py +++ b/step_recorder_vs_manual_demo.py @@ -4,8 +4,8 @@ Agent Control PR #287 (https://github.com/agentcontrol/agent-control/pull/287) added `agent_control.record_step()` / `StepRecorder` as an incremental way to build a trace/session `Step` tree. Before that, the only way to populate -`Step.children` was the pattern already used by `banking_multilevel_cases.py` -and `banking_multilevel_streamlit_app.py` in this repo: +`Step.children` was the hand-built record pattern retained in +`banking_multilevel_cases.py` for comparison: 1. Every leaf function returns a `StepExecution(value=..., record=...)` - the real return value plus a hand-built dict shaped like a `Step`. @@ -16,7 +16,9 @@ 4. `evaluate_controls()` is called with the built `Step`'s fields unpacked back out again (input=, output=, children=, ...). -This script builds the *same* three scenarios both ways and asserts the +The reusable StepRecorder workflows live in `banking_multilevel_cases.py`; +the Streamlit app imports them directly. This separate script builds the +*same* three scenarios both ways and asserts the resulting `Step` trees are byte-for-byte equivalent - including `context`, the one field that has to be populated by hand on both sides - so a reader can see directly, not just read a claim, that the new API produces the same @@ -60,31 +62,53 @@ import inspect import json import os -from copy import deepcopy from typing import Any from agent_control_models import Step from banking_multilevel_cases import ( - SCENARIO_BY_KEY, - TOOL_DEFINITIONS, - BankingSandbox, - BankingScenarioInput, - default_scenario_input, - run_banking_model, + recorder_session_step, + recorder_trace_step, scenario_record, - search_policy_documents, ) -from banking_multilevel_streamlit_app import build_agent_control_step from common import ( DEFAULT_AGENT_CONTROL_URL, resolve_agent_control_api_key, resolve_agent_control_api_key_header, ) -SYSTEM_PROMPT = "Help with banking requests without exposing private data." CACHE_HIT_STEP_NAME = "banking_multilevel_trace_cache_hit" -DEMO_AGENT_NAME = os.environ.get("AGENT_CONTROL_AGENT_NAME", "step-recorder-vs-manual-demo") +DEMO_AGENT_NAME = os.environ.get( + "AGENT_CONTROL_AGENT_NAME", "step-recorder-vs-manual-demo" +) + + +def build_agent_control_step(record: dict[str, Any]) -> Step: + """Translate the legacy dictionary tree without importing the Streamlit app.""" + record_type = str(record["type"]) + input_value = ( + record.get("input", record.get("query")) + if record_type == "retriever" + else record.get("input") + ) + child_records = ( + record.get("traces") if record_type == "session" else record.get("spans") + ) + children = ( + [build_agent_control_step(child) for child in child_records] + if isinstance(child_records, list) + else None + ) + metadata = record.get("metadata") + return Step( + type=record_type, + name=str(record["name"]), + input=input_value, + output=record.get("output"), + context={"metadata": metadata} if isinstance(metadata, dict) else None, + tools=record.get("tools") if record_type == "llm" else None, + children=children, + ) # -------------------------------------------------------------------------- @@ -105,7 +129,10 @@ def old_way_empty_trace(*, explicit_empty_children: bool) -> Step: "type": "trace", "name": CACHE_HIT_STEP_NAME, "input": {"request": "What is my account balance?"}, - "output": {"status": "answered_from_cache", "message": "Your balance is $1,875.25."}, + "output": { + "status": "answered_from_cache", + "message": "Your balance is $1,875.25.", + }, } if explicit_empty_children: # Easy to forget: nothing enforces that an intentionally childless @@ -120,269 +147,8 @@ def old_way_empty_trace(*, explicit_empty_children: bool) -> Step: # -------------------------------------------------------------------------- -def new_way_trace(inputs: BankingScenarioInput | None = None) -> Step: - import agent_control - - scenario = SCENARIO_BY_KEY["trace"] - inputs = inputs or default_scenario_input(scenario) - sandbox = BankingSandbox() - - with agent_control.record_step( - "trace", - scenario.step_name, - input={"request": inputs.request, "customer_id": "cust-demo-001"}, - ) as trace: - with trace.child( - "retriever", - "banking_multilevel_trace_policy_lookup", - input=inputs.policy_query, - context={ - "metadata": { - "executed": True, - "executed_function": "search_policy_documents", - "partitions": ["wire"], - "external_call": False, - } - }, - ) as retrieval: - retrieval.output = search_policy_documents( - inputs.policy_query, partitions=("wire",), limit=1 - ) - - account = trace.call( - sandbox.lookup_account, - account_id=inputs.account_id, - step_type="tool", - step_name="banking_multilevel_trace_account_lookup", - step_context={ - "metadata": { - "executed": True, - "executed_function": "BankingSandbox.lookup_account", - "external_call": False, - } - }, - ) - - with trace.child( - "llm", - "banking_multilevel_trace_plan_llm", - input=[ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": inputs.request}, - ], - tools=deepcopy(TOOL_DEFINITIONS), - context={ - "metadata": { - "executed": True, - "executed_function": "run_banking_model", - "model": "deterministic-local-banking-model", - "behavior": "sensitive_transfer_plan", - "external_call": False, - } - }, - ) as model: - model.output = run_banking_model( - inputs.request, - behavior="sensitive_transfer_plan", - context={ - "policies": retrieval.output, - "account": account, - "recipient": { - "name": inputs.recipient, - "email": inputs.recipient_email, - "ssn": inputs.recipient_ssn, - }, - "amount": inputs.amount, - }, - ) - - trace.output = {"status": "planned", "message": model.output["content"]} - # Set after the children exist, same as the old way: trace_record() - # only knows each child's executed_function once it has built them. - trace.context = { - "metadata": { - "executed": True, - "executed_function": "trace_record", - "child_functions": [ - "search_policy_documents", - "BankingSandbox.lookup_account", - "run_banking_model", - ], - "scenario": scenario.key, - "external_call": False, - } - } - - return trace.build() - - -def _new_way_session_turn( - session: Any, - sandbox: BankingSandbox, - *, - trace_name: str, - request: str, - behavior: str, - llm_name: str, - tool_name: str, - account_id: str, - amount: float, - recipient: str, - manager_approved: bool, - bypass_approval: bool, -) -> dict[str, Any]: - with session.child( - "trace", - trace_name, - input={"request": request}, - context={ - "metadata": { - "executed": True, - "executed_function": "_execute_session_turn", - "external_call": False, - } - }, - ) as trace: - with trace.child( - "llm", - llm_name, - input=[ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": request}, - ], - tools=deepcopy(TOOL_DEFINITIONS), - context={ - "metadata": { - "executed": True, - "executed_function": "run_banking_model", - "model": "deterministic-local-banking-model", - "behavior": behavior, - "external_call": False, - } - }, - ) as model: - model.output = run_banking_model( - request, - behavior=behavior, - context={ - "account_id": account_id, - "amount": amount, - "manager_approved": manager_approved, - }, - ) - - # Unlike the trace scenario's account lookup, this leaf can't move to - # .call() even with step_context=: sandbox_transaction_count is only - # known after submit_transfer() runs (it reads len(sandbox.transactions) - # post-call, same as the old way's execute_transfer()), but .call()'s - # step_context is evaluated eagerly, before the call happens. .child() - # stays necessary here because its context can be set after output. - with trace.child( - "tool", - tool_name, - input={ - "account_id": account_id, - "amount": amount, - "recipient": recipient, - "manager_approved": manager_approved, - "bypass_approval": bypass_approval, - }, - ) as transfer_step: - transfer_step.output = sandbox.submit_transfer( - account_id=account_id, - amount=amount, - recipient=recipient, - manager_approved=manager_approved, - bypass_approval=bypass_approval, - ) - transfer_step.context = { - "metadata": { - "executed": True, - "executed_function": "BankingSandbox.submit_transfer", - "sandbox_transaction_count": len(sandbox.transactions), - "external_call": False, - } - } - transfer = transfer_step.output - trace.output = deepcopy(transfer) - return transfer - - -def new_way_session(inputs: BankingScenarioInput | None = None) -> Step: - import agent_control - - scenario = SCENARIO_BY_KEY["session"] - inputs = inputs or default_scenario_input(scenario) - sandbox = BankingSandbox() - first_request = inputs.request - retry_request = inputs.retry_request - - with agent_control.record_step( - "session", - scenario.step_name, - input={ - "messages": [ - {"role": "user", "content": first_request}, - {"role": "user", "content": retry_request}, - ] - }, - ) as session: - first_output = _new_way_session_turn( - session, - sandbox, - trace_name="banking_multilevel_session_first_attempt", - request=first_request, - behavior="approval_attempt", - llm_name="banking_multilevel_session_first_llm", - tool_name="banking_multilevel_session_first_tool", - account_id=inputs.account_id, - amount=inputs.amount, - recipient=inputs.recipient, - manager_approved=inputs.manager_approved, - bypass_approval=inputs.bypass_approval, - ) - second_output = _new_way_session_turn( - session, - sandbox, - trace_name="banking_multilevel_session_retry_attempt", - request=retry_request, - behavior="approval_bypass_retry" if inputs.retry_bypass_approval else "approval_attempt", - llm_name="banking_multilevel_session_retry_llm", - tool_name="banking_multilevel_session_retry_tool", - account_id=inputs.account_id, - amount=inputs.amount, - recipient=inputs.recipient, - manager_approved=inputs.manager_approved, - bypass_approval=inputs.retry_bypass_approval, - ) - completed = [ - output for output in (first_output, second_output) if output["status"] == "completed" - ] - session.output = { - "status": "completed" if completed else "blocked", - "summary": ( - "The assistant repeated the approval bypass and reported a completed transfer." - if completed and inputs.retry_bypass_approval - else "The assistant completed an approved transfer." - if completed - else "Both transfer attempts remained blocked by the approval requirement." - ), - "attempts": 2, - "completed_transactions": len(completed), - } - # Set after both turns, same as the old way: session_record() only - # knows the final transaction count once both turns have run. - session.context = { - "metadata": { - "executed": True, - "executed_function": "session_record", - "sandbox_transaction_count": len(sandbox.transactions), - "scenario": scenario.key, - "external_call": False, - } - } - - return session.build() +new_way_trace = recorder_trace_step +new_way_session = recorder_session_step def new_way_empty_trace(*, explicit_empty_children: bool) -> Step: @@ -395,7 +161,10 @@ def new_way_empty_trace(*, explicit_empty_children: bool) -> Step: input={"request": "What is my account balance?"}, container_types=container_types, ) as trace: - trace.output = {"status": "answered_from_cache", "message": "Your balance is $1,875.25."} + trace.output = { + "status": "answered_from_cache", + "message": "Your balance is $1,875.25.", + } return trace.build() @@ -424,14 +193,18 @@ def print_equivalence(label: str, old_step: Step, new_step: Step) -> bool: def print_source_contrast(old_fn: Any, new_fn: Any) -> None: old_source = inspect.getsource(old_fn) new_source = inspect.getsource(new_fn) - print(f"\n--- old way: {old_fn.__module__}.{old_fn.__name__} ({len(old_source.splitlines())} lines) ---") + print( + f"\n--- old way: {old_fn.__module__}.{old_fn.__name__} ({len(old_source.splitlines())} lines) ---" + ) print(old_source.rstrip()) print( " (plus the leaf helpers it calls - execute_policy_search / " "execute_account_lookup / execute_banking_llm - and the recursive " "converter build_agent_control_step(), none of which the new way needs)" ) - print(f"\n--- new way: {new_fn.__module__}.{new_fn.__name__} ({len(new_source.splitlines())} lines) ---") + print( + f"\n--- new way: {new_fn.__module__}.{new_fn.__name__} ({len(new_source.splitlines())} lines) ---" + ) print(new_source.rstrip()) @@ -440,7 +213,9 @@ def print_source_contrast(old_fn: Any, new_fn: Any) -> None: # -------------------------------------------------------------------------- -async def evaluate_both(label: str, old_step: Step, new_step: Step, *, stage: str) -> None: +async def evaluate_both( + label: str, old_step: Step, new_step: Step, *, stage: str +) -> None: import agent_control agent_control.init( @@ -464,7 +239,9 @@ async def evaluate_both(label: str, old_step: Step, new_step: Step, *, stage: st agent_name=DEMO_AGENT_NAME, ) # New way: evaluate the recorder's own built Step directly. - new_result = await agent_control.evaluate_step(new_step, agent_name=DEMO_AGENT_NAME, stage=stage) + new_result = await agent_control.evaluate_step( + new_step, agent_name=DEMO_AGENT_NAME, stage=stage + ) for way, result in (("old", old_result), ("new", new_result)): matches = len(getattr(result, "matches", None) or []) @@ -483,7 +260,9 @@ async def evaluate_both(label: str, old_step: Step, new_step: Step, *, stage: st def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--scenario", choices=["trace", "session", "empty", "all"], default="all") + parser.add_argument( + "--scenario", choices=["trace", "session", "empty", "all"], default="all" + ) parser.add_argument("--mode", choices=["old", "new", "both"], default="both") parser.add_argument("--stage", choices=["pre", "post"], default="post") parser.add_argument( @@ -494,7 +273,9 @@ def main() -> None: ) args = parser.parse_args() - scenarios = ["trace", "session", "empty"] if args.scenario == "all" else [args.scenario] + scenarios = ( + ["trace", "session", "empty"] if args.scenario == "all" else [args.scenario] + ) builders: dict[str, tuple[Any, Any]] = { "trace": (old_way_trace, new_way_trace), @@ -515,16 +296,22 @@ def main() -> None: # both ways agree that children is simply omitted. default_old = old_way_empty_trace(explicit_empty_children=False) default_new = new_way_empty_trace(explicit_empty_children=False) - print_equivalence("empty (default, children omitted)", default_old, default_new) + print_equivalence( + "empty (default, children omitted)", default_old, default_new + ) for label, step in (("old", old_step), ("new", new_step)): if step is not None: - print(f"\n[{scenario}:{label}] {json.dumps(comparable(step), indent=2, sort_keys=True)}") + print( + f"\n[{scenario}:{label}] {json.dumps(comparable(step), indent=2, sort_keys=True)}" + ) if old_step is not None and new_step is not None: print_equivalence(scenario, old_step, new_step) if args.evaluate: - asyncio.run(evaluate_both(scenario, old_step, new_step, stage=args.stage)) + asyncio.run( + evaluate_both(scenario, old_step, new_step, stage=args.stage) + ) if args.scenario in ("all", "trace") and args.mode == "both": from banking_multilevel_cases import trace_record diff --git a/tests/test_banking_multilevel_demo.py b/tests/test_banking_multilevel_demo.py index 179a368..7a2f828 100644 --- a/tests/test_banking_multilevel_demo.py +++ b/tests/test_banking_multilevel_demo.py @@ -16,6 +16,8 @@ execute_banking_llm, execute_policy_search, execute_transfer, + recorder_session_step, + recorder_trace_step, scenario_record, step_tree, ) @@ -24,18 +26,55 @@ _emit_record_spans, _evaluate_parent_scenario, _initialize_agent, + _log_result_to_stderr, _log_stream_url, _run_decorated_span_scenario, + _scenario_explanation, _scorer_response_from_evaluation, _tool_output_for_scorer, _verify_manual_controls, build_agent_control_step, + record_from_agent_control_step, scorer_invoke_request_preview, ) ROOT = Path(__file__).parents[1] +@pytest.mark.parametrize( + ("scenario", "expected_phrase"), + [ + ("trace", "sensitive personal information"), + ("session", "approval-bypass attempt"), + ("llm", "selected `close_account`"), + ("tool", "`APPROVAL_REQUIRED`"), + ("retriever", "not relevant"), + ], +) +def test_denied_results_have_plain_language_explanations( + scenario: str, expected_phrase: str +) -> None: + explanation = _scenario_explanation( + {"scenario": scenario, "decision": "deny", "outcome": "denied"} + ) + assert expected_phrase in explanation + + +def test_complete_result_is_logged_to_stderr(capsys) -> None: + _log_result_to_stderr( + { + "scenario": "llm", + "decision": "deny", + "record": {"type": "llm", "name": "example"}, + } + ) + + captured = capsys.readouterr() + assert captured.out == "" + assert '"decision": "deny"' in captured.err + assert '"record"' in captured.err + + def test_parent_evaluation_uses_active_galileo_context(monkeypatch) -> None: captured: dict[str, object] = {} @@ -53,10 +92,11 @@ async def fake_evaluate_controls(step_name: str, **kwargs): monkeypatch.setattr(agent_control, "evaluate_controls", fake_evaluate_controls) scenario = SCENARIO_BY_KEY["trace"] + step = recorder_trace_step() result = asyncio.run( _evaluate_parent_scenario( scenario, - scenario_record("trace"), + step, {"AGENT_CONTROL_AGENT_NAME": "banking-demo"}, trace_id="1d053adf-cee0-4b84-bda8-3b197010e179", span_id="cf7a70bf-b9a3-4392-9ffc-61dd957873ac", @@ -66,9 +106,23 @@ async def fake_evaluate_controls(step_name: str, **kwargs): assert captured["trace_id"] == "1d053adf-cee0-4b84-bda8-3b197010e179" assert captured["span_id"] == "cf7a70bf-b9a3-4392-9ffc-61dd957873ac" assert captured["step_type"] == "trace" + assert captured["children"] == step.children + assert captured["context"] == step.context assert result["evaluation_path"] == "evaluate_controls" +@pytest.mark.parametrize( + ("scenario_key", "builder"), + [("trace", recorder_trace_step), ("session", recorder_session_step)], +) +def test_recorder_built_parent_records_match_existing_e2e_shapes( + scenario_key: str, builder +) -> None: + step = builder() + + assert record_from_agent_control_step(step) == scenario_record(scenario_key) + + def test_scenario_selection_covers_exactly_five_levels() -> None: assert [scenario.key for scenario in SCENARIOS] == [ "trace", @@ -85,7 +139,9 @@ def test_scenario_selection_covers_exactly_five_levels() -> None: "retriever", } assert len({scenario.step_name for scenario in SCENARIOS}) == 5 - assert all(scenario.step_name_regex == f"^{scenario.step_name}$" for scenario in SCENARIOS) + assert all( + scenario.step_name_regex == f"^{scenario.step_name}$" for scenario in SCENARIOS + ) @pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda scenario: scenario.key) @@ -120,7 +176,10 @@ def test_trace_and_session_have_nested_child_steps() -> None: session = scenario_record("session") assert {span["type"] for span in trace["spans"]} == {"llm", "tool", "retriever"} assert len(session["traces"]) == 2 - assert all({span["type"] for span in trace["spans"]} == {"llm", "tool"} for trace in session["traces"]) + assert all( + {span["type"] for span in trace["spans"]} == {"llm", "tool"} + for trace in session["traces"] + ) assert len(step_tree(session)) == 7 @@ -239,8 +298,14 @@ def test_trace_children_are_captured_from_executed_functions() -> None: trace = scenario_record("trace") children = {span["type"]: span for span in trace["spans"]} - assert children["retriever"]["metadata"]["executed_function"] == "search_policy_documents" - assert children["tool"]["metadata"]["executed_function"] == "BankingSandbox.lookup_account" + assert ( + children["retriever"]["metadata"]["executed_function"] + == "search_policy_documents" + ) + assert ( + children["tool"]["metadata"]["executed_function"] + == "BankingSandbox.lookup_account" + ) assert children["llm"]["metadata"]["executed_function"] == "run_banking_model" assert all(child["metadata"]["executed"] is True for child in children.values()) assert trace["output"]["message"] == children["llm"]["output"]["content"] @@ -561,6 +626,13 @@ def test_banking_project_uses_local_agent_control_sources() -> None: assert all("/code/agent-control/" in path for path in sources.values()) +def test_streamlit_uses_reusable_recorder_workflows_not_comparison_demo() -> None: + app_source = (ROOT / "banking_multilevel_streamlit_app.py").read_text() + assert "recorder_trace_step" in app_source + assert "recorder_session_step" in app_source + assert "step_recorder_vs_manual_demo" not in app_source + + def test_local_dependency_override_and_no_sources_escape_hatch_are_documented() -> None: pyproject = (ROOT / "banking-multilevel" / "pyproject.toml").read_text() docs = (ROOT / "docs" / "11_banking_multilevel_runtime_e2e.md").read_text() @@ -585,7 +657,9 @@ def test_demo_contains_no_real_external_banking_calls() -> None: for node in ast.walk(cases_tree) if isinstance(node, ast.ImportFrom) } - assert not ({"httpx", "requests", "urllib", "socket"} & (imported_roots | imported_from)) - assert "external_call\": False" in cases_source + assert not ( + {"httpx", "requests", "urllib", "socket"} & (imported_roots | imported_from) + ) + assert 'external_call": False' in cases_source app_source = (ROOT / "banking_multilevel_streamlit_app.py").read_text() assert ".post(" not in app_source From fd2af84b48a6272ff066b87bae763bba5fb2ee21 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 12 Oct 2026 15:05:49 -0700 Subject: [PATCH 6/8] Document local multilevel SDK overrides --- docs/11_banking_multilevel_runtime_e2e.md | 25 +++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/11_banking_multilevel_runtime_e2e.md b/docs/11_banking_multilevel_runtime_e2e.md index aabd4cc..305d376 100644 --- a/docs/11_banking_multilevel_runtime_e2e.md +++ b/docs/11_banking_multilevel_runtime_e2e.md @@ -138,12 +138,23 @@ export AGENT_CONTROL_API_KEY_HEADER="Galileo-API-Key" `AGENT_CONTROL_TARGET_ID` must be the actual log-stream ID, not its display name. Credentials remain in environment variables only. -## Local Agent Control override +## Local Agent Control and Galileo SDK overrides The dedicated `banking-multilevel/pyproject.toml` resolves every relevant Agent Control package from the sibling `/Users/namratag/code/agent-control` -checkout. The client still talks to `AGENT_CONTROL_URL` in the DevStack; this -does not start or modify a local server, and Orbit stays in the DevStack. +checkout. It also resolves `galileo` from the sibling +`/Users/namratag/code/galileo-python` checkout. These overrides are required +while testing the unreleased StepRecorder and control-span telemetry changes: + +- Agent Control supplies the trace/session StepRecorder behavior used by this + app and sends evaluations to the remote Agent Control service. +- Galileo Python accepts `retriever_call`, `trace_call`, and `session_call` + control spans. The latest PyPI release (`galileo==2.6.0`) supports only + `llm_call` and `tool_call`, so it cannot replace the local checkout yet. + +Both dependencies are local client libraries only. The client still talks to +`AGENT_CONTROL_URL` in the DevStack; this does not start or modify a local +Agent Control server, and Orbit stays in the DevStack. ```bash cd /Users/namratag/code/ace-demo @@ -152,9 +163,11 @@ UV_NO_CONFIG=1 uv run --project banking-multilevel --default-index https://pypi. streamlit run banking_multilevel_streamlit_app.py ``` -To disable local source overrides for normal usage, add `--no-sources` to both -commands. That makes uv resolve the declared Agent Control dependencies from -PyPI instead of the sibling checkout: +To disable all local source overrides for normal usage, add `--no-sources` to +both commands. That makes uv resolve both Agent Control and Galileo Python from +PyPI instead of the sibling checkouts. Do not use this mode for the trace, +session, or retriever control-span tests until compatible releases are +published: ```bash UV_NO_CONFIG=1 uv sync --project banking-multilevel --no-sources --default-index https://pypi.org/simple From 63fb71ab7bf05891d6e7e4e759ce7cd134fac7ed Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 13 Oct 2026 12:54:08 -0700 Subject: [PATCH 7/8] Use Agent Control 8.13 from PyPI --- banking-multilevel/pyproject.toml | 18 +-- banking-multilevel/uv.lock | 166 +++------------------- banking_multilevel_streamlit_app.py | 2 + docs/11_banking_multilevel_runtime_e2e.md | 30 ++-- tests/test_banking_multilevel_demo.py | 17 ++- 5 files changed, 54 insertions(+), 179 deletions(-) diff --git a/banking-multilevel/pyproject.toml b/banking-multilevel/pyproject.toml index deb7848..cba9b5e 100644 --- a/banking-multilevel/pyproject.toml +++ b/banking-multilevel/pyproject.toml @@ -4,12 +4,7 @@ version = "0.1.0" description = "Five-level Streamlit banking demo for Agent Control and Orbit runtime scorers" requires-python = ">=3.12,<3.15" dependencies = [ - "agent-control-sdk[galileo,otel]>=8.7.0", - "agent-control-models>=8.7.0", - "agent-control-engine>=8.7.0", - "agent-control-telemetry>=8.7.0", - "agent-control-evaluators>=8.7.0", - "agent-control-evaluator-galileo>=8.7.0", + "agent-control-sdk[galileo,otel]==8.13.0", "galileo[otel]>=2.3.0", "httpx>=0.26.0", "python-dotenv>=1.0.0", @@ -22,16 +17,9 @@ dev = ["pytest>=8.0.0", "ruff>=0.11.0"] [tool.uv] package = false -# The default development sync uses the local Agent Control checkout while -# still talking to the remote DevStack at runtime. Pass `uv sync --no-sources` -# (and `uv run --no-sources`) to disable every local override and use PyPI. +# Agent Control 8.13.0 comes from PyPI. Galileo remains editable until a PyPI +# release includes retriever_call, trace_call, and session_call control spans. [tool.uv.sources] -agent-control-sdk = { path = "../../agent-control/sdks/python", editable = true } -agent-control-models = { path = "../../agent-control/models", editable = true } -agent-control-engine = { path = "../../agent-control/engine", editable = true } -agent-control-telemetry = { path = "../../agent-control/telemetry", editable = true } -agent-control-evaluators = { path = "../../agent-control/evaluators/builtin", editable = true } -agent-control-evaluator-galileo = { path = "../../agent-control/evaluators/contrib/galileo", editable = true } galileo = { path = "../../galileo-python", editable = true } [tool.ruff] diff --git a/banking-multilevel/uv.lock b/banking-multilevel/uv.lock index cfc2e11..037340b 100644 --- a/banking-multilevel/uv.lock +++ b/banking-multilevel/uv.lock @@ -18,12 +18,7 @@ name = "ace-banking-multilevel-example" version = "0.1.0" source = { virtual = "." } dependencies = [ - { name = "agent-control-engine" }, - { name = "agent-control-evaluator-galileo" }, - { name = "agent-control-evaluators" }, - { name = "agent-control-models" }, { name = "agent-control-sdk", extra = ["galileo", "otel"] }, - { name = "agent-control-telemetry" }, { name = "galileo", extra = ["otel"] }, { name = "httpx" }, { name = "python-dotenv" }, @@ -38,12 +33,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "agent-control-engine", editable = "../../agent-control/engine" }, - { name = "agent-control-evaluator-galileo", editable = "../../agent-control/evaluators/contrib/galileo" }, - { name = "agent-control-evaluators", editable = "../../agent-control/evaluators/builtin" }, - { name = "agent-control-models", editable = "../../agent-control/models" }, - { name = "agent-control-sdk", extras = ["galileo", "otel"], editable = "../../agent-control/sdks/python" }, - { name = "agent-control-telemetry", editable = "../../agent-control/telemetry" }, + { name = "agent-control-sdk", extras = ["galileo", "otel"], specifier = "==8.13.0" }, { name = "galileo", extras = ["otel"], editable = "../../galileo-python" }, { name = "httpx", specifier = ">=0.26.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, @@ -56,36 +46,10 @@ dev = [ { name = "ruff", specifier = ">=0.11.0" }, ] -[[package]] -name = "agent-control-engine" -version = "8.12.0" -source = { editable = "../../agent-control/engine" } -dependencies = [ - { name = "agent-control-evaluators" }, - { name = "agent-control-models" }, - { name = "google-re2" }, -] - -[package.metadata] -requires-dist = [ - { name = "agent-control-evaluators", editable = "../../agent-control/evaluators/builtin" }, - { name = "agent-control-models", editable = "../../agent-control/models" }, - { name = "google-re2", specifier = ">=1.1" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "mypy", specifier = ">=1.8.0" }, - { name = "pytest", specifier = ">=8.0.0" }, - { name = "pytest-asyncio", specifier = ">=0.23.0" }, - { name = "pytest-cov", specifier = ">=4.0.0" }, - { name = "ruff", specifier = ">=0.1.0" }, -] - [[package]] name = "agent-control-evaluator-galileo" -version = "8.12.0" -source = { editable = "../../agent-control/evaluators/contrib/galileo" } +version = "8.13.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "agent-control-evaluators" }, { name = "agent-control-models" }, @@ -93,29 +57,15 @@ dependencies = [ { name = "httpx" }, { name = "pydantic" }, ] - -[package.metadata] -requires-dist = [ - { name = "agent-control-engine", marker = "extra == 'dev'", editable = "../../agent-control/engine" }, - { name = "agent-control-evaluators", editable = "../../agent-control/evaluators/builtin" }, - { name = "agent-control-models", editable = "../../agent-control/models" }, - { name = "agent-control-sdk", marker = "extra == 'dev'", editable = "../../agent-control/sdks/python" }, - { name = "agent-control-telemetry", marker = "extra == 'dev'", editable = "../../agent-control/telemetry" }, - { name = "galileo-core", specifier = ">=4.4.0,<5.0.0" }, - { name = "httpx", specifier = ">=0.24.0" }, - { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8.0" }, - { name = "pydantic", specifier = ">=2.12.4" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, -] -provides-extras = ["dev"] +sdist = { url = "https://files.pythonhosted.org/packages/7d/8f/216cf045483085c8ea32269a4a0a26abfe6c60b244b110bc94955976f12b/agent_control_evaluator_galileo-8.13.0.tar.gz", hash = "sha256:43d0826f95c229eee376482f2224712f9ae8fe3916e8321ca8129cfbe4f40c05", size = 43351, upload-time = "2026-10-13T16:30:50.986Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/1b/a5c99ebb1bb7d366de8624870a75a6703cec50fd48181fed2d15b8fa94fd/agent_control_evaluator_galileo-8.13.0-py3-none-any.whl", hash = "sha256:0e9f09b75c8dbc2bea5160123c081dc41f235e393f73410caefd8c01e9b8fad0", size = 23182, upload-time = "2026-10-13T16:30:46.972Z" }, +] [[package]] name = "agent-control-evaluators" -version = "8.12.0" -source = { editable = "../../agent-control/evaluators/builtin" } +version = "8.13.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "agent-control-models" }, { name = "google-re2" }, @@ -123,52 +73,29 @@ dependencies = [ { name = "pydantic" }, { name = "sqlglot", extra = ["c"] }, ] - -[package.metadata] -requires-dist = [ - { name = "agent-control-evaluator-budget", marker = "extra == 'budget'", editable = "../../agent-control/evaluators/contrib/budget" }, - { name = "agent-control-evaluator-cisco", marker = "extra == 'cisco'", editable = "../../agent-control/evaluators/contrib/cisco" }, - { name = "agent-control-evaluator-defenseclaw", marker = "extra == 'defenseclaw'", editable = "../../agent-control/evaluators/contrib/defenseclaw" }, - { name = "agent-control-evaluator-galileo", marker = "extra == 'galileo'", editable = "../../agent-control/evaluators/contrib/galileo" }, - { name = "agent-control-models", editable = "../../agent-control/models" }, - { name = "google-re2", specifier = ">=1.1" }, - { name = "jsonschema", specifier = ">=4.0.0" }, - { name = "pydantic", specifier = ">=2.12.4" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, - { name = "sqlglot", extras = ["c"], specifier = ">=30.11.0,<30.12.0" }, +sdist = { url = "https://files.pythonhosted.org/packages/b7/44/a17ab9e5fa08f790f74e8fc9d7415be2dce83e069fa4ac044e7be74b893d/agent_control_evaluators-8.13.0.tar.gz", hash = "sha256:ec332f3e9f36b45a4836777fbd36d6e2f2c992502d0b9b0983ab5712402acde9", size = 53223, upload-time = "2026-10-13T16:30:51.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/61/d28e78256df9253bafd3159a2609fdcf57d5c7d4f999f333bad72323d2eb/agent_control_evaluators-8.13.0-py3-none-any.whl", hash = "sha256:b4b0bc0ef9df39a38f313de14da2d3f94dbf17a6670f181efc26e27976aa712a", size = 35798, upload-time = "2026-10-13T16:30:47.898Z" }, ] -provides-extras = ["galileo", "budget", "cisco", "defenseclaw", "dev"] [[package]] name = "agent-control-models" -version = "8.12.0" -source = { editable = "../../agent-control/models" } +version = "8.13.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-re2" }, { name = "jsonschema" }, { name = "pydantic" }, ] - -[package.metadata] -requires-dist = [ - { name = "google-re2", specifier = ">=1.1" }, - { name = "jsonschema", specifier = ">=4.0.0" }, - { name = "pydantic", specifier = ">=2.12.4" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "mypy", specifier = ">=1.8.0" }, - { name = "pytest", specifier = ">=8.0.0" }, - { name = "pytest-cov", specifier = ">=4.0.0" }, - { name = "ruff", specifier = ">=0.1.0" }, +sdist = { url = "https://files.pythonhosted.org/packages/7c/ec/d784612dd460bb9b7fa833c87aa5a5ec0e8f6c84df755cc4b70e02840df4/agent_control_models-8.13.0.tar.gz", hash = "sha256:9fbfe9b455c8b2dd2cf2c2e1a8876a8d4798fcb1732eee28bd7193f128eeabc2", size = 34236, upload-time = "2026-10-13T16:29:00.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/5f/3df2016d4a80949d3c9dd774ab29f09b81a3f66b9e4c02352fee7a78147b/agent_control_models-8.13.0-py3-none-any.whl", hash = "sha256:bfe6ccdb1a8114d0df7eda80789dc22a182f2efaa176f174acb9bf65fbc0b636", size = 33420, upload-time = "2026-10-13T16:28:59.302Z" }, ] [[package]] name = "agent-control-sdk" -version = "8.12.0" -source = { editable = "../../agent-control/sdks/python" } +version = "8.13.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "agent-control-evaluators" }, { name = "docstring-parser" }, @@ -178,6 +105,10 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/12/82/0cb8dc1ece08c1cf22d8e8982c149cc10ca49ddb45dcccf432f81ce74a86/agent_control_sdk-8.13.0.tar.gz", hash = "sha256:2f03003305647a2bbad1b67a7411a812f61aaa73547b46fff1bc91938f2aab7c", size = 222913, upload-time = "2026-10-13T16:31:26.114Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/2b/63662ff15d1ebcd88c1d030bfb39e910dafd6df1c5d25dcc93820636ea48/agent_control_sdk-8.13.0-py3-none-any.whl", hash = "sha256:ca04d6005732e7514522ffbcaeb6ef4607e0ed490c7436d14bdd7d9ac5916dd0", size = 143263, upload-time = "2026-10-13T16:31:24.768Z" }, +] [package.optional-dependencies] galileo = [ @@ -189,57 +120,6 @@ otel = [ { name = "opentelemetry-sdk" }, ] -[package.metadata] -requires-dist = [ - { name = "agent-control-evaluator-galileo", marker = "extra == 'galileo'", editable = "../../agent-control/evaluators/contrib/galileo" }, - { name = "agent-control-evaluators", editable = "../../agent-control/evaluators/builtin" }, - { name = "docstring-parser", specifier = ">=0.15" }, - { name = "google-adk", marker = "extra == 'google-adk'", specifier = ">=1.0.0" }, - { name = "google-re2", specifier = ">=1.1" }, - { name = "httpx", specifier = ">=0.26.0" }, - { name = "jsonschema", specifier = ">=4.0.0" }, - { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.24.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.24.0" }, - { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.24.0" }, - { name = "pydantic", specifier = ">=2.5.0" }, - { name = "pydantic-settings", specifier = ">=2.0.0" }, - { name = "strands-agents", marker = "extra == 'strands-agents'", specifier = ">=1.26.0" }, -] -provides-extras = ["strands-agents", "google-adk", "otel", "galileo"] - -[package.metadata.requires-dev] -dev = [ - { name = "agent-control-engine", editable = "../../agent-control/engine" }, - { name = "agent-control-evaluators", editable = "../../agent-control/evaluators/builtin" }, - { name = "agent-control-models", editable = "../../agent-control/models" }, - { name = "agent-control-telemetry", editable = "../../agent-control/telemetry" }, - { name = "mypy", specifier = ">=1.8.0" }, - { name = "pytest", specifier = ">=8.0.0" }, - { name = "pytest-asyncio", specifier = ">=0.23.0" }, - { name = "pytest-cov", specifier = ">=4.0.0" }, - { name = "ruff", specifier = ">=0.1.0" }, - { name = "strands-agents", specifier = ">=1.26.0" }, -] - -[[package]] -name = "agent-control-telemetry" -version = "8.12.0" -source = { editable = "../../agent-control/telemetry" } -dependencies = [ - { name = "agent-control-models" }, -] - -[package.metadata] -requires-dist = [{ name = "agent-control-models", editable = "../../agent-control/models" }] - -[package.metadata.requires-dev] -dev = [ - { name = "agent-control-models", editable = "../../agent-control/models" }, - { name = "mypy", specifier = ">=1.8.0" }, - { name = "pytest", specifier = ">=8.0.0" }, - { name = "ruff", specifier = ">=0.1.0" }, -] - [[package]] name = "altair" version = "6.3.0" diff --git a/banking_multilevel_streamlit_app.py b/banking_multilevel_streamlit_app.py index 1ac53f9..8725648 100755 --- a/banking_multilevel_streamlit_app.py +++ b/banking_multilevel_streamlit_app.py @@ -428,6 +428,7 @@ def _dependency_sources() -> dict[str, str]: import agent_control import agent_control_evaluator_galileo import agent_control_models + import galileo return { "agent_control": str(Path(agent_control.__file__).resolve()), @@ -435,6 +436,7 @@ def _dependency_sources() -> dict[str, str]: Path(agent_control_evaluator_galileo.__file__).resolve() ), "agent_control_models": str(Path(agent_control_models.__file__).resolve()), + "galileo": str(Path(galileo.__file__).resolve()), } diff --git a/docs/11_banking_multilevel_runtime_e2e.md b/docs/11_banking_multilevel_runtime_e2e.md index 305d376..10e16c4 100644 --- a/docs/11_banking_multilevel_runtime_e2e.md +++ b/docs/11_banking_multilevel_runtime_e2e.md @@ -138,23 +138,23 @@ export AGENT_CONTROL_API_KEY_HEADER="Galileo-API-Key" `AGENT_CONTROL_TARGET_ID` must be the actual log-stream ID, not its display name. Credentials remain in environment variables only. -## Local Agent Control and Galileo SDK overrides +## Published Agent Control and local Galileo SDK -The dedicated `banking-multilevel/pyproject.toml` resolves every relevant -Agent Control package from the sibling `/Users/namratag/code/agent-control` -checkout. It also resolves `galileo` from the sibling -`/Users/namratag/code/galileo-python` checkout. These overrides are required -while testing the unreleased StepRecorder and control-span telemetry changes: +The dedicated `banking-multilevel/pyproject.toml` installs +`agent-control-sdk[galileo,otel]==8.13.0` and its published dependencies from +PyPI. That release supplies the trace/session StepRecorder behavior used by +this app. No local Agent Control checkout is required. + +The project still resolves `galileo` from the sibling +`/Users/namratag/code/galileo-python` checkout because its control-span changes +have not been published yet: -- Agent Control supplies the trace/session StepRecorder behavior used by this - app and sends evaluations to the remote Agent Control service. - Galileo Python accepts `retriever_call`, `trace_call`, and `session_call` control spans. The latest PyPI release (`galileo==2.6.0`) supports only `llm_call` and `tool_call`, so it cannot replace the local checkout yet. -Both dependencies are local client libraries only. The client still talks to -`AGENT_CONTROL_URL` in the DevStack; this does not start or modify a local -Agent Control server, and Orbit stays in the DevStack. +These are client dependencies only. The client still talks to +`AGENT_CONTROL_URL` in the DevStack, and Orbit stays in the DevStack. ```bash cd /Users/namratag/code/ace-demo @@ -163,11 +163,9 @@ UV_NO_CONFIG=1 uv run --project banking-multilevel --default-index https://pypi. streamlit run banking_multilevel_streamlit_app.py ``` -To disable all local source overrides for normal usage, add `--no-sources` to -both commands. That makes uv resolve both Agent Control and Galileo Python from -PyPI instead of the sibling checkouts. Do not use this mode for the trace, -session, or retriever control-span tests until compatible releases are -published: +Adding `--no-sources` disables the remaining local Galileo override. Do not use +this mode for trace, session, or retriever control-span tests until a compatible +Galileo Python release is published: ```bash UV_NO_CONFIG=1 uv sync --project banking-multilevel --no-sources --default-index https://pypi.org/simple diff --git a/tests/test_banking_multilevel_demo.py b/tests/test_banking_multilevel_demo.py index 7a2f828..d828f50 100644 --- a/tests/test_banking_multilevel_demo.py +++ b/tests/test_banking_multilevel_demo.py @@ -621,9 +621,14 @@ def test_scorer_response_view_keeps_allow_score_and_status() -> None: } -def test_banking_project_uses_local_agent_control_sources() -> None: +def test_banking_project_uses_published_agent_control_and_local_galileo() -> None: sources = _dependency_sources() - assert all("/code/agent-control/" in path for path in sources.values()) + assert all( + "/code/agent-control/" not in path + for name, path in sources.items() + if name.startswith("agent_control") + ) + assert "/code/galileo-python/" in sources["galileo"] def test_streamlit_uses_reusable_recorder_workflows_not_comparison_demo() -> None: @@ -633,13 +638,15 @@ def test_streamlit_uses_reusable_recorder_workflows_not_comparison_demo() -> Non assert "step_recorder_vs_manual_demo" not in app_source -def test_local_dependency_override_and_no_sources_escape_hatch_are_documented() -> None: +def test_dependency_sources_and_no_sources_escape_hatch_are_documented() -> None: pyproject = (ROOT / "banking-multilevel" / "pyproject.toml").read_text() docs = (ROOT / "docs" / "11_banking_multilevel_runtime_e2e.md").read_text() assert "[tool.uv.sources]" in pyproject - assert "../../agent-control/sdks/python" in pyproject - assert "../../agent-control/evaluators/contrib/galileo" in pyproject + assert '"agent-control-sdk[galileo,otel]==8.13.0"' in pyproject + assert "../../agent-control/" not in pyproject + assert '../../galileo-python"' in pyproject assert '"galileo[otel]>=2.3.0"' in pyproject + assert "No local Agent Control checkout is required" in docs assert "--no-sources" in docs assert "AGENT_CONTROL_URL" in docs From 1c26f894be0bfce9906d7cf08cc40f03850b01de Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 13 Oct 2026 13:14:23 -0700 Subject: [PATCH 8/8] Document temporary Galileo SDK override --- banking-multilevel/pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/banking-multilevel/pyproject.toml b/banking-multilevel/pyproject.toml index cba9b5e..06a425d 100644 --- a/banking-multilevel/pyproject.toml +++ b/banking-multilevel/pyproject.toml @@ -17,8 +17,10 @@ dev = ["pytest>=8.0.0", "ruff>=0.11.0"] [tool.uv] package = false -# Agent Control 8.13.0 comes from PyPI. Galileo remains editable until a PyPI -# release includes retriever_call, trace_call, and session_call control spans. +# Agent Control 8.13.0 comes from PyPI. The local galileo-python checkout is +# temporarily required for retriever_call, trace_call, and session_call control +# spans. Once those changes are published, pin `galileo[otel]` above to that +# PyPI version, remove this path override, and regenerate uv.lock. [tool.uv.sources] galileo = { path = "../../galileo-python", editable = true }