|
| 1 | +"""What the second look's model check reads in a task run (#455): the task's own request, in plain words. |
| 2 | +
|
| 3 | +It used to read the latest 1500 characters of the executor's system prompt as "the user said", which are Sentient's |
| 4 | +instructions ("If a tool fails, analyze the error..."), so a flagged code step gave reasons like "the user asked to |
| 5 | +analyze an error". A held call still asks exactly as before. |
| 6 | +""" |
| 7 | + |
| 8 | +from __future__ import annotations |
| 9 | + |
| 10 | +from types import SimpleNamespace |
| 11 | + |
| 12 | +from sentient.agent import second_look as sl |
| 13 | +from sentient.tasks import email_inbox |
| 14 | +from sentient.tasks.prompts import build_second_look_request |
| 15 | +from sentient.tools.base import Risk, ToolContext, ToolPlugin, tool |
| 16 | +from tests.conftest import FakeProvider, tool_call |
| 17 | +from tests.tasks.conftest import json_calls |
| 18 | + |
| 19 | +NAME = "Prime squares" |
| 20 | +DESCRIPTION = "Add up the squares of the first 500 prime numbers and tell me the total." |
| 21 | +PROMPT = "work out the sum of the squares of the first 500 primes for me" |
| 22 | +STEP = "Run a short Python script that adds up the squares of the first 500 primes and prints the total" |
| 23 | +RUN = [tool_call("runner_run", code="print(sum(p * p for p in primes(500)))")] |
| 24 | + |
| 25 | + |
| 26 | +def _runner(log: list[str]) -> ToolPlugin: |
| 27 | + @tool("runner_run", risk=Risk.exec) |
| 28 | + async def runner_run(ctx: ToolContext, code: str) -> dict: |
| 29 | + """Run a Python script.""" |
| 30 | + log.append(code) |
| 31 | + return {"output": "1"} |
| 32 | + |
| 33 | + class Runner(ToolPlugin): |
| 34 | + id = "runner" |
| 35 | + display_name = "Runner" |
| 36 | + tools = [runner_run] |
| 37 | + |
| 38 | + return Runner() |
| 39 | + |
| 40 | + |
| 41 | +async def _held_run(make_app, config, **fields) -> tuple[FakeProvider, dict, list[str]]: |
| 42 | + config.tools.approvals.second_look_model = True |
| 43 | + log: list[str] = [] |
| 44 | + reason = "The user asked for the sum of squares of primes, but this runs code" |
| 45 | + llm = FakeProvider(replies=[RUN, "Waiting for the OK."], json_replies=[{"match": False, "reason": reason}]) |
| 46 | + app = await make_app(llm) |
| 47 | + app.registry.register(_runner(log)) |
| 48 | + now = app.tasks.now_iso() |
| 49 | + task_id = await app.tasks.repo.insert_task({ |
| 50 | + "name": NAME, "description": DESCRIPTION, "original_prompt": PROMPT, "status": "approval_pending", |
| 51 | + "schedule": {"type": "once", "run_at": None}, "plan": [{"tool": "runner", "description": STEP}], |
| 52 | + "created_at": now, "updated_at": now, **fields, |
| 53 | + }) |
| 54 | + await app.tasks.approve(task_id) |
| 55 | + await app.tasks.drain() |
| 56 | + return llm, (await app.tasks.get(task_id))["runs"][-1], log |
| 57 | + |
| 58 | + |
| 59 | +async def test_a_flagged_code_step_is_checked_against_the_tasks_request(make_app, config): |
| 60 | + llm, run, log = await _held_run(make_app, config) |
| 61 | + checks = json_calls(llm, "fast") |
| 62 | + assert len(checks) == 1 |
| 63 | + said = checks[0]["messages"][1]["content"].split("The assistant now wants", 1)[0] |
| 64 | + assert f"My task: {NAME} - {DESCRIPTION}" in said |
| 65 | + assert f"In my words: {PROMPT}" in said |
| 66 | + assert f"1. {STEP} (with runner)" in said |
| 67 | + # none of the executor's own instructions read as the user's words |
| 68 | + for engine_words in ("analyze the error", "EXECUTION STRATEGY", "Begin executing the plan", "Step 1"): |
| 69 | + assert engine_words not in said |
| 70 | + # the hold itself is unchanged: nothing ran, the run asks, with the reason first |
| 71 | + assert log == [] and run["status"] == "waiting_for_user" |
| 72 | + question = run["pending_question"]["question"] |
| 73 | + assert question.startswith("A second look found this may not match what you asked: The user asked for the sum") |
| 74 | + assert 'This task wants to use "Run" in Runner' in question |
| 75 | + |
| 76 | + |
| 77 | +async def test_a_task_by_email_still_never_reads_its_brief_as_the_users_words(make_app, config, monkeypatch): |
| 78 | + monkeypatch.setattr(email_inbox, "is_email_task", lambda task: True) |
| 79 | + monkeypatch.setattr(email_inbox, "app_name", lambda task: "Email (IMAP)") |
| 80 | + llm, run, log = await _held_run(make_app, config) |
| 81 | + said = json_calls(llm, "fast")[0]["messages"][1]["content"].split("The assistant now wants", 1)[0] |
| 82 | + assert sl.BRIEF_REQUEST in said and DESCRIPTION not in said and "My task:" not in said |
| 83 | + assert log == [] and run["status"] == "waiting_for_user" |
| 84 | + |
| 85 | + |
| 86 | +def test_the_request_puts_the_objective_first_and_stays_short(): |
| 87 | + steps = [{"tool": "web", "description": "Read the page " + "x" * 400}] * 12 |
| 88 | + confirmed = "**What the user confirmed for this task:**\n- The user expects: a short list\n\n" |
| 89 | + text = build_second_look_request( |
| 90 | + name="Studio events", description="Email me new studio events", original_prompt="Email me new studio events", |
| 91 | + plan=steps, confirmed=confirmed, |
| 92 | + ) |
| 93 | + assert text.startswith("My task: Studio events - Email me new studio events\n- The user expects: a short list\n" |
| 94 | + "The plan I approved:\n1. Read the page") |
| 95 | + assert "In my words" not in text # the same words as the description are not repeated |
| 96 | + assert len(text) <= sl.MAX_REQUEST and "9. " not in text # whole steps only, as many as fit, each cut short |
| 97 | + assert text.splitlines()[-1].endswith("... (with web)") |
| 98 | + # a long objective, long words and a long plan still keep what the user confirmed |
| 99 | + long = build_second_look_request(name="N" * 600, description="", original_prompt="P" * 600, plan=steps, |
| 100 | + confirmed=confirmed) |
| 101 | + assert "\n- The user expects: a short list\n" in long and len(long) <= sl.MAX_REQUEST |
| 102 | + short = build_second_look_request(name="Digest", description="", plan=[], confirmed=confirmed) |
| 103 | + assert short == "My task: Digest\n- The user expects: a short list" |
| 104 | + |
| 105 | + |
| 106 | +def test_the_brief_is_used_only_in_a_task_run_and_never_for_a_brief_from_email(): |
| 107 | + agent = SimpleNamespace(registry=None, config=SimpleNamespace(assistant=SimpleNamespace(user_name="Maya"))) |
| 108 | + messages = [{"role": "system", "content": "**Primary Objective:** 'Email Arjun the notes'"}, |
| 109 | + {"role": "user", "content": "Begin."}] |
| 110 | + |
| 111 | + def request(extra: dict) -> str: |
| 112 | + return sl.Look(agent, None, None, SimpleNamespace(extra=extra), None, messages).request() |
| 113 | + |
| 114 | + assert request({"task_id": "t1", sl.TASK_REQUEST_KEY: "My task: Email Arjun"}) == "My task: Email Arjun" |
| 115 | + assert "Email Arjun the notes" in request({"task_id": "t1"}) # no brief: what gather found, as before |
| 116 | + assert request({sl.TASK_REQUEST_KEY: "My task: x"}) != "My task: x" # not a task run |
| 117 | + assert request({"task_id": "t1", sl.TASK_REQUEST_KEY: "My task: x", sl.BRIEF_FROM_KEY: "Gmail"}).startswith( |
| 118 | + sl.BRIEF_REQUEST) |
0 commit comments