Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
Test configuration for agent tests.
"""

import sys
from pathlib import Path

import pytest

# Add the agents path
Expand Down
14 changes: 13 additions & 1 deletion src/backend/orchestration/orchestration_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,20 @@ async def init_orchestration(

manager_agent = Agent(manager_chat_client, name="MagenticManager")

# Collect participant agent names so the orchestrator plan prompt can
# enforce mandatory inclusion of every team agent (e.g. TriageAgent,
# ComplianceAgent) — otherwise the manager silently drops them.
participant_agent_names = []
for ag in agents:
nm = getattr(ag, "agent_name", None) or getattr(ag, "name", None)
if nm:
participant_agent_names.append(nm)

# Get prompt customization kwargs
prompt_kwargs = get_magentic_prompt_kwargs(has_user_responses=has_user_responses)
prompt_kwargs = get_magentic_prompt_kwargs(
has_user_responses=has_user_responses,
participant_names=participant_agent_names,
)

cls.logger.info(
"Building MagenticBuilder for user '%s' with max_rounds=%d, "
Expand Down
93 changes: 67 additions & 26 deletions src/backend/orchestration/plan_review_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,23 @@
# Prompt kwargs builder
# ---------------------------------------------------------------------------

def get_magentic_prompt_kwargs(*, has_user_responses: bool = False) -> dict:
def get_magentic_prompt_kwargs(
*,
has_user_responses: bool = False,
participant_names: Optional[list[str]] = None,
) -> dict:
Comment thread
Copilot marked this conversation as resolved.
"""Build the prompt-override kwargs dict for ``MagenticBuilder``.

Args:
has_user_responses: Whether any agent has ``user_responses: true``,
giving it access to the ``ask_user`` tool for user clarification.
When True, prompts allow agents to gather info via their tools;
when False, agents must use defaults only.
participant_names: Names of the team's participant agents. When provided,
the orchestrator plan prompt is augmented with a MANDATORY AGENTS
clause requiring every one of these agents to appear as a plan step,
so coordinator-like agents (e.g. TriageAgent) and final-validation
agents (e.g. ComplianceAgent) are not silently dropped.

Returns:
A dict suitable for unpacking into ``MagenticBuilder(**kwargs)``.
Expand Down Expand Up @@ -70,12 +79,26 @@ def get_magentic_prompt_kwargs(*, has_user_responses: bool = False) -> dict:
- Ask EXACTLY 0 questions. Always proceed with sensible defaults.
"""

mandatory_block = ""
if participant_names:
agent_lines = "\n".join(f"- {name}" for name in participant_names)
mandatory_block = (
"\n\nMANDATORY AGENTS (CRITICAL — NON-NEGOTIABLE):\n"
"Every plan you generate MUST include EVERY ONE of the following agents,\n"
"each as its own distinct step, each invoked at least once:\n"
+ agent_lines
+ "\nDo NOT omit any of these agents, even if a step seems optional,\n"
"redundant, already covered by another agent, or something you "
"(MagenticManager)\ncould do yourself. A plan missing any listed agent "
"is INVALID — regenerate it\nuntil every listed agent appears as a step.\n"
)

plan_append = """

PLAN RULES:
- Steps are HIGH-LEVEL task assignments — one step per agent. Do NOT prescribe
sub-tasks, parameters, or data retrieval. Agents discover their own processes.
""" + clarification_policy + """
""" + mandatory_block + clarification_policy + """
OUTPUT FORMAT (CRITICAL — use EXACTLY this JSON structure, nothing else):
```json
Comment thread
Akhileswara-Microsoft marked this conversation as resolved.
[
Expand All @@ -86,11 +109,14 @@ def get_magentic_prompt_kwargs(*, has_user_responses: bool = False) -> dict:
Use exact agent names from the team list above. Output ONLY the JSON array — no
markdown fences, no commentary before or after.

IMPORTANT: There is NO UserInteractionAgent. Do NOT include any user-interaction
""" + ("""IMPORTANT: There is NO UserInteractionAgent. Do NOT include any user-interaction
agent in the plan. Domain agents gather user info themselves via their
request_user_clarification tool — the framework pauses automatically when they
call it and resumes when the user answers.

""" if has_user_responses else """IMPORTANT: There is NO UserInteractionAgent. Do NOT include any user-interaction
agent in the plan. Agents apply sensible defaults for missing details and proceed
without asking the user any questions.
""") + """
Example plan:
[
{{"agent": "HRHelperAgent", "action": "execute the onboarding process for the new employee"}},
Expand Down Expand Up @@ -143,12 +169,42 @@ def get_magentic_prompt_kwargs(*, has_user_responses: bool = False) -> dict:
ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT + facts_append
)

progress_append = """
# Completion-enforcement progress-ledger rules. Applied ALWAYS (not only when
# has_user_responses) so that EVERY plan-step agent — e.g. TriageAgent and
# ComplianceAgent in the content_gen team, whose agents all have
# user_responses=false — must actually run before the request can be marked
# satisfied, and the orchestrator re-selects any uninvoked plan-step agent
# instead of silently finishing early.
progress_append = """

EXECUTION RULES:
- When selecting next_speaker, prefer a work agent that has NOT yet been invoked.
- MagenticManager MUST NOT generate answers, ask questions, or list missing info.
It only routes tasks to the appropriate agent.
- MagenticManager MUST NOT generate answers or fabricate content on behalf of
work agents. It only routes tasks and compiles the final output.

COMPLETION CHECK (CRITICAL):
Before setting is_request_satisfied to true, you MUST verify:
1. Review the conversation history and list every agent that has actually produced
a substantive response (meaningful output — calling tools and returning results
where the agent has tools, or producing a substantive text response otherwise).
2. Compare that list against the plan steps. If ANY plan-step agent has NOT been
invoked and produced a substantive response, set is_request_satisfied to false
and select the next uninvoked agent as next_speaker.
3. is_request_satisfied = true ONLY when ALL plan-step agents have completed
their work (produced a substantive response — tool results, or meaningful text
output for agents that have no tools).
- Each agent handles a DISTINCT domain. One agent's output does NOT satisfy
another agent's step.
- Do NOT re-invoke an agent that already completed its step successfully.
- IGNORE agent-level completion language (e.g. "all steps are complete",
"onboarding is done"). An individual agent only knows about its own domain.
The workflow is NOT complete until every plan-step agent has been invoked."""

if has_user_responses:
progress_append += """

USER-CLARIFICATION EXECUTION RULES:
- MagenticManager MUST NOT ask questions or list missing info — it only routes.
- There is NO UserInteractionAgent. Do NOT select it as next_speaker.
- Domain agents that need user info will call their request_user_clarification
tool. The framework handles the pause/resume automatically via
Expand All @@ -168,26 +224,11 @@ def get_magentic_prompt_kwargs(*, has_user_responses: bool = False) -> dict:
STALL DETECTION OVERRIDE:
- An agent calling request_user_clarification is NOT stalling. The framework
pauses automatically. Set is_progress_being_made=true and is_in_loop=false.
- Do NOT treat a framework pause as a stall or loop.
- Do NOT treat a framework pause as a stall or loop."""

COMPLETION CHECK (CRITICAL):
Before setting is_request_satisfied to true, you MUST verify:
1. Review the conversation history and list every agent that has actually produced
a substantive response (called tools and returned results).
2. Compare that list against the plan steps. If ANY plan-step agent has NOT been
invoked and produced a substantive response, set is_request_satisfied to false
and select the next uninvoked agent as next_speaker.
3. is_request_satisfied = true ONLY when ALL plan-step agents have completed
their work successfully (called their tools, returned results).
- Each agent handles a DISTINCT domain. One agent's output does NOT satisfy
another agent's step.
- Do NOT re-invoke an agent that already completed its step successfully.
- IGNORE agent-level completion language (e.g. "all steps are complete",
"onboarding is done"). An individual agent only knows about its own domain.
The workflow is NOT complete until every plan-step agent has been invoked."""
kwargs["progress_ledger_prompt"] = (
ORCHESTRATOR_PROGRESS_LEDGER_PROMPT + progress_append
)
kwargs["progress_ledger_prompt"] = (
ORCHESTRATOR_PROGRESS_LEDGER_PROMPT + progress_append
)

return kwargs

Expand Down
34 changes: 33 additions & 1 deletion src/tests/backend/orchestration/test_plan_review_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,9 @@ def test_given_no_user_responses_when_called_then_returns_base_keys(self):
assert "task_ledger_plan_update_prompt" in result
assert "final_answer_prompt" in result
assert "task_ledger_facts_prompt" not in result
assert "progress_ledger_prompt" not in result
# progress_ledger_prompt (completion enforcement) is now always present,
# so plan-step agents must run even for teams without user_responses.
assert "progress_ledger_prompt" in result

def test_given_user_responses_when_called_then_returns_extended_keys(self):
# Act
Expand Down Expand Up @@ -198,6 +200,36 @@ def test_given_user_responses_when_called_then_progress_contains_execution_rules
assert "EXECUTION RULES" in result["progress_ledger_prompt"]
assert "COMPLETION CHECK" in result["progress_ledger_prompt"]

def test_given_no_user_responses_when_called_then_progress_still_enforces_completion(self):
# Act
result = get_magentic_prompt_kwargs(has_user_responses=False)

# Assert — completion enforcement applies even without user_responses
assert "progress_ledger_prompt" in result
assert "COMPLETION CHECK" in result["progress_ledger_prompt"]
# User-clarification-only rules must NOT leak in for non-interactive teams
assert "request_user_clarification" not in result["progress_ledger_prompt"]

def test_given_participant_names_when_called_then_plan_lists_mandatory_agents(self):
# Act
result = get_magentic_prompt_kwargs(
has_user_responses=False,
participant_names=["TriageAgent", "ComplianceAgent"],
)

# Assert — every listed agent is required to appear in the plan
plan_prompt = result["task_ledger_plan_prompt"]
assert "MANDATORY AGENTS" in plan_prompt
assert "- TriageAgent" in plan_prompt
assert "- ComplianceAgent" in plan_prompt

def test_given_no_participant_names_when_called_then_no_mandatory_block(self):
# Act
result = get_magentic_prompt_kwargs(has_user_responses=False)

# Assert
assert "MANDATORY AGENTS" not in result["task_ledger_plan_prompt"]

def test_given_no_user_responses_when_called_then_final_has_answer_rules(self):
# Act
result = get_magentic_prompt_kwargs(has_user_responses=False)
Expand Down
Loading