|
| 1 | +"""Step 1 must actually run verify_orchestration_prompt on what it emits. |
| 2 | +
|
| 3 | +The function was written, exported and unit-tested, and then called by nothing. |
| 4 | +Its sibling verify_orchestrator_result was deleted for that reason plus a false |
| 5 | +premise; this one encodes a true contract, so it was wired in instead. |
| 6 | +
|
| 7 | +A wire-in needs a test that would notice the wire coming loose, otherwise it |
| 8 | +recreates the original problem in a new place -- a validator that exists, passes |
| 9 | +its own unit tests, and never sees production data. So every assertion here goes |
| 10 | +through step1_task_analysis_node rather than calling the verifier directly: |
| 11 | +tests/test_schema_verifier.py already covers the function in isolation, and that |
| 12 | +coverage is exactly what failed to notice it had no callers. |
| 13 | +
|
| 14 | +The two cases are a pair on purpose. A test that only asserts "a good prompt |
| 15 | +produces no warnings" passes just as well when the verifier is never invoked, |
| 16 | +because an unwired check and a satisfied check both produce an empty list. |
| 17 | +""" |
| 18 | + |
| 19 | +import sys |
| 20 | +from pathlib import Path |
| 21 | +from unittest.mock import patch |
| 22 | + |
| 23 | +_PROJECT_ROOT = Path(__file__).resolve().parent.parent |
| 24 | +if str(_PROJECT_ROOT) not in sys.path: |
| 25 | + sys.path.insert(0, str(_PROJECT_ROOT)) |
| 26 | + |
| 27 | +# Long enough to clear the 200-char floor and carrying the "Phase" keyword the |
| 28 | +# verifier looks for, so it stands in for a real master template. |
| 29 | +WELL_FORMED_PROMPT = "Phase A -- foundation. " + ("orchestration detail. " * 40) |
| 30 | + |
| 31 | +# Non-empty, so neither degraded path in the node fires and nothing else logs: |
| 32 | +# this is the case the wire-in exists for. |
| 33 | +MALFORMED_PROMPT = "do the thing" |
| 34 | + |
| 35 | + |
| 36 | +def _script_mock(prompt_text): |
| 37 | + """Return a call_execution_script stand-in yielding the given prompt.""" |
| 38 | + |
| 39 | + def _side_effect(script_name, args=None, model_tier=None, silence_interval=None): |
| 40 | + if script_name == "prompt_gen_expert_caller": |
| 41 | + return {"status": "SUCCESS", "llm_response": prompt_text, "prompt": "raw"} |
| 42 | + return {"status": "SUCCESS"} |
| 43 | + |
| 44 | + return _side_effect |
| 45 | + |
| 46 | + |
| 47 | +def _run_node(prompt_text): |
| 48 | + """Drive step1_task_analysis_node with a mocked prompt-gen response.""" |
| 49 | + from langgraph_engine.sdlc_pipeline.nodes.task_orchestration import step1_task_analysis_node |
| 50 | + |
| 51 | + state = {"user_message": "Add a new endpoint", "project_root": str(_PROJECT_ROOT)} |
| 52 | + with patch( |
| 53 | + "langgraph_engine.sdlc_pipeline.helpers.call_execution_script", |
| 54 | + side_effect=_script_mock(prompt_text), |
| 55 | + ): |
| 56 | + return step1_task_analysis_node(state) |
| 57 | + |
| 58 | + |
| 59 | +class TestTheVerifierRunsOnWhatIsEmitted: |
| 60 | + """The check must reach the emitted prompt, not merely exist.""" |
| 61 | + |
| 62 | + def test_a_well_formed_prompt_records_no_warnings(self): |
| 63 | + result = _run_node(WELL_FORMED_PROMPT) |
| 64 | + orch = result["orchestrator_result"] |
| 65 | + assert "prompt_warnings" in orch, "the verifier result must reach orchestrator_result" |
| 66 | + assert orch["prompt_warnings"] == [] |
| 67 | + |
| 68 | + def test_a_malformed_prompt_records_warnings(self): |
| 69 | + """The half that fails if the verifier is never called. |
| 70 | +
|
| 71 | + A non-empty but unusable prompt is the case neither degraded path in the |
| 72 | + node covers -- prompt_gen returned something, so nothing else logs. If |
| 73 | + this list came back empty the check would be unwired, and the pass case |
| 74 | + above could not tell the difference. |
| 75 | + """ |
| 76 | + result = _run_node(MALFORMED_PROMPT) |
| 77 | + warnings = result["orchestrator_result"]["prompt_warnings"] |
| 78 | + assert warnings, "a 12-character prompt with no 'Phase' keyword must be reported" |
| 79 | + joined = " ".join(warnings).lower() |
| 80 | + assert "short" in joined or "200" in joined |
| 81 | + assert "phase" in joined |
| 82 | + |
| 83 | + |
| 84 | +class TestWarningsAreNotFatal: |
| 85 | + """A questionable prompt is reported, not turned into a failed run.""" |
| 86 | + |
| 87 | + def test_the_step_still_emits_and_reports_success(self): |
| 88 | + """The raw-task fallback trips both checks by design. |
| 89 | +
|
| 90 | + STEP1_CONTRACT documents a short raw task as a legitimate degraded path, |
| 91 | + so a verifier that failed the step would convert every recoverable run |
| 92 | + into a dead one. |
| 93 | + """ |
| 94 | + result = _run_node(MALFORMED_PROMPT) |
| 95 | + orch = result["orchestrator_result"] |
| 96 | + assert orch["prompt_warnings"] |
| 97 | + assert orch["success"] is True |
| 98 | + assert orch["mode"] == "emitted" |
| 99 | + assert result["orchestration_prompt"] == MALFORMED_PROMPT |
| 100 | + |
| 101 | + def test_prompt_chars_still_counts_the_full_emitted_prompt(self): |
| 102 | + """Guards against the wire-in being placed where it truncates the result.""" |
| 103 | + result = _run_node(WELL_FORMED_PROMPT) |
| 104 | + assert result["orchestrator_result"]["prompt_chars"] > len(WELL_FORMED_PROMPT) |
0 commit comments